rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-16 00:42:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(event): add event context to procedure events (#8734)

Browse Source 
* feat(event): add trigger context to procedure events

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test(event): fix trigger context event contracts

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(event): preserve trigger context origins

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(event): preserve lifecycle trigger contexts

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(meta): gate enterprise trigger imports

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test(event): assert trigger contexts exactly

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(test): restore migration test literals

Signed-off-by: WenyXu <wenymedia@gmail.com>

* feat(event): add trigger context to non-table ddl

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(event): simplify trigger context encoding

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(event): preserve auto alter trigger reason

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(event): require explicit trigger context

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(event): derive trigger context at ddl boundary

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(event): propagate DDL trigger reasons

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(meta): resolve alter table rebase conflict

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test(meta): fix alter table trigger context setup

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(meta): pass trigger context to region migration

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test(event): fix unknown trigger context assertions

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(meta): centralize trigger context protocol mapping

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test(event): fix migration trigger context assertion

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(procedure): rename event runtime context

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(event): rename trigger context

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test(event): fix migration event context assertion

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>
(cherry picked from commit 1f1c9270a8)
This commit is contained in:
Weny Xu
2026-08-20 14:45:56 +08:00
committed by discord9
parent 065d0dfaad
commit 408929ffd8
93 changed files with 2556 additions and 621 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1 -1
View File
@@ -2645,6 +2645,7 @@ dependencies = [
"common-query",
"common-recordbatch",
"common-runtime",
"common-session",
"common-telemetry",
"common-test-util",
"common-time",
@@ -8404,7 +8405,6 @@ dependencies = [
"serde",
"serde_json",
"servers",
"session",
"snafu 0.8.6",
"sqlx",
"store-api",
src/common/event-recorder/src/event_table.rs
+3
View File
@@ -79,6 +79,9 @@ pub const TYPE_COLUMN: EventTableColumn =
/// The canonical event payload column.
pub const PAYLOAD_COLUMN: EventTableColumn =
EventTableColumn::json_binary("payload", ColumnDataType::Binary, SemanticType::Field);
/// The optional context describing why an event was triggered.
pub const EVENT_CONTEXT_COLUMN: EventTableColumn =
EventTableColumn::json_binary("event_context", ColumnDataType::Binary, SemanticType::Field);
/// The canonical event timestamp column.
pub const TIMESTAMP_COLUMN: EventTableColumn = EventTableColumn::new(
"timestamp",
src/common/meta/Cargo.toml
+1
View File
@@ -41,6 +41,7 @@ common-procedure.workspace = true
common-query.workspace = true
common-recordbatch.workspace = true
common-runtime.workspace = true
common-session.workspace = true
common-telemetry.workspace = true
common-time.workspace = true
common-wal.workspace = true
src/common/meta/src/ddl/alter_database.rs
+17 -6
View File
@@ -15,7 +15,7 @@
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Status,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status,
};
use common_telemetry::tracing::info;
use serde::{Deserialize, Serialize};
@@ -32,7 +32,7 @@ use crate::key::DeserializedValueWithBytes;
use crate::key::schema_name::{SchemaName, SchemaNameKey, SchemaNameValue};
use crate::lock_key::{CatalogLock, SchemaLock};
use crate::rpc::ddl::UnsetDatabaseOption::{self};
use crate::rpc::ddl::{AlterDatabaseKind, AlterDatabaseTask, SetDatabaseOption};
use crate::rpc::ddl::{AlterDatabaseKind, AlterDatabaseTask, EventContext, SetDatabaseOption};
pub struct AlterDatabaseProcedure {
pub context: DdlContext,
@@ -73,10 +73,14 @@ fn build_new_schema_value(
impl AlterDatabaseProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterDatabase";
pub fn new(task: AlterDatabaseTask, context: DdlContext) -> Result<Self> {
pub fn new(
task: AlterDatabaseTask,
event_context: EventContext,
context: DdlContext,
) -> Result<Self> {
Ok(Self {
context,
data: AlterDatabaseData::new(task)?,
data: AlterDatabaseData::new(task, event_context)?,
})
}
@@ -176,7 +180,10 @@ impl Procedure for AlterDatabaseProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(ALTER_DATABASE_EVENT_TYPE) {
return None;
}
@@ -186,6 +193,7 @@ impl Procedure for AlterDatabaseProcedure {
self.data.catalog(),
self.data.schema(),
&self.data.kind,
self.data.event_context.clone(),
)
} else {
DatabaseDdlEvent::alter_lifecycle()
@@ -209,16 +217,19 @@ pub struct AlterDatabaseData {
catalog_name: String,
schema_name: String,
schema_value: Option<DeserializedValueWithBytes<SchemaNameValue>>,
#[serde(default)]
event_context: EventContext,
}
impl AlterDatabaseData {
pub fn new(task: AlterDatabaseTask) -> Result<Self> {
pub fn new(task: AlterDatabaseTask, event_context: EventContext) -> Result<Self> {
Ok(Self {
state: AlterDatabaseState::Prepare,
kind: AlterDatabaseKind::try_from(task.alter_expr.kind.unwrap())?,
catalog_name: task.alter_expr.catalog_name,
schema_name: task.alter_expr.schema_name,
schema_value: None,
event_context,
})
}
src/common/meta/src/ddl/alter_logical_tables.rs
+11 -3
View File
@@ -20,7 +20,7 @@ use api::region::RegionResponse;
use async_trait::async_trait;
use common_catalog::format_full_table_name;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{Context, EventContext, EventTrigger, LockKey, Procedure, Status};
use common_procedure::{Context, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status};
use common_telemetry::{debug, error, info, warn};
pub use executor::make_alter_region_request;
use serde::{Deserialize, Serialize};
@@ -47,7 +47,7 @@ use crate::key::table_info::TableInfoValue;
use crate::key::table_route::PhysicalTableRouteValue;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
use crate::metrics;
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::router::RegionRoute;
pub struct AlterLogicalTablesProcedure {
@@ -88,6 +88,7 @@ impl AlterLogicalTablesProcedure {
pub fn new(
tasks: Vec<AlterTableTask>,
physical_table_id: TableId,
event_context: EventContext,
context: DdlContext,
) -> Self {
Self {
@@ -100,6 +101,7 @@ impl AlterLogicalTablesProcedure {
physical_table_info: None,
physical_columns: vec![],
table_cache_keys_to_invalidate: vec![],
event_context,
},
physical_table_route: None,
}
@@ -320,7 +322,10 @@ impl Procedure for AlterLogicalTablesProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::AlterLogicalTables.as_str())
@@ -348,6 +353,7 @@ impl Procedure for AlterLogicalTablesProcedure {
locators,
self.data.tasks.len(),
kinds,
self.data.event_context.clone(),
)))
}
}
@@ -364,6 +370,8 @@ pub struct AlterTablesData {
physical_table_info: Option<DeserializedValueWithBytes<TableInfoValue>>,
physical_columns: Vec<ColumnMetadata>,
table_cache_keys_to_invalidate: Vec<CacheIdent>,
#[serde(default)]
event_context: EventContext,
}
impl AlterTablesData {
src/common/meta/src/ddl/alter_table.rs
+25 -8
View File
@@ -27,7 +27,7 @@ use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE};
use common_error::ext::BoxedError;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, ContextProvider, Error as ProcedureError, EventContext,
Context as ProcedureContext, ContextProvider, Error as ProcedureError, EventRuntimeContext,
EventTrigger, LockKey, PoisonKey, PoisonKeys, Procedure, ProcedureId, Status, StringKey,
};
use common_telemetry::{error, info, warn};
@@ -58,7 +58,7 @@ use crate::key::{DeserializedValueWithBytes, RegionDistribution};
use crate::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock, TableNameLock};
use crate::metrics;
use crate::poison_key::table_poison_key;
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::router::{RegionRoute, find_leaders, region_distribution};
/// The alter table procedure
@@ -97,18 +97,24 @@ fn build_executor_from_alter_expr(alter_data: &AlterTableData) -> AlterTableExec
impl AlterTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterTable";
pub fn new(table_id: TableId, task: AlterTableTask, context: DdlContext) -> Result<Self> {
Self::new_with_region_locks(table_id, task, vec![], context)
pub fn new(
table_id: TableId,
task: AlterTableTask,
event_context: EventContext,
context: DdlContext,
) -> Result<Self> {
Self::new_with_region_locks(table_id, task, event_context, vec![], context)
}
pub(crate) fn new_with_region_locks(
table_id: TableId,
task: AlterTableTask,
event_context: EventContext,
region_locks: Vec<RegionId>,
context: DdlContext,
) -> Result<Self> {
task.validate()?;
let data = AlterTableData::new(task, table_id, region_locks);
let data = AlterTableData::new(task, table_id, event_context, region_locks);
let executor = build_executor_from_alter_expr(&data);
Ok(Self {
context,
@@ -563,7 +569,10 @@ impl Procedure for AlterTableProcedure {
PoisonKeys::new(vec![self.table_poison_key()])
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::AlterTable.as_str())
@@ -583,7 +592,7 @@ impl Procedure for AlterTableProcedure {
.kind
.as_ref()
.and_then(alter_table_kind_name);
TableDdlEvent::alter_table_submitted(locator, kind)
TableDdlEvent::alter_table_submitted(locator, kind, self.data.event_context.clone())
}
_ => TableDdlEvent::lifecycle(TableDdlEventType::AlterTable),
};
@@ -611,6 +620,8 @@ pub struct AlterTableData {
task: AlterTableTask,
table_id: TableId,
#[serde(default)]
event_context: EventContext,
#[serde(default)]
column_metadatas: Vec<ColumnMetadata>,
/// Table info value before alteration.
table_info_value: Option<DeserializedValueWithBytes<TableInfoValue>>,
@@ -622,11 +633,17 @@ pub struct AlterTableData {
}
impl AlterTableData {
pub fn new(task: AlterTableTask, table_id: TableId, region_locks: Vec<RegionId>) -> Self {
pub fn new(
task: AlterTableTask,
table_id: TableId,
event_context: EventContext,
region_locks: Vec<RegionId>,
) -> Self {
Self {
state: AlterTableState::Prepare,
task,
table_id,
event_context,
column_metadatas: vec![],
table_info_value: None,
region_distribution: None,
src/common/meta/src/ddl/alter_table/region_request.rs
+5 -3
View File
@@ -144,7 +144,7 @@ mod tests {
};
use crate::key::table_route::TableRouteValue;
use crate::peer::Peer;
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::router::{Region, RegionRoute};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -240,7 +240,8 @@ mod tests {
},
};
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context).unwrap();
procedure.on_prepare().await.unwrap();
let alter_kind = procedure.make_region_alter_kind().unwrap();
let Some(Body::Alter(alter_region_request)) =
@@ -296,7 +297,8 @@ mod tests {
},
};
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context).unwrap();
procedure.on_prepare().await.unwrap();
let alter_kind = procedure.make_region_alter_kind().unwrap();
let Some(Body::Alter(alter_region_request)) =
src/common/meta/src/ddl/create_database.rs
+15 -4
View File
@@ -19,8 +19,8 @@ use async_trait::async_trait;
use common_error::ext::{BoxedError, ErrorExt};
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureId,
Status,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureId, Status,
};
use serde::{Deserialize, Serialize};
use serde_with::{DefaultOnNull, serde_as};
@@ -34,7 +34,7 @@ use crate::error::{self, Result};
use crate::instruction::{CacheIdent, UserCacheIdent};
use crate::key::schema_name::{SchemaNameKey, SchemaNameValue};
use crate::lock_key::{CatalogLock, SchemaLock};
use crate::rpc::ddl::CreatorGrantIntent;
use crate::rpc::ddl::{CreatorGrantIntent, EventContext};
/// Describes the creator-access result of an atomic create.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -93,6 +93,7 @@ impl CreateDatabaseProcedure {
create_if_not_exists: bool,
options: HashMap<String, String>,
creator: Option<CreatorGrantIntent>,
event_context: EventContext,
context: DdlContext,
) -> Self {
Self {
@@ -104,6 +105,7 @@ impl CreateDatabaseProcedure {
create_if_not_exists,
options,
creator,
event_context,
},
}
}
@@ -278,7 +280,10 @@ impl Procedure for CreateDatabaseProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(CREATE_DATABASE_EVENT_TYPE) {
return None;
}
@@ -289,6 +294,7 @@ impl Procedure for CreateDatabaseProcedure {
&self.data.schema,
self.data.create_if_not_exists,
&self.data.options,
self.data.event_context.clone(),
)
} else {
DatabaseDdlEvent::create_lifecycle()
@@ -321,6 +327,8 @@ pub struct CreateDatabaseData {
/// Authenticated creator whose access is ensured, absent for legacy schema-only requests.
#[serde(default)]
pub creator: Option<CreatorGrantIntent>,
#[serde(default)]
pub event_context: EventContext,
}
#[cfg(test)]
@@ -415,6 +423,7 @@ mod tests {
username: "alice".to_string(),
created_at_ns: 1,
}),
EventContext::default(),
context,
);
(procedure, committer)
@@ -574,6 +583,7 @@ mod tests {
false,
HashMap::new(),
None,
EventContext::default(),
context,
);
procedure.data.state = CreateDatabaseState::CreateMetadata;
@@ -609,6 +619,7 @@ mod tests {
username: "alice".to_string(),
created_at_ns: 1,
}),
EventContext::default(),
context,
);
procedure.data.state = CreateDatabaseState::CreateMetadata;
src/common/meta/src/ddl/create_flow.rs
+17 -5
View File
@@ -24,8 +24,8 @@ use async_trait::async_trait;
use common_catalog::format_full_flow_name;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState,
Result as ProcedureResult, Status,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureState, Result as ProcedureResult, Status,
};
use common_telemetry::info;
use common_telemetry::tracing_context::TracingContext;
@@ -50,7 +50,7 @@ use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId};
use crate::lock_key::{CatalogLock, FlowNameLock};
use crate::metrics;
use crate::peer::Peer;
use crate::rpc::ddl::{CreateFlowTask, FlowQueryContext, QueryContext};
use crate::rpc::ddl::{CreateFlowTask, EventContext, FlowQueryContext, QueryContext};
/// The procedure of flow creation.
pub struct CreateFlowProcedure {
@@ -62,7 +62,12 @@ impl CreateFlowProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateFlow";
/// Returns a new [CreateFlowProcedure].
pub fn new(task: CreateFlowTask, query_context: QueryContext, context: DdlContext) -> Self {
pub fn new(
task: CreateFlowTask,
query_context: QueryContext,
event_context: EventContext,
context: DdlContext,
) -> Self {
Self {
context,
data: CreateFlowData {
@@ -76,6 +81,7 @@ impl CreateFlowProcedure {
prev_flow_info_value: None,
did_replace: false,
flow_type: None,
event_context,
},
}
}
@@ -404,7 +410,10 @@ impl Procedure for CreateFlowProcedure {
])
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(CREATE_FLOW_EVENT_TYPE) {
return None;
}
@@ -419,6 +428,7 @@ impl Procedure for CreateFlowProcedure {
expire_after: self.data.task.expire_after,
eval_interval_secs: self.data.task.eval_interval_secs,
},
self.data.event_context.clone(),
),
EventTrigger::Succeeded => {
let flow_id = match ctx.lifecycle_state {
@@ -701,6 +711,8 @@ pub struct CreateFlowData {
#[serde(default)]
pub(crate) did_replace: bool,
pub(crate) flow_type: Option<FlowType>,
#[serde(default)]
pub(crate) event_context: EventContext,
}
impl CreateFlowData {
src/common/meta/src/ddl/create_logical_tables.rs
+13 -5
View File
@@ -24,8 +24,8 @@ use common_catalog::consts::METRIC_ENGINE;
use common_event_recorder::Event;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState,
Status,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureState, Status,
};
use common_telemetry::{debug, error, warn};
use futures::future;
@@ -48,7 +48,7 @@ use crate::error::Result;
use crate::key::table_route::TableRouteValue;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
use crate::metrics;
use crate::rpc::ddl::CreateTableTask;
use crate::rpc::ddl::{CreateTableTask, EventContext};
use crate::rpc::router::{RegionRoute, find_leaders};
pub struct CreateLogicalTablesProcedure {
@@ -62,6 +62,7 @@ impl CreateLogicalTablesProcedure {
pub fn new(
tasks: Vec<CreateTableTask>,
physical_table_id: TableId,
event_context: EventContext,
context: DdlContext,
) -> Self {
Self {
@@ -74,6 +75,7 @@ impl CreateLogicalTablesProcedure {
physical_region_numbers: vec![],
physical_columns: vec![],
physical_partition_columns: vec![],
event_context,
},
}
}
@@ -258,7 +260,7 @@ impl Procedure for CreateLogicalTablesProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::CreateLogicalTables.as_str())
@@ -275,7 +277,11 @@ impl Procedure for CreateLogicalTablesProcedure {
)
.with_physical_table_id(self.data.physical_table_id)
});
TableDdlEvent::create_logical_tables_submitted(locators, self.data.tasks.len())
TableDdlEvent::create_logical_tables_submitted(
locators,
self.data.tasks.len(),
self.data.event_context.clone(),
)
}
EventTrigger::Succeeded => match ctx.lifecycle_state {
ProcedureState::Done {
@@ -321,6 +327,8 @@ pub struct CreateTablesData {
physical_region_numbers: Vec<RegionNumber>,
physical_columns: Vec<ColumnMetadata>,
physical_partition_columns: Vec<String>,
#[serde(default)]
event_context: EventContext,
}
impl CreateTablesData {
src/common/meta/src/ddl/create_table.rs
+23 -11
View File
@@ -23,8 +23,8 @@ use common_procedure::error::{
};
use common_procedure::local::DynamicKeyLockGuard;
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureId,
ProcedureState, Status,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureId, ProcedureState, Status,
};
use common_telemetry::info;
use serde::{Deserialize, Serialize};
@@ -47,7 +47,7 @@ use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock};
use crate::metrics;
use crate::peer::PeerAllocContext;
use crate::region_keeper::OperatingRegionGuard;
use crate::rpc::ddl::{CreateTableTask, QueryContext};
use crate::rpc::ddl::{CreateTableTask, EventContext, QueryContext};
use crate::rpc::router::{RegionRoute, operating_leader_region_roles};
use crate::wal_provider::{
RegionWalOptions, acquire_remote_wal_read_locks, optional_region_wal_options_serde,
@@ -84,20 +84,17 @@ fn build_executor_from_create_table_data(
impl CreateTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateTable";
pub fn new(task: CreateTableTask, context: DdlContext) -> Result<Self> {
Self::new_with_query_context(task, QueryContext::default(), context)
}
pub fn new_with_query_context(
pub fn new(
task: CreateTableTask,
query_context: QueryContext,
event_context: EventContext,
context: DdlContext,
) -> Result<Self> {
let executor = build_executor_from_create_table_data(&task.create_table)?;
Ok(Self {
context,
data: CreateTableData::new(task, query_context),
data: CreateTableData::new_with_query_context(task, event_context, query_context),
opening_regions: vec![],
executor,
remote_wal_lock_guards: vec![],
@@ -399,7 +396,10 @@ impl Procedure for CreateTableProcedure {
])
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::CreateTable.as_str())
@@ -416,6 +416,7 @@ impl Procedure for CreateTableProcedure {
locator,
create_table.create_if_not_exists,
&create_table.engine,
self.data.event_context.clone(),
)
}
EventTrigger::Succeeded => match ctx.lifecycle_state {
@@ -452,6 +453,8 @@ pub struct CreateTableData {
#[serde(default)]
pub query_context: QueryContext,
#[serde(default)]
pub event_context: EventContext,
#[serde(default)]
pub column_metadatas: Vec<ColumnMetadata>,
/// None stands for not allocated yet.
pub(crate) table_route: Option<PhysicalTableRouteValue>,
@@ -462,12 +465,21 @@ pub struct CreateTableData {
}
impl CreateTableData {
pub fn new(task: CreateTableTask, query_context: QueryContext) -> Self {
pub fn new(task: CreateTableTask, event_context: EventContext) -> Self {
Self::new_with_query_context(task, event_context, QueryContext::default())
}
pub fn new_with_query_context(
task: CreateTableTask,
event_context: EventContext,
query_context: QueryContext,
) -> Self {
CreateTableData {
state: CreateTableState::Prepare,
column_metadatas: vec![],
task,
query_context,
event_context,
table_route: None,
region_wal_options: None,
}
src/common/meta/src/ddl/create_view.rs
+9 -5
View File
@@ -16,8 +16,8 @@ use async_trait::async_trait;
use common_event_recorder::Event;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState,
Status,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureState, Status,
};
use common_telemetry::info;
use serde::{Deserialize, Serialize};
@@ -35,7 +35,7 @@ use crate::instruction::CacheIdent;
use crate::key::table_name::TableNameKey;
use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock};
use crate::metrics;
use crate::rpc::ddl::CreateViewTask;
use crate::rpc::ddl::{CreateViewTask, EventContext};
// The procedure to execute `[CreateViewTask]`.
pub struct CreateViewProcedure {
@@ -46,13 +46,14 @@ pub struct CreateViewProcedure {
impl CreateViewProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateView";
pub fn new(task: CreateViewTask, context: DdlContext) -> Self {
pub fn new(task: CreateViewTask, event_context: EventContext, context: DdlContext) -> Self {
Self {
context,
data: CreateViewData {
state: CreateViewState::Prepare,
task,
need_update: false,
event_context,
},
}
}
@@ -271,7 +272,7 @@ impl Procedure for CreateViewProcedure {
])
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
if !ctx.event_type_filter.allows(CREATE_VIEW_EVENT_TYPE) {
return None;
}
@@ -289,6 +290,7 @@ impl Procedure for CreateViewProcedure {
referenced_table_count: self.data.task.table_names().len(),
column_count: self.data.task.columns().len(),
},
self.data.event_context.clone(),
)
}
EventTrigger::Succeeded => match ctx.lifecycle_state {
@@ -321,6 +323,8 @@ pub struct CreateViewData {
pub task: CreateViewTask,
/// Whether to update the view info.
pub need_update: bool,
#[serde(default)]
pub event_context: EventContext,
}
impl CreateViewData {
src/common/meta/src/ddl/drop_database.rs
+22 -3
View File
@@ -23,7 +23,7 @@ use std::fmt::Debug;
use common_error::ext::BoxedError;
use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use futures::stream::BoxStream;
@@ -38,11 +38,13 @@ use crate::ddl::utils::map_to_procedure_error;
use crate::error::Result;
use crate::key::table_name::TableNameValue;
use crate::lock_key::{CatalogLock, SchemaLock};
use crate::rpc::ddl::EventContext;
pub struct DropDatabaseProcedure {
/// The context of procedure runtime.
runtime_context: DdlContext,
context: DropDatabaseContext,
event_context: EventContext,
state: Box<dyn State>,
}
@@ -85,9 +87,16 @@ pub(crate) trait State: Send + Debug {
impl DropDatabaseProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::DropDatabase";
pub fn new(catalog: String, schema: String, drop_if_exists: bool, context: DdlContext) -> Self {
pub fn new(
catalog: String,
schema: String,
drop_if_exists: bool,
event_context: EventContext,
context: DdlContext,
) -> Self {
Self {
runtime_context: context,
event_context,
context: DropDatabaseContext {
catalog,
schema,
@@ -104,11 +113,13 @@ impl DropDatabaseProcedure {
catalog,
schema,
drop_if_exists,
event_context,
state,
} = serde_json::from_str(json).context(FromJsonSnafu)?;
Ok(Self {
runtime_context,
event_context,
context: DropDatabaseContext {
catalog,
schema,
@@ -159,6 +170,7 @@ impl Procedure for DropDatabaseProcedure {
catalog: &self.context.catalog,
schema: &self.context.schema,
drop_if_exists: self.context.drop_if_exists,
event_context: &self.event_context,
state: self.state.as_ref(),
};
@@ -174,7 +186,10 @@ impl Procedure for DropDatabaseProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(DROP_DATABASE_EVENT_TYPE) {
return None;
}
@@ -184,6 +199,7 @@ impl Procedure for DropDatabaseProcedure {
&self.context.catalog,
&self.context.schema,
self.context.drop_if_exists,
self.event_context.clone(),
)
} else {
DatabaseDdlEvent::drop_lifecycle()
@@ -199,6 +215,7 @@ struct DropDatabaseData<'a> {
// The schema name
schema: &'a str,
drop_if_exists: bool,
event_context: &'a EventContext,
state: &'a dyn State,
}
@@ -209,5 +226,7 @@ struct DropDatabaseOwnedData {
// The schema name
schema: String,
drop_if_exists: bool,
#[serde(default)]
event_context: EventContext,
state: Box<dyn State>,
}
src/common/meta/src/ddl/drop_flow.rs
+11 -4
View File
@@ -21,7 +21,7 @@ use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::info;
@@ -41,7 +41,7 @@ use crate::key::flow::flow_info::FlowInfoValue;
use crate::key::flow::flow_route::FlowRouteValue;
use crate::lock_key::{CatalogLock, FlowLock};
use crate::metrics;
use crate::rpc::ddl::DropFlowTask;
use crate::rpc::ddl::{DropFlowTask, EventContext};
/// The procedure for dropping a flow.
pub struct DropFlowProcedure {
@@ -54,7 +54,7 @@ pub struct DropFlowProcedure {
impl DropFlowProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::DropFlow";
pub fn new(task: DropFlowTask, context: DdlContext) -> Self {
pub fn new(task: DropFlowTask, event_context: EventContext, context: DdlContext) -> Self {
Self {
context,
data: DropFlowData {
@@ -62,6 +62,7 @@ impl DropFlowProcedure {
task,
flow_info_value: None,
flow_route_values: vec![],
event_context,
},
}
}
@@ -222,7 +223,10 @@ impl Procedure for DropFlowProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(DROP_FLOW_EVENT_TYPE) {
return None;
}
@@ -233,6 +237,7 @@ impl Procedure for DropFlowProcedure {
&self.data.task.flow_name,
self.data.task.flow_id,
self.data.task.drop_if_exists,
self.data.event_context.clone(),
),
_ => FlowDdlEvent::drop_lifecycle(),
};
@@ -248,6 +253,8 @@ pub(crate) struct DropFlowData {
task: DropFlowTask,
pub(crate) flow_info_value: Option<FlowInfoValue>,
pub(crate) flow_route_values: Vec<FlowRouteValue>,
#[serde(default)]
pub(crate) event_context: EventContext,
}
/// The state of drop flow
src/common/meta/src/ddl/drop_table.rs
+15 -6
View File
@@ -22,8 +22,8 @@ use common_error::ext::BoxedError;
use common_event_recorder::Event;
use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, Result as ProcedureResult, Status,
Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger,
LockKey, Procedure, Result as ProcedureResult, Status,
};
use common_telemetry::info;
use common_telemetry::tracing::warn;
@@ -48,7 +48,7 @@ use crate::key::table_route::TableRouteValue;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
use crate::metrics;
use crate::region_keeper::OperatingRegionGuard;
use crate::rpc::ddl::DropTableTask;
use crate::rpc::ddl::{DropTableTask, EventContext};
use crate::rpc::router::{RegionRoute, operating_leader_region_roles};
#[cfg(feature = "enterprise")]
@@ -74,11 +74,12 @@ pub struct DropTableProcedure {
impl DropTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::DropTable";
pub fn new(task: DropTableTask, context: DdlContext) -> Self {
pub fn new(task: DropTableTask, context: DdlContext, event_context: EventContext) -> Self {
let data = DropTableData::new(
task,
cfg!(feature = "enterprise") && context.soft_drop_enabled,
context.soft_drop_retention,
event_context,
);
let executor = data.build_executor();
Self {
@@ -413,7 +414,7 @@ impl Procedure for DropTableProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::DropTable.as_str())
@@ -425,7 +426,11 @@ impl Procedure for DropTableProcedure {
let task = &self.data.task;
let locator = TableDdlLocator::new(&task.catalog, &task.schema, &task.table)
.with_table_id(task.table_id);
TableDdlEvent::drop_table_submitted(locator, task.drop_if_exists)
TableDdlEvent::drop_table_submitted(
locator,
task.drop_if_exists,
self.data.event_context.clone(),
)
}
_ => TableDdlEvent::lifecycle(TableDdlEventType::DropTable),
};
@@ -480,6 +485,8 @@ pub struct DropTableData {
pub soft_drop_retention_millis: Option<i64>,
#[serde(default)]
pub drop_generation: Option<String>,
#[serde(default)]
pub event_context: EventContext,
}
impl DropTableData {
@@ -487,6 +494,7 @@ impl DropTableData {
task: DropTableTask,
soft_drop_enabled: bool,
soft_drop_retention: Option<std::time::Duration>,
event_context: EventContext,
) -> Self {
Self {
state: DropTableState::Prepare,
@@ -500,6 +508,7 @@ impl DropTableData {
retention_expires_at: None,
soft_drop_retention_millis: soft_drop_retention_millis(soft_drop_retention),
drop_generation: None,
event_context,
}
}
src/common/meta/src/ddl/drop_view.rs
+8 -4
View File
@@ -16,7 +16,7 @@ use async_trait::async_trait;
use common_event_recorder::Event;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::info;
@@ -35,7 +35,7 @@ use crate::instruction::CacheIdent;
use crate::key::table_name::TableNameKey;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
use crate::metrics;
use crate::rpc::ddl::DropViewTask;
use crate::rpc::ddl::{DropViewTask, EventContext};
/// The procedure for dropping a view.
pub struct DropViewProcedure {
@@ -48,12 +48,13 @@ pub struct DropViewProcedure {
impl DropViewProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::DropView";
pub fn new(task: DropViewTask, context: DdlContext) -> Self {
pub fn new(task: DropViewTask, event_context: EventContext, context: DdlContext) -> Self {
Self {
context,
data: DropViewData {
state: DropViewState::Prepare,
task,
event_context,
},
}
}
@@ -213,7 +214,7 @@ impl Procedure for DropViewProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
if !ctx.event_type_filter.allows(DROP_VIEW_EVENT_TYPE) {
return None;
}
@@ -227,6 +228,7 @@ impl Procedure for DropViewProcedure {
table_ref.table,
self.data.view_id(),
self.data.task.drop_if_exists,
self.data.event_context.clone(),
)
}
_ => ViewDdlEvent::drop_lifecycle(),
@@ -241,6 +243,8 @@ impl Procedure for DropViewProcedure {
pub(crate) struct DropViewData {
state: DropViewState,
task: DropViewTask,
#[serde(default)]
event_context: EventContext,
}
impl DropViewData {
src/common/meta/src/ddl/event/database.rs
+22 -3
View File
@@ -19,13 +19,14 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN,
SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, column_schemas, nullable_string,
CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN,
SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, column_schemas, nullable_json,
nullable_string,
};
use serde::Serialize;
use snafu::ResultExt;
use crate::rpc::ddl::{AlterDatabaseKind, SetDatabaseOption, UnsetDatabaseOption};
use crate::rpc::ddl::{AlterDatabaseKind, EventContext, SetDatabaseOption, UnsetDatabaseOption};
pub(crate) const CREATE_DATABASE_EVENT_TYPE: &str = "create_database";
pub(crate) const ALTER_DATABASE_EVENT_TYPE: &str = "alter_database";
@@ -39,6 +40,7 @@ pub(crate) struct DatabaseDdlEvent {
catalog_name: Option<String>,
schema_name: Option<String>,
payload: Option<DatabaseDdlPayload>,
event_context: Option<EventContext>,
}
#[derive(Debug, Serialize)]
@@ -88,6 +90,7 @@ impl DatabaseDdlEvent {
schema_name: &str,
create_if_not_exists: bool,
options: &HashMap<String, String>,
event_context: EventContext,
) -> Self {
let mut options = options.iter().collect::<Vec<_>>();
options.sort_unstable_by_key(|(left, _)| *left);
@@ -107,6 +110,7 @@ impl DatabaseDdlEvent {
create_if_not_exists,
options,
}),
event_context,
)
}
@@ -114,6 +118,7 @@ impl DatabaseDdlEvent {
catalog_name: &str,
schema_name: &str,
kind: &AlterDatabaseKind,
event_context: EventContext,
) -> Self {
let intent = match kind {
AlterDatabaseKind::SetDatabaseOptions(options) => AlterDatabaseIntent::Set {
@@ -152,6 +157,7 @@ impl DatabaseDdlEvent {
version: PAYLOAD_VERSION,
intent,
}),
event_context,
)
}
@@ -159,6 +165,7 @@ impl DatabaseDdlEvent {
catalog_name: &str,
schema_name: &str,
drop_if_exists: bool,
event_context: EventContext,
) -> Self {
Self::submitted(
DROP_DATABASE_EVENT_TYPE,
@@ -168,6 +175,7 @@ impl DatabaseDdlEvent {
version: PAYLOAD_VERSION,
drop_if_exists,
}),
event_context,
)
}
@@ -188,12 +196,14 @@ impl DatabaseDdlEvent {
catalog_name: &str,
schema_name: &str,
payload: DatabaseDdlPayload,
event_context: EventContext,
) -> Self {
Self {
event_type,
catalog_name: Some(catalog_name.to_string()),
schema_name: Some(schema_name.to_string()),
payload: Some(payload),
event_context: Some(event_context),
}
}
@@ -203,6 +213,7 @@ impl DatabaseDdlEvent {
catalog_name: None,
schema_name: None,
payload: None,
event_context: None,
}
}
}
@@ -223,14 +234,22 @@ impl Event for DatabaseDdlEvent {
column_schemas([
&EVENT_TABLE_CATALOG_NAME_COLUMN,
&EVENT_TABLE_SCHEMA_NAME_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
Ok(vec![Row {
values: vec![
nullable_string(self.catalog_name.as_deref()),
nullable_string(self.schema_name.as_deref()),
nullable_json(event_context.as_ref()),
],
}])
}
src/common/meta/src/ddl/event/flow.rs
+23 -5
View File
@@ -19,12 +19,14 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, column_schemas, nullable_string,
nullable_value,
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, column_schemas,
nullable_json, nullable_string, nullable_value,
};
use serde::Serialize;
use snafu::ResultExt;
use crate::rpc::ddl::EventContext;
pub(crate) const CREATE_FLOW_EVENT_TYPE: &str = "create_flow";
pub(crate) const DROP_FLOW_EVENT_TYPE: &str = "drop_flow";
@@ -69,14 +71,15 @@ pub(crate) struct FlowDdlEvent {
flow_name: Option<String>,
flow_id: Option<u32>,
payload: Option<FlowDdlPayload>,
event_context: Option<EventContext>,
}
impl FlowDdlEvent {
/// Builds the bounded event emitted when creating a Flow is submitted.
pub(crate) fn create_submitted(
catalog_name: &str,
flow_name: &str,
intent: CreateFlowEventIntent,
event_context: EventContext,
) -> Self {
Self {
event_type: CREATE_FLOW_EVENT_TYPE,
@@ -90,15 +93,16 @@ impl FlowDdlEvent {
expire_after: intent.expire_after,
eval_interval_secs: intent.eval_interval_secs,
})),
event_context: Some(event_context),
}
}
/// Builds the bounded event emitted when dropping a Flow is submitted.
pub(crate) fn drop_submitted(
catalog_name: &str,
flow_name: &str,
flow_id: u32,
drop_if_exists: bool,
event_context: EventContext,
) -> Self {
Self {
event_type: DROP_FLOW_EVENT_TYPE,
@@ -109,6 +113,7 @@ impl FlowDdlEvent {
version: PAYLOAD_VERSION,
drop_if_exists,
})),
event_context: Some(event_context),
}
}
@@ -137,6 +142,7 @@ impl FlowDdlEvent {
flow_name: None,
flow_id: None,
payload: None,
event_context: None,
}
}
}
@@ -154,15 +160,27 @@ impl Event for FlowDdlEvent {
}
fn extra_schema(&self) -> Vec<ColumnSchema> {
column_schemas([&CATALOG_NAME_COLUMN, &FLOW_NAME_COLUMN, &FLOW_ID_COLUMN])
column_schemas([
&CATALOG_NAME_COLUMN,
&FLOW_NAME_COLUMN,
&FLOW_ID_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
Ok(vec![Row {
values: vec![
nullable_string(self.catalog_name.as_deref()),
nullable_string(self.flow_name.as_deref()),
nullable_value(self.flow_id.map(ValueData::U32Value)),
nullable_json(event_context.as_ref()),
],
}])
}
src/common/meta/src/ddl/event/table.rs
+49 -10
View File
@@ -21,14 +21,17 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN,
TABLE_NAME_COLUMN, column_schemas, nullable_string, nullable_value,
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN,
TABLE_ID_COLUMN, TABLE_NAME_COLUMN, column_schemas, nullable_json, nullable_string,
nullable_value,
};
use serde::Serialize;
use serde_json::Value as JsonValue;
use snafu::ResultExt;
use store_api::storage::TableId;
use crate::rpc::ddl::EventContext;
/// Current version of table DDL event payloads.
pub(crate) const TABLE_DDL_PAYLOAD_VERSION: u8 = 1;
@@ -211,6 +214,7 @@ pub(crate) struct TableDdlEvent {
event_type: TableDdlEventType,
locators: Vec<TableDdlLocator>,
payload: Option<TableDdlPayload>,
event_context: Option<EventContext>,
}
impl TableDdlEvent {
@@ -219,6 +223,7 @@ impl TableDdlEvent {
locator: TableDdlLocator,
create_if_not_exists: bool,
engine: &str,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::CreateTable,
@@ -228,6 +233,7 @@ impl TableDdlEvent {
create_if_not_exists,
engine: engine.to_string(),
}),
Some(event_context),
)
}
@@ -235,6 +241,7 @@ impl TableDdlEvent {
pub(crate) fn create_logical_tables_submitted(
locators: impl IntoIterator<Item = TableDdlLocator>,
table_count: usize,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::CreateLogicalTables,
@@ -243,6 +250,7 @@ impl TableDdlEvent {
version: TABLE_DDL_PAYLOAD_VERSION,
table_count,
}),
Some(event_context),
)
}
@@ -250,6 +258,7 @@ impl TableDdlEvent {
pub(crate) fn alter_table_submitted(
locator: TableDdlLocator,
kind: Option<&'static str>,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::AlterTable,
@@ -258,6 +267,7 @@ impl TableDdlEvent {
version: TABLE_DDL_PAYLOAD_VERSION,
kind,
}),
Some(event_context),
)
}
@@ -266,6 +276,7 @@ impl TableDdlEvent {
locators: impl IntoIterator<Item = TableDdlLocator>,
table_count: usize,
kinds: impl IntoIterator<Item = &'static str>,
event_context: EventContext,
) -> Self {
let kinds = kinds
.into_iter()
@@ -280,11 +291,16 @@ impl TableDdlEvent {
table_count,
kinds,
}),
Some(event_context),
)
}
/// Builds the bounded event emitted when dropping a table is submitted.
pub(crate) fn drop_table_submitted(locator: TableDdlLocator, drop_if_exists: bool) -> Self {
pub(crate) fn drop_table_submitted(
locator: TableDdlLocator,
drop_if_exists: bool,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::DropTable,
[locator],
@@ -292,30 +308,39 @@ impl TableDdlEvent {
version: TABLE_DDL_PAYLOAD_VERSION,
drop_if_exists,
}),
Some(event_context),
)
}
/// Builds the bounded event emitted when restoring a dropped table is submitted.
#[cfg(feature = "enterprise")]
pub(crate) fn undrop_table_submitted(locator: TableDdlLocator) -> Self {
pub(crate) fn undrop_table_submitted(
locator: TableDdlLocator,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::UndropTable,
[locator],
TableDdlPayload::UndropTable(UndropTablePayload {
version: TABLE_DDL_PAYLOAD_VERSION,
}),
Some(event_context),
)
}
/// Builds the bounded event emitted when purging a dropped table is submitted.
#[cfg(feature = "enterprise")]
pub(crate) fn purge_dropped_table_submitted(locator: TableDdlLocator) -> Self {
pub(crate) fn purge_dropped_table_submitted(
locator: TableDdlLocator,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::PurgeDroppedTable,
[locator],
TableDdlPayload::PurgeDroppedTable(PurgeDroppedTablePayload {
version: TABLE_DDL_PAYLOAD_VERSION,
}),
Some(event_context),
)
}
@@ -323,6 +348,7 @@ impl TableDdlEvent {
pub(crate) fn truncate_table_submitted(
locator: TableDdlLocator,
time_range_count: usize,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::TruncateTable,
@@ -331,6 +357,7 @@ impl TableDdlEvent {
version: TABLE_DDL_PAYLOAD_VERSION,
time_range_count,
}),
Some(event_context),
)
}
@@ -340,6 +367,7 @@ impl TableDdlEvent {
event_type,
locators: vec![TableDdlLocator::default()],
payload: None,
event_context: None,
}
}
@@ -349,6 +377,7 @@ impl TableDdlEvent {
event_type: TableDdlEventType::CreateTable,
locators: vec![TableDdlLocator::from_table_id(table_id)],
payload: None,
event_context: None,
}
}
@@ -360,6 +389,7 @@ impl TableDdlEvent {
event_type: TableDdlEventType::CreateLogicalTables,
locators: locators.into_iter().collect(),
payload: None,
event_context: None,
}
}
@@ -367,11 +397,13 @@ impl TableDdlEvent {
event_type: TableDdlEventType,
locators: impl IntoIterator<Item = TableDdlLocator>,
payload: TableDdlPayload,
event_context: Option<EventContext>,
) -> Self {
Self {
event_type,
locators: locators.into_iter().collect(),
payload: Some(payload),
event_context,
}
}
@@ -384,7 +416,7 @@ impl TableDdlEvent {
])
}
fn locator_row(&self, locator: &TableDdlLocator) -> Row {
fn locator_row(&self, locator: &TableDdlLocator) -> Result<Row> {
let mut values = vec![
nullable_string(locator.catalog_name.as_deref()),
nullable_string(locator.schema_name.as_deref()),
@@ -394,7 +426,14 @@ impl TableDdlEvent {
if self.event_type.has_physical_table_id() {
values.push(nullable_table_id(locator.physical_table_id));
}
Row { values }
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
values.push(nullable_json(event_context.as_ref()));
Ok(Row { values })
}
}
@@ -415,15 +454,15 @@ impl Event for TableDdlEvent {
if self.event_type.has_physical_table_id() {
schema.push(PHYSICAL_TABLE_ID_COLUMN.column_schema());
}
schema.push(EVENT_CONTEXT_COLUMN.column_schema());
schema
}
fn extra_rows(&self) -> Result<Vec<Row>> {
Ok(self
.locators
self.locators
.iter()
.map(|locator| self.locator_row(locator))
.collect())
.collect()
}
fn as_any(&self) -> &dyn Any {
src/common/meta/src/ddl/event/view.rs
+21 -4
View File
@@ -19,12 +19,14 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, VIEW_NAME_COLUMN, column_schemas,
nullable_string, nullable_value,
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN,
VIEW_NAME_COLUMN, column_schemas, nullable_json, nullable_string, nullable_value,
};
use serde::Serialize;
use snafu::ResultExt;
use crate::rpc::ddl::EventContext;
pub(crate) const CREATE_VIEW_EVENT_TYPE: &str = "create_view";
pub(crate) const DROP_VIEW_EVENT_TYPE: &str = "drop_view";
@@ -62,6 +64,7 @@ pub(crate) struct ViewDdlEvent {
view_name: Option<String>,
view_id: Option<u32>,
payload: Option<ViewDdlPayload>,
event_context: Option<EventContext>,
}
#[derive(Debug, Serialize)]
@@ -72,12 +75,12 @@ enum ViewDdlPayload {
}
impl ViewDdlEvent {
/// Builds the bounded event emitted when creating a View is submitted.
pub(crate) fn create_submitted(
catalog_name: &str,
schema_name: &str,
view_name: &str,
intent: CreateViewEventIntent,
event_context: EventContext,
) -> Self {
Self::submitted(
CREATE_VIEW_EVENT_TYPE,
@@ -92,16 +95,17 @@ impl ViewDdlEvent {
referenced_table_count: intent.referenced_table_count,
column_count: intent.column_count,
}),
event_context,
)
}
/// Builds the bounded event emitted when dropping a View is submitted.
pub(crate) fn drop_submitted(
catalog_name: &str,
schema_name: &str,
view_name: &str,
view_id: u32,
drop_if_exists: bool,
event_context: EventContext,
) -> Self {
Self::submitted(
DROP_VIEW_EVENT_TYPE,
@@ -113,6 +117,7 @@ impl ViewDdlEvent {
version: PAYLOAD_VERSION,
drop_if_exists,
}),
event_context,
)
}
@@ -138,6 +143,7 @@ impl ViewDdlEvent {
view_name: &str,
view_id: Option<u32>,
payload: ViewDdlPayload,
event_context: EventContext,
) -> Self {
Self {
event_type,
@@ -146,6 +152,7 @@ impl ViewDdlEvent {
view_name: Some(view_name.to_string()),
view_id,
payload: Some(payload),
event_context: Some(event_context),
}
}
@@ -157,6 +164,7 @@ impl ViewDdlEvent {
view_name: None,
view_id: None,
payload: None,
event_context: None,
}
}
@@ -168,6 +176,7 @@ impl ViewDdlEvent {
view_name: None,
view_id: Some(view_id),
payload: None,
event_context: None,
}
}
}
@@ -190,16 +199,24 @@ impl Event for ViewDdlEvent {
&SCHEMA_NAME_COLUMN,
&VIEW_NAME_COLUMN,
&VIEW_ID_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
Ok(vec![Row {
values: vec![
nullable_string(self.catalog_name.as_deref()),
nullable_string(self.schema_name.as_deref()),
nullable_string(self.view_name.as_deref()),
nullable_value(self.view_id.map(ValueData::U32Value)),
nullable_json(event_context.as_ref()),
],
}])
}
src/common/meta/src/ddl/purge_dropped_table.rs
+28 -11
View File
@@ -14,7 +14,7 @@ use std::collections::HashMap;
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::info;
@@ -38,7 +38,7 @@ use crate::error::{self, Result};
use crate::key::DroppedTableMetadata;
use crate::key::table_route::TableRouteValue;
use crate::lock_key::TableLock;
use crate::rpc::ddl::PurgeDroppedTableTask;
use crate::rpc::ddl::{EventContext, PurgeDroppedTableTask};
use crate::rpc::router::RegionRoute;
pub struct PurgeDroppedTableProcedure {
@@ -50,17 +50,25 @@ impl PurgeDroppedTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::PurgeDroppedTable";
pub const EXPIRED_TYPE_NAME: &'static str = "metasrv-procedure::PurgeExpiredDroppedTable";
pub fn new(task: PurgeDroppedTableTask, context: DdlContext) -> Self {
pub fn new(
task: PurgeDroppedTableTask,
context: DdlContext,
event_context: EventContext,
) -> Self {
Self {
context,
data: PurgeDroppedTableData::new(task),
data: PurgeDroppedTableData::new(task, event_context),
}
}
pub fn new_if_expired(task: PurgeDroppedTableTask, context: DdlContext) -> Self {
pub fn new_if_expired(
task: PurgeDroppedTableTask,
context: DdlContext,
event_context: EventContext,
) -> Self {
Self {
context,
data: PurgeDroppedTableData::new_if_expired(task),
data: PurgeDroppedTableData::new_if_expired(task, event_context),
}
}
@@ -256,7 +264,10 @@ impl Procedure for PurgeDroppedTableProcedure {
LockKey::new(vec![TableLock::Write(self.data.task.table_id).into()])
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::PurgeDroppedTable.as_str())
@@ -266,7 +277,10 @@ impl Procedure for PurgeDroppedTableProcedure {
let event = match &ctx.trigger {
EventTrigger::Submitted => {
let locator = TableDdlLocator::from_table_id(self.data.task.table_id);
TableDdlEvent::purge_dropped_table_submitted(locator)
TableDdlEvent::purge_dropped_table_submitted(
locator,
self.data.event_context.clone(),
)
}
_ => TableDdlEvent::lifecycle(TableDdlEventType::PurgeDroppedTable),
};
@@ -293,10 +307,12 @@ pub struct PurgeDroppedTableData {
drop_generation_loaded: bool,
#[serde(default)]
purging_claimed: bool,
#[serde(default)]
event_context: EventContext,
}
impl PurgeDroppedTableData {
fn new(task: PurgeDroppedTableTask) -> Self {
fn new(task: PurgeDroppedTableTask, event_context: EventContext) -> Self {
Self {
state: PurgeDroppedTableState::Prepare,
task,
@@ -309,13 +325,14 @@ impl PurgeDroppedTableData {
drop_generation: None,
drop_generation_loaded: false,
purging_claimed: false,
event_context,
}
}
fn new_if_expired(task: PurgeDroppedTableTask) -> Self {
fn new_if_expired(task: PurgeDroppedTableTask, event_context: EventContext) -> Self {
Self {
check_expired: true,
..Self::new(task)
..Self::new(task, event_context)
}
}
src/common/meta/src/ddl/test_util.rs
+7 -2
View File
@@ -45,7 +45,7 @@ use crate::key::node_address::{NodeAddressKey, NodeAddressValue};
use crate::key::table_route::TableRouteValue;
use crate::key::{MetadataKey, MetadataValue};
use crate::peer::Peer;
use crate::rpc::ddl::CreateTableTask;
use crate::rpc::ddl::{CreateTableTask, EventContext};
use crate::rpc::store::PutRequest;
pub async fn create_physical_table_metadata(
@@ -104,7 +104,12 @@ pub async fn create_logical_table(
table_name: &str,
) -> TableId {
let tasks = vec![test_create_logical_table_task(table_name)];
let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let mut procedure = CreateLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
src/common/meta/src/ddl/tests/alter_logical_tables.rs
+39 -10
View File
@@ -43,7 +43,7 @@ use crate::error::Error::{AlterLogicalTablesInvalidArguments, TableNotFound};
use crate::error::Result;
use crate::key::table_name::TableNameKey;
use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue};
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::router::{Region, RegionRoute};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -162,7 +162,12 @@ async fn test_on_prepare_check_schema() {
),
];
let physical_table_id = 1024u32;
let mut procedure = AlterLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, AlterLogicalTablesInvalidArguments { .. });
}
@@ -177,7 +182,12 @@ async fn test_on_prepare_check_alter_kind() {
"new_table1",
)];
let physical_table_id = 1024u32;
let mut procedure = AlterLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, AlterLogicalTablesInvalidArguments { .. });
}
@@ -197,7 +207,8 @@ async fn test_on_prepare_different_physical_table() {
make_alter_logical_table_add_column_task(None, "table2", vec!["column2".to_string()]),
];
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy1_id, ddl_context);
let mut procedure =
AlterLogicalTablesProcedure::new(tasks, phy1_id, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, AlterLogicalTablesInvalidArguments { .. });
}
@@ -218,7 +229,8 @@ async fn test_on_prepare_logical_table_not_exists() {
make_alter_logical_table_add_column_task(None, "table2", vec!["column2".to_string()]),
];
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context);
let mut procedure =
AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, TableNotFound { .. });
}
@@ -241,7 +253,8 @@ async fn test_on_prepare() {
make_alter_logical_table_add_column_task(None, "table3", vec!["column3".to_string()]),
];
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context);
let mut procedure =
AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context);
let result = procedure.on_prepare().await;
assert_matches!(
result,
@@ -277,7 +290,12 @@ async fn test_on_update_metadata() {
make_alter_logical_table_add_column_task(None, "table3", vec!["new_col".to_string()]),
];
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone());
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
phy_id,
EventContext::default(),
ddl_context.clone(),
);
let mut status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -361,7 +379,12 @@ async fn test_on_part_duplicate_alter_request() {
make_alter_logical_table_add_column_task(None, "table2", vec!["col_0".to_string()]),
];
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone());
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
phy_id,
EventContext::default(),
ddl_context.clone(),
);
let mut status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -446,7 +469,12 @@ async fn test_on_part_duplicate_alter_request() {
),
];
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone());
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
phy_id,
EventContext::default(),
ddl_context.clone(),
);
let mut status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -617,7 +645,8 @@ async fn test_on_submit_alter_region_request() {
make_alter_logical_table_add_column_task(None, "table2", vec!["mew_col".to_string()]),
];
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context);
let mut procedure =
AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context);
procedure.on_prepare().await.unwrap();
procedure.on_submit_alter_region_requests().await.unwrap();
let mut results = Vec::new();
src/common/meta/src/ddl/tests/alter_table.rs
+56 -16
View File
@@ -59,7 +59,7 @@ use crate::lock_key::RegionLock;
use crate::node_manager::NodeManagerRef;
use crate::peer::Peer;
use crate::poison_key::table_poison_key;
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::router::{Region, RegionRoute};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -121,7 +121,8 @@ async fn test_on_prepare_table_exists_err() {
.unwrap();
let task = test_rename_alter_table_task("non-exists", "foo");
let mut procedure = AlterTableProcedure::new(1024, task, ddl_context).unwrap();
let mut procedure =
AlterTableProcedure::new(1024, task, EventContext::default(), ddl_context).unwrap();
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err.status_code(), StatusCode::TableAlreadyExists);
}
@@ -131,7 +132,8 @@ async fn test_on_prepare_table_not_exists_err() {
let node_manager = Arc::new(MockDatanodeManager::new(()));
let ddl_context = new_ddl_context(node_manager);
let task = test_rename_alter_table_task("non-exists", "foo");
let mut procedure = AlterTableProcedure::new(1024, task, ddl_context).unwrap();
let mut procedure =
AlterTableProcedure::new(1024, task, EventContext::default(), ddl_context).unwrap();
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err.status_code(), StatusCode::TableNotFound);
}
@@ -245,8 +247,13 @@ async fn test_on_submit_alter_request() {
let alter_table_task = test_alter_table_task(table_name);
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure =
AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap();
let mut procedure = AlterTableProcedure::new(
table_id,
alter_table_task,
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
procedure.on_prepare().await.unwrap();
procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -299,8 +306,13 @@ async fn test_on_submit_alter_request_without_sync_request() {
let alter_table_task = test_alter_table_task(table_name);
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure =
AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap();
let mut procedure = AlterTableProcedure::new(
table_id,
alter_table_task,
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
procedure.on_prepare().await.unwrap();
procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -357,7 +369,13 @@ async fn test_on_submit_alter_request_with_outdated_request() {
};
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure = AlterTableProcedure::new(table_id, alter_table_task, ddl_context).unwrap();
let mut procedure = AlterTableProcedure::new(
table_id,
alter_table_task,
EventContext::default(),
ddl_context,
)
.unwrap();
procedure.on_prepare().await.unwrap();
let err = procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -386,7 +404,9 @@ async fn test_on_update_metadata_rename() {
.unwrap();
let task = test_rename_alter_table_task(table_name, new_table_name);
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone())
.unwrap();
procedure.on_prepare().await.unwrap();
procedure.on_update_metadata().await.unwrap();
@@ -466,7 +486,9 @@ async fn test_on_update_metadata_add_columns() {
};
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone())
.unwrap();
procedure.on_prepare().await.unwrap();
procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -563,7 +585,9 @@ async fn test_on_update_table_options() {
};
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone())
.unwrap();
procedure.on_prepare().await.unwrap();
procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -634,6 +658,7 @@ async fn test_skip_wal_rejects_mixed_table_options() {
})),
},
},
EventContext::default(),
ddl_context,
)
.unwrap();
@@ -662,8 +687,13 @@ async fn test_skip_wal_rejects_logical_table() {
},
};
let mut procedure =
AlterTableProcedure::new(logical_table_id, task, ddl_context.clone()).unwrap();
let mut procedure = AlterTableProcedure::new(
logical_table_id,
task,
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let error = procedure.on_prepare().await.unwrap_err();
assert_matches!(error, Error::Unsupported { .. });
@@ -709,7 +739,9 @@ async fn test_skip_wal_rejects_file_engine_table() {
},
};
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone())
.unwrap();
let error = procedure.on_prepare().await.unwrap_err();
assert_matches!(error, Error::Unsupported { .. });
@@ -746,6 +778,7 @@ fn test_skip_wal_holds_region_locks() {
let procedure = AlterTableProcedure::new_with_region_locks(
table_id,
task,
EventContext::default(),
region_ids.clone(),
context.clone(),
)
@@ -797,6 +830,7 @@ async fn test_skip_wal_detects_region_route_change() {
let mut procedure = AlterTableProcedure::new_with_region_locks(
table_id,
alter_task,
EventContext::default(),
stale_region_locks,
ddl_context.clone(),
)
@@ -916,6 +950,7 @@ async fn test_skip_wal_updates_metadata_before_all_replicas() {
let mut procedure = AlterTableProcedure::new_with_region_locks(
table_id,
alter_task,
EventContext::default(),
region_locks,
ddl_context.clone(),
)
@@ -997,8 +1032,13 @@ async fn prepare_alter_table_procedure(
},
};
let procedure_id = ProcedureId::random();
let mut procedure =
AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap();
let mut procedure = AlterTableProcedure::new(
table_id,
alter_table_task,
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
procedure.on_prepare().await.unwrap();
(procedure, procedure_id)
}
src/common/meta/src/ddl/tests/create_flow.rs
+71 -16
View File
@@ -37,7 +37,7 @@ use crate::key::flow::flow_info::{
FlowInfoValue, FlowMissedTickPolicy, FlowScheduleConfig, FlowStatus,
};
use crate::key::table_route::TableRouteValue;
use crate::rpc::ddl::{CreateFlowTask, FlowQueryContext, QueryContext};
use crate::rpc::ddl::{CreateFlowTask, EventContext, FlowQueryContext, QueryContext};
use crate::test_util::{MockFlownodeManager, new_ddl_context};
pub(crate) fn test_query_context() -> QueryContext {
@@ -120,7 +120,8 @@ async fn test_create_flow_source_table_not_found() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let mut procedure =
CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
assert!(
@@ -143,7 +144,12 @@ async fn test_create_pending_flow_source_table_not_found_with_defer() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
let mut procedure = CreateFlowProcedure::new(
task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(status, Status::Executing { persist: true, .. });
assert_eq!(procedure.data.unresolved_source_table_names.len(), 1);
@@ -183,7 +189,8 @@ async fn test_create_pending_flow_source_table_not_found_with_defer_false() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let mut procedure =
CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
assert!(
@@ -233,7 +240,12 @@ async fn test_create_pending_flow_records_partial_source_resolution() {
);
enable_defer_on_missing_source(&mut task);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
let mut procedure = CreateFlowProcedure::new(
task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(status, Status::Executing { persist: true, .. });
assert_eq!(procedure.data.source_table_ids, vec![existing_table_id]);
@@ -428,6 +440,7 @@ fn test_resolved_schedule_defaults_in_create_request() {
// Construct CreateFlowData and verify CreateRequest carries internal key.
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateFlows,
task,
flow_id: Some(1024),
@@ -441,6 +454,7 @@ fn test_resolved_schedule_defaults_in_create_request() {
};
let data2 = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateMetadata,
task: data.task.clone(),
flow_id: Some(1024),
@@ -580,7 +594,8 @@ async fn test_create_flow_rejects_unknown_option_in_meta_task() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let mut procedure =
CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unexpected { .. });
@@ -601,6 +616,7 @@ fn test_create_request_strips_defer_on_missing_source_runtime_option() {
enable_defer_on_missing_source(&mut task);
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateFlows,
task,
flow_id: Some(1024),
@@ -650,7 +666,12 @@ pub(crate) async fn create_test_flow(
false,
);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context.clone());
let mut procedure = CreateFlowProcedure::new(
task.clone(),
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let output = execute_procedure_until_done(&mut procedure).await.unwrap();
let flow_id = output.downcast_ref::<FlowId>().unwrap();
@@ -671,7 +692,12 @@ pub(crate) async fn create_test_pending_flow(
);
enable_defer_on_missing_source(&mut task);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
let mut procedure = CreateFlowProcedure::new(
task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let output = execute_procedure_until_done(&mut procedure).await.unwrap();
let flow_id = output.downcast_ref::<FlowId>().unwrap();
@@ -718,7 +744,12 @@ async fn test_create_flow() {
true,
);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context.clone());
let mut procedure = CreateFlowProcedure::new(
task.clone(),
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let output = execute_procedure_until_done(&mut procedure).await.unwrap();
let flow_id = output.downcast_ref::<FlowId>().unwrap();
assert_eq!(*flow_id, 1024);
@@ -726,7 +757,12 @@ async fn test_create_flow() {
// Creates again
let task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context);
let mut procedure = CreateFlowProcedure::new(
task.clone(),
query_ctx,
EventContext::default(),
ddl_context,
);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowAlreadyExists { .. });
}
@@ -783,7 +819,12 @@ async fn test_replace_pending_flow_with_active_flow_is_unsupported() {
);
replace_task.or_replace = true;
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
let mut procedure = CreateFlowProcedure::new(
replace_task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
assert!(
@@ -841,7 +882,12 @@ async fn test_replace_active_flow_with_pending_flow_is_unsupported() {
enable_defer_on_missing_source(&mut replace_task);
replace_task.or_replace = true;
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
let mut procedure = CreateFlowProcedure::new(
replace_task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
assert!(
@@ -901,7 +947,12 @@ async fn test_replace_pending_flow_with_pending_flow_updates_metadata() {
enable_defer_on_missing_source(&mut replace_task);
replace_task.or_replace = true;
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
let mut procedure = CreateFlowProcedure::new(
replace_task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let output = execute_procedure_until_done(&mut procedure).await.unwrap();
let replaced_flow_id = *output.downcast_ref::<FlowId>().unwrap();
assert_eq!(replaced_flow_id, original_flow_id);
@@ -953,7 +1004,8 @@ async fn test_create_flow_same_source_and_sink_table() {
// Try to create a flow with same source and sink table - should fail
let task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let mut procedure =
CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
@@ -986,7 +1038,7 @@ fn test_create_flow_lock_key_does_not_lock_sink_table_name() {
let task = create_test_flow_task_for_serialization();
let query_ctx = test_query_context();
let ddl_context = new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)));
let procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let procedure = CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let lock_keys = procedure.lock_key().get_keys();
@@ -1061,6 +1113,7 @@ fn test_create_flow_data_new_format_serialization() {
};
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::Prepare,
task: create_test_flow_task_for_serialization(),
flow_id: None,
@@ -1129,7 +1182,7 @@ fn test_create_flow_procedure_strips_scheduled_time_extension() {
.extensions
.insert("flow.other".to_string(), "kept".to_string());
let procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let procedure = CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
assert!(
!procedure
@@ -1162,6 +1215,7 @@ fn test_flow_info_conversion_with_flow_context() {
};
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateMetadata,
task: create_test_flow_task_for_serialization(),
flow_id: Some(123),
@@ -1204,6 +1258,7 @@ fn test_flow_info_conversion_strips_scheduled_time_extension() {
};
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateMetadata,
task: create_test_flow_task_for_serialization(),
flow_id: Some(123),
src/common/meta/src/ddl/tests/create_logical_tables.rs
+30 -6
View File
@@ -39,6 +39,7 @@ use crate::ddl::test_util::{
};
use crate::error::{Error, Result};
use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue};
use crate::rpc::ddl::EventContext;
use crate::rpc::router::{Region, RegionRoute};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -96,7 +97,12 @@ async fn test_on_prepare_physical_table_not_found() {
let ddl_context = new_ddl_context(node_manager);
let tasks = vec![test_create_logical_table_task("foo")];
let physical_table_id = 1024u32;
let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let mut procedure = CreateLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableRouteNotFound { .. });
}
@@ -126,7 +132,12 @@ async fn test_on_prepare() {
// The create logical table procedure.
let tasks = vec![test_create_logical_table_task("foo")];
let physical_table_id = table_id;
let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let mut procedure = CreateLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -172,8 +183,12 @@ async fn test_on_prepare_logical_table_exists_err() {
.unwrap();
// The create logical table procedure.
let physical_table_id = table_id;
let mut procedure =
CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context);
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task],
physical_table_id,
EventContext::default(),
ddl_context,
);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
@@ -216,8 +231,12 @@ async fn test_on_prepare_with_create_if_table_exists() {
let physical_table_id = table_id;
// Sets `create_if_not_exists`
task.create_table.create_if_not_exists = true;
let mut procedure =
CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context);
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task],
physical_table_id,
EventContext::default(),
ddl_context,
);
let status = procedure.on_prepare().await.unwrap();
let output = status.downcast_output_ref::<Vec<u32>>().unwrap();
assert_eq!(*output, vec![8192]);
@@ -264,6 +283,7 @@ async fn test_on_prepare_part_logical_tables_exist() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task, non_exist_task],
physical_table_id,
EventContext::default(),
ddl_context,
);
let status = procedure.on_prepare().await.unwrap();
@@ -311,6 +331,7 @@ async fn test_on_create_metadata() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task, yet_another_task],
physical_table_id,
EventContext::default(),
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
@@ -404,6 +425,7 @@ async fn test_on_create_metadata_part_logical_tables_exist() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task, non_exist_task],
physical_table_id,
EventContext::default(),
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
@@ -476,6 +498,7 @@ async fn test_on_create_metadata_err() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task.clone(), yet_another_task],
physical_table_id,
EventContext::default(),
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
@@ -536,6 +559,7 @@ async fn test_on_submit_create_request() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task, yet_another_task],
physical_table_id,
EventContext::default(),
ddl_context,
);
procedure.on_prepare().await.unwrap();
src/common/meta/src/ddl/tests/create_table.rs
+64 -10
View File
@@ -48,7 +48,7 @@ use crate::ddl::test_util::{assert_column_name, get_raw_table_info, put_datanode
use crate::error::{Error, Result};
use crate::key::table_route::TableRouteValue;
use crate::kv_backend::memory::MemoryKvBackend;
use crate::rpc::ddl::CreateTableTask;
use crate::rpc::ddl::{CreateTableTask, EventContext, QueryContext};
use crate::test_util::{MockDatanodeManager, new_ddl_context, new_ddl_context_with_kv_backend};
fn create_request_handler(_peer: Peer, request: RegionRequest) -> Result<RegionResponse> {
@@ -189,7 +189,13 @@ async fn test_on_prepare_table_exists_err() {
)
.await
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
@@ -212,7 +218,13 @@ async fn test_on_prepare_with_create_if_table_exists() {
)
.await
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
let status = procedure.on_prepare().await.unwrap();
assert_matches!(status, Status::Done { output: Some(..) });
let table_id = *status.downcast_output_ref::<u32>().unwrap();
@@ -225,7 +237,13 @@ async fn test_on_prepare_without_create_if_table_exists() {
let ddl_context = new_ddl_context(node_manager);
let mut task = test_create_table_task("foo");
task.create_table.create_if_not_exists = true;
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -244,7 +262,13 @@ async fn test_on_datanode_create_regions_should_retry() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
assert!(!task.create_table.create_if_not_exists);
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
@@ -261,7 +285,13 @@ async fn test_on_datanode_create_regions_should_not_retry() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
assert!(!task.create_table.create_if_not_exists);
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
@@ -278,7 +308,13 @@ async fn test_on_datanode_create_regions_remaps_addresses_when_retrying() {
let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
procedure.on_prepare().await.unwrap();
let table_route = procedure.data.table_route.as_mut().unwrap();
@@ -299,7 +335,13 @@ async fn test_on_create_metadata_error() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
assert!(!task.create_table.create_if_not_exists);
let mut procedure = CreateTableProcedure::new(task.clone(), ddl_context.clone()).unwrap();
let mut procedure = CreateTableProcedure::new(
task.clone(),
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
@@ -332,7 +374,13 @@ async fn test_on_create_metadata() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
assert!(!task.create_table.create_if_not_exists);
let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
@@ -360,7 +408,13 @@ async fn test_memory_region_keeper_guard_dropped_on_procedure_done() {
let ddl_context = new_ddl_context_with_kv_backend(node_manager, kv_backend);
let task = test_create_table_task("foo");
let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
execute_procedure_until(&mut procedure, |p| {
p.data.state == CreateTableState::CreateMetadata
src/common/meta/src/ddl/tests/create_view.rs
+20 -10
View File
@@ -29,7 +29,7 @@ use crate::ddl::create_view::CreateViewProcedure;
use crate::ddl::test_util::datanode_handler::NaiveDatanodeHandler;
use crate::ddl::tests::create_table::test_create_table_task;
use crate::error::Error;
use crate::rpc::ddl::CreateViewTask;
use crate::rpc::ddl::{CreateViewTask, EventContext, QueryContext};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
pub(crate) fn test_table_names() -> HashSet<table::table_name::TableName> {
@@ -112,7 +112,7 @@ async fn test_on_prepare_view_exists_err() {
)
.await
.unwrap();
let mut procedure = CreateViewProcedure::new(task, ddl_context);
let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::ViewAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
@@ -138,7 +138,7 @@ async fn test_on_prepare_with_create_if_view_exists() {
)
.await
.unwrap();
let mut procedure = CreateViewProcedure::new(task, ddl_context);
let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(status, Status::Done { output: Some(..) });
let table_id = *status.downcast_output_ref::<u32>().unwrap();
@@ -151,7 +151,7 @@ async fn test_on_prepare_without_create_if_table_exists() {
let ddl_context = new_ddl_context(node_manager);
let mut task = test_create_view_task("foo");
task.create_view.create_if_not_exists = true;
let mut procedure = CreateViewProcedure::new(task, ddl_context);
let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -170,7 +170,7 @@ async fn test_on_create_metadata() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_view_task("foo");
assert!(!task.create_view.create_if_not_exists);
let mut procedure = CreateViewProcedure::new(task, ddl_context);
let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context);
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
@@ -189,7 +189,8 @@ async fn test_replace_view_metadata() {
let ddl_context = new_ddl_context(node_manager.clone());
let task = test_create_view_task("foo");
assert!(!task.create_view.create_if_not_exists);
let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone());
let mut procedure =
CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone());
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
@@ -214,7 +215,8 @@ async fn test_replace_view_metadata() {
let mut task = test_create_view_task("foo");
// The view already exists, prepare should fail
{
let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone());
let mut procedure =
CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone());
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::ViewAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
@@ -225,7 +227,8 @@ async fn test_replace_view_metadata() {
task.create_view.logical_plan = vec![4, 5, 6];
task.create_view.definition = "new_definition".to_string();
let mut procedure = CreateViewProcedure::new(task, ddl_context.clone());
let mut procedure =
CreateViewProcedure::new(task, EventContext::default(), ddl_context.clone());
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
@@ -259,7 +262,13 @@ async fn test_replace_table() {
{
// Create a `foo` table.
let task = test_create_table_task("foo");
let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
@@ -272,7 +281,8 @@ async fn test_replace_table() {
// Try to replace a view named `foo` too.
let mut task = test_create_view_task("foo");
task.create_view.or_replace = true;
let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone());
let mut procedure =
CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone());
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
src/common/meta/src/ddl/tests/drop_database.rs
+4
View File
@@ -26,6 +26,7 @@ use crate::ddl::drop_database::executor::DropDatabaseExecutor;
use crate::ddl::test_util::datanode_handler::{NaiveDatanodeHandler, RetryErrorDatanodeHandler};
use crate::ddl::test_util::{create_logical_table, create_physical_table};
use crate::key::schema_name::SchemaNameKey;
use crate::rpc::ddl::EventContext;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
#[tokio::test]
@@ -54,6 +55,7 @@ async fn test_drop_database_with_logical_tables() {
DEFAULT_CATALOG_NAME.to_string(),
DEFAULT_SCHEMA_NAME.to_string(),
false,
EventContext::default(),
ddl_context.clone(),
);
@@ -102,6 +104,7 @@ async fn test_drop_database_retryable_error() {
DEFAULT_CATALOG_NAME.to_string(),
DEFAULT_SCHEMA_NAME.to_string(),
false,
EventContext::default(),
ddl_context.clone(),
);
@@ -146,6 +149,7 @@ async fn test_drop_database_recover() {
DEFAULT_CATALOG_NAME.to_string(),
DEFAULT_SCHEMA_NAME.to_string(),
false,
EventContext::default(),
ddl_context.clone(),
);
let num_operating_regions = 1;
src/common/meta/src/ddl/tests/drop_flow.rs
+7 -7
View File
@@ -26,7 +26,7 @@ use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
use crate::ddl::tests::create_flow::{create_test_flow, create_test_pending_flow};
use crate::error;
use crate::key::table_route::TableRouteValue;
use crate::rpc::ddl::DropFlowTask;
use crate::rpc::ddl::{DropFlowTask, EventContext};
use crate::test_util::{MockFlownodeManager, new_ddl_context};
pub(crate) fn test_drop_flow_task(
@@ -48,7 +48,7 @@ async fn test_drop_flow_not_found() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let task = test_drop_flow_task("my_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, ddl_context);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowNotFound { .. });
}
@@ -81,17 +81,17 @@ async fn test_drop_flow() {
create_test_flow(&ddl_context, "my_flow", source_table_names, sink_table_name).await;
// Drops the flows
let task = test_drop_flow_task("my_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, ddl_context.clone());
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
// Drops if not exists
let task = test_drop_flow_task("my_flow", flow_id, true);
let mut procedure = DropFlowProcedure::new(task, ddl_context.clone());
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
// Drops again
let task = test_drop_flow_task("my_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, ddl_context);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowNotFound { .. });
}
@@ -129,11 +129,11 @@ async fn test_drop_pending_flow_without_routes() {
assert!(flow_info.flownode_ids().is_empty());
let task = test_drop_flow_task("drop_pending_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, ddl_context.clone());
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
let task = test_drop_flow_task("drop_pending_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, ddl_context);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowNotFound { .. });
}
src/common/meta/src/ddl/tests/drop_table.rs
+181 -53
View File
@@ -65,9 +65,9 @@ use crate::key::table_route::TableRouteValue;
use crate::kv_backend::KvBackend;
use crate::kv_backend::memory::MemoryKvBackend;
use crate::peer::Peer;
use crate::rpc::ddl::DropTableTask;
use crate::rpc::ddl::{DropTableTask, EventContext};
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::{PurgeDroppedTableTask, UndropTableTask};
use crate::rpc::ddl::{PurgeDroppedTableTask, QueryContext, UndropTableTask};
use crate::rpc::router::{Region, RegionRoute};
#[cfg(feature = "enterprise")]
use crate::rpc::store::{BatchDeleteRequest, PutRequest};
@@ -128,7 +128,11 @@ async fn undrop_at_restore_metadata(
table_id: TableId,
) -> UndropTableProcedure {
create_dropped_table(context, table_id, Some(1), Some(i64::MAX)).await;
let mut undrop = UndropTableProcedure::new(new_undrop_table_task(table_id), context.clone());
let mut undrop = UndropTableProcedure::new(
new_undrop_table_task(table_id),
context.clone(),
EventContext::default(),
);
let procedure_context = new_test_procedure_context();
undrop.execute(&procedure_context).await.unwrap();
undrop.execute(&procedure_context).await.unwrap();
@@ -157,6 +161,7 @@ fn test_old_drop_table_json_defaults_to_hard_drop() {
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
runtime_context.clone(),
EventContext::default(),
);
let mut old_data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
old_data
@@ -164,6 +169,7 @@ fn test_old_drop_table_json_defaults_to_hard_drop() {
.unwrap()
.remove("soft_drop_enabled");
old_data.as_object_mut().unwrap().remove("dropped_at");
old_data.as_object_mut().unwrap().remove("event_context");
let recovered = DropTableProcedure::from_json(&old_data.to_string(), runtime_context).unwrap();
let recovered_data: serde_json::Value =
@@ -171,12 +177,20 @@ fn test_old_drop_table_json_defaults_to_hard_drop() {
assert_eq!(recovered_data["soft_drop_enabled"], false);
assert_eq!(recovered_data["dropped_at"], serde_json::Value::Null);
assert_eq!(
recovered_data["event_context"],
serde_json::json!({"reason": "unknown"})
);
}
#[test]
fn test_drop_table_lock_key_includes_table_name() {
let context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context);
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
context,
EventContext::default(),
);
let keys = procedure
.lock_key()
@@ -205,7 +219,11 @@ fn test_new_drop_table_procedure_disables_soft_drop_without_enterprise() {
let mut context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
context.soft_drop_enabled = true;
let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context);
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
context,
EventContext::default(),
);
assert!(!procedure.data.soft_drop_enabled);
}
@@ -216,7 +234,11 @@ fn test_new_drop_table_procedure_preserves_soft_drop_with_enterprise() {
let mut context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
context.soft_drop_enabled = true;
let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context);
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
context,
EventContext::default(),
);
assert!(procedure.data.soft_drop_enabled);
}
@@ -251,6 +273,7 @@ async fn test_recovered_soft_drop_preserves_persisted_mode() {
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
soft_context.clone(),
EventContext::default(),
);
let hard_context = new_ddl_context_with_kv_backend(node_manager, kv_backend);
@@ -290,9 +313,13 @@ async fn test_legacy_soft_drop_prepare_preserves_persisted_mode() {
.unwrap();
let mut persisted: serde_json::Value = serde_json::from_str(
&DropTableProcedure::new(new_drop_table_task("foo", table_id, false), context.clone())
.dump()
.unwrap(),
&DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
context.clone(),
EventContext::default(),
)
.dump()
.unwrap(),
)
.unwrap();
persisted["soft_drop_enabled"] = true.into();
@@ -322,7 +349,11 @@ async fn test_disabled_soft_drop_creates_hard_drop_procedure() {
context.soft_drop_enabled = false;
context.soft_drop_retention = Some(Duration::from_millis(100));
let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context);
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
context,
EventContext::default(),
);
assert!(!procedure.data.soft_drop_enabled);
assert_eq!(None, procedure.data.dropped_at);
@@ -356,6 +387,7 @@ async fn test_recovered_hard_drop_ignores_soft_runtime_context() {
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
hard_context.clone(),
EventContext::default(),
);
let mut soft_context = new_ddl_context_with_kv_backend(node_manager, kv_backend);
@@ -397,7 +429,7 @@ async fn test_on_prepare_table_not_exists_err() {
.unwrap();
let task = new_drop_table_task("bar", table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context);
let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default());
assert_eq!(procedure.data.dropped_at, None);
assert_eq!(procedure.data.retention_expires_at, None);
let err = procedure.on_prepare().await.unwrap_err();
@@ -427,6 +459,7 @@ async fn test_soft_drop_prepare_assigns_stable_deadline_and_recovers_it() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
ddl_context.clone(),
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
@@ -463,6 +496,7 @@ async fn test_soft_drop_prepare_rejects_deadline_overflow_before_metadata_delete
let mut procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
ddl_context.clone(),
EventContext::default(),
);
let error = procedure.on_prepare().await.unwrap_err();
@@ -502,13 +536,13 @@ async fn test_on_prepare_table() {
let task = new_drop_table_task("bar", table_id, true);
// Drop if exists
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
procedure.on_prepare().await.unwrap();
assert!(!procedure.rollback_supported());
let task = new_drop_table_task(table_name, table_id, false);
// Drop table
let mut procedure = DropTableProcedure::new(task, ddl_context);
let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default());
procedure.on_prepare().await.unwrap();
}
@@ -559,7 +593,7 @@ async fn test_on_datanode_drop_regions() {
let task = new_drop_table_task(table_name, table_id, false);
// Drop table
let mut procedure = DropTableProcedure::new(task, ddl_context);
let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default());
procedure.on_prepare().await.unwrap();
procedure.on_datanode_drop_regions(false).await.unwrap();
@@ -628,7 +662,7 @@ async fn test_on_datanode_drop_regions_remaps_addresses_when_retrying() {
.unwrap();
let task = new_drop_table_task(table_name, table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
procedure.on_prepare().await.unwrap();
put_datanode_address(&ddl_context, 1, "new-leader").await;
@@ -687,7 +721,7 @@ async fn test_soft_drop_closes_regions_and_keeps_tombstone() {
.unwrap();
let task = new_drop_table_task(table_name, table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
execute_procedure_until_done(&mut procedure).await;
@@ -801,7 +835,7 @@ async fn test_soft_drop_keeps_metadata_live_until_regions_close() {
.unwrap();
let task = new_drop_table_task(table_name, table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
put_datanode_address(&ddl_context, 1, "new-leader").await;
@@ -888,6 +922,7 @@ async fn test_soft_drop_timestamp_is_stable_across_retry_and_recovery() {
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
ddl_context.clone(),
EventContext::default(),
);
assert_eq!(procedure.data.dropped_at, None);
let mut procedure = procedure;
@@ -958,7 +993,7 @@ async fn test_hard_drop_keeps_delete_tombstone_flow() {
.unwrap();
let task = new_drop_table_task(table_name, table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
execute_procedure_until(&mut procedure, |p| {
p.data.state == DropTableState::DeleteTombstone
@@ -1001,13 +1036,20 @@ async fn test_create_table_succeeds_while_tombstone_exists() {
.unwrap();
let drop_task = new_drop_table_task(table_name, dropped_table_id, false);
let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone());
let mut drop_procedure =
DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default());
execute_procedure_until_done(&mut drop_procedure).await;
let mut create_task = test_create_table_task(table_name, 1025);
create_task.create_table.table_id = None;
create_task.table_info.ident.table_id = 0;
let mut create_procedure = CreateTableProcedure::new(create_task, ddl_context.clone()).unwrap();
let mut create_procedure = CreateTableProcedure::new(
create_task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
execute_procedure_until_done(&mut create_procedure).await;
let live_table = ddl_context
@@ -1052,7 +1094,8 @@ async fn test_hard_drop_recreated_table_fails_when_soft_tombstone_exists() {
.unwrap();
let drop_task = new_drop_table_task(table_name, original_table_id, false);
let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone());
let mut drop_procedure =
DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default());
execute_procedure_until_done(&mut drop_procedure).await;
ddl_context.soft_drop_enabled = false;
@@ -1069,6 +1112,7 @@ async fn test_hard_drop_recreated_table_fails_when_soft_tombstone_exists() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, recreated_table_id, false),
ddl_context,
EventContext::default(),
);
let err = procedure.on_prepare().await.unwrap_err();
@@ -1098,7 +1142,8 @@ async fn test_hard_drop_recreated_table_ignores_previous_orphan_tombstone() {
.unwrap();
let drop_task = new_drop_table_task(table_name, original_table_id, false);
let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone());
let mut drop_procedure =
DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default());
execute_procedure_until(&mut drop_procedure, |p| {
p.data.state == DropTableState::DeleteTombstone
})
@@ -1117,6 +1162,7 @@ async fn test_hard_drop_recreated_table_ignores_previous_orphan_tombstone() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, recreated_table_id, false),
ddl_context,
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
@@ -1156,6 +1202,7 @@ async fn test_undrop_table_restores_metadata_and_reopens_regions() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
assert!(
@@ -1172,8 +1219,11 @@ async fn test_undrop_table_restores_metadata_and_reopens_regions() {
while rx.try_recv().is_ok() {}
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut procedure).await;
let live_table = ddl_context
@@ -1264,12 +1314,16 @@ async fn test_undrop_table_opens_regions_before_restoring_live_metadata() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
while rx.try_recv().is_ok() {}
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
procedure.execute(&ctx).await.unwrap();
@@ -1333,13 +1387,17 @@ async fn test_undrop_logical_table_skips_datanode_open() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, logical_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
while rx.try_recv().is_ok() {}
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(logical_table_id), ddl_context.clone());
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(logical_table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut procedure).await;
let live_table = ddl_context
@@ -1370,6 +1428,7 @@ async fn test_soft_drop_metric_logical_table_falls_back_to_hard_drop() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task("foo", logical_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
assert!(!procedure.data.soft_drop_enabled);
@@ -1413,6 +1472,7 @@ async fn test_soft_drop_metric_physical_table_remains_enabled() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut procedure).await;
@@ -1457,6 +1517,7 @@ async fn test_soft_drop_file_engine_table_falls_back_to_hard_drop() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
assert!(!procedure.data.soft_drop_enabled);
@@ -1493,6 +1554,7 @@ async fn test_file_engine_fallback_is_resolved_before_tombstone_conflict() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, recreated_table_id, false),
ddl_context,
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
@@ -1508,8 +1570,11 @@ async fn test_undrop_metric_logical_table_fails() {
let logical_table_id =
create_metric_logical_table_tombstone(&ddl_context, physical_table_id, "foo").await;
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(logical_table_id), ddl_context);
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(logical_table_id),
ddl_context,
EventContext::default(),
);
let err = procedure.on_prepare().await.unwrap_err();
assert_eq!(err.status_code(), StatusCode::Unsupported);
@@ -1527,6 +1592,7 @@ async fn test_purge_metric_logical_table_fails() {
let mut procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(logical_table_id),
ddl_context,
EventContext::default(),
);
let err = procedure
.execute(&new_test_procedure_context())
@@ -1558,6 +1624,7 @@ async fn test_undrop_table_fails_when_live_name_exists() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, dropped_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
ddl_context
@@ -1570,8 +1637,11 @@ async fn test_undrop_table_fails_when_live_name_exists() {
.await
.unwrap();
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context);
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(dropped_table_id),
ddl_context,
EventContext::default(),
);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableAlreadyExists { .. });
@@ -1599,11 +1669,15 @@ async fn test_undrop_table_fails_when_live_name_is_created_after_prepare() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, dropped_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context.clone());
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(dropped_table_id),
ddl_context.clone(),
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
ddl_context
.table_metadata_manager
@@ -1667,13 +1741,17 @@ async fn test_undrop_table_closes_opened_regions_when_restore_metadata_races_wit
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, dropped_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
while rx.try_recv().is_ok() {}
detector_controller.clear().await;
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context.clone());
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(dropped_table_id),
ddl_context.clone(),
EventContext::default(),
);
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
procedure.execute(&ctx).await.unwrap();
@@ -1767,6 +1845,7 @@ async fn test_undrop_table_lock_key_includes_original_table_name_before_prepare(
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
@@ -1781,6 +1860,7 @@ async fn test_undrop_table_lock_key_includes_original_table_name_before_prepare(
new_undrop_table_task(table_id),
ddl_context,
Some(original_table_name),
EventContext::default(),
);
let keys = procedure
@@ -1848,10 +1928,12 @@ async fn test_undrop_rejects_stale_id_after_name_tombstone_is_consumed() {
new_undrop_table_task(original_table_id),
ddl_context.clone(),
Some(original_table_name.clone()),
EventContext::default(),
);
let mut hard_drop = DropTableProcedure::new(
new_drop_table_task(table_name, recreated_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut hard_drop).await;
assert!(
@@ -1906,11 +1988,15 @@ async fn test_undrop_table_replayed_restore_metadata_is_idempotent() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
procedure.execute(&ctx).await.unwrap();
@@ -2014,6 +2100,7 @@ async fn test_purge_dropped_table_cleans_regions_offline_and_deletes_tombstone()
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
while rx.try_recv().is_ok() {}
@@ -2022,6 +2109,7 @@ async fn test_purge_dropped_table_cleans_regions_offline_and_deletes_tombstone()
let mut procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut procedure).await;
@@ -2100,6 +2188,7 @@ async fn test_automatic_purge_rechecks_unexpired_tombstone() {
let procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
data["check_expired"] = true.into();
@@ -2156,6 +2245,7 @@ async fn test_recovered_automatic_purge_rechecks_unexpired_tombstone() {
let procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
data["check_expired"] = true.into();
@@ -2233,6 +2323,7 @@ async fn test_recovered_automatic_purge_obeys_runtime_disable() {
let procedure = PurgeDroppedTableProcedure::new_if_expired(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut recovered =
PurgeDroppedTableProcedure::from_json(&procedure.dump().unwrap(), ddl_context.clone())
@@ -2264,6 +2355,7 @@ async fn test_recovered_automatic_purge_finishes_after_cleanup_is_claimed() {
let mut procedure = PurgeDroppedTableProcedure::new_if_expired(
new_purge_dropped_table_task(table_id),
enabled_context.clone(),
EventContext::default(),
);
procedure
.execute(&new_test_procedure_context())
@@ -2327,6 +2419,7 @@ async fn test_recovered_automatic_purge_ignores_foreign_generation_claim() {
let mut procedure = PurgeDroppedTableProcedure::new_if_expired(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
procedure
.execute(&new_test_procedure_context())
@@ -2369,8 +2462,11 @@ async fn test_undrop_rejects_table_claimed_for_purge() {
.await
.unwrap();
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let error = procedure.on_prepare().await.unwrap_err();
assert_eq!(StatusCode::TableNotFound, error.status_code());
@@ -2392,8 +2488,11 @@ async fn test_recovered_undrop_rejects_tombstone_deleted_by_purge() {
let table_id = 1024;
create_dropped_table(&ddl_context, table_id, Some(1), Some(i64::MAX)).await;
let mut undrop =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let mut undrop = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
undrop.on_prepare().await.unwrap();
let open_regions_state = undrop.dump().unwrap();
undrop.execute(&new_test_procedure_context()).await.unwrap();
@@ -2402,6 +2501,7 @@ async fn test_recovered_undrop_rejects_tombstone_deleted_by_purge() {
let mut purge = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut purge).await;
@@ -2433,8 +2533,11 @@ async fn test_recovered_undrop_rejects_replacement_generation() {
.await
.unwrap();
let mut undrop =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let mut undrop = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
undrop.on_prepare().await.unwrap();
let persisted = undrop.dump().unwrap();
kv_backend
@@ -2500,8 +2603,11 @@ async fn test_legacy_unmarked_tombstone_can_be_undropped_and_purged() {
assert_eq!(dropped_tables.len(), 1);
assert_eq!(dropped_tables[0].dropped_at, None);
let mut undrop =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let mut undrop = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut undrop).await;
ddl_context
@@ -2518,6 +2624,7 @@ async fn test_legacy_unmarked_tombstone_can_be_undropped_and_purged() {
let mut purge = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut purge).await;
assert!(
@@ -2560,6 +2667,7 @@ async fn test_purge_dropped_table_by_id_selects_tombstone_when_live_table_exists
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, dropped_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
ddl_context
@@ -2576,6 +2684,7 @@ async fn test_purge_dropped_table_by_id_selects_tombstone_when_live_table_exists
let mut procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(dropped_table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut procedure).await;
@@ -2626,8 +2735,12 @@ async fn test_on_rollback() {
let physical_table_id = table_id;
// Creates the logical table metadata.
let task = test_create_logical_table_task("foo");
let mut procedure =
CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context.clone());
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task],
physical_table_id,
EventContext::default(),
ddl_context.clone(),
);
procedure.on_prepare().await.unwrap();
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
@@ -2640,12 +2753,25 @@ async fn test_on_rollback() {
// Drops the physical table
{
let task = new_drop_table_task("phy_table", physical_table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure =
DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
procedure.on_prepare().await.unwrap();
assert!(procedure.rollback_supported());
procedure.on_delete_metadata().await.unwrap();
assert!(procedure.rollback_supported());
procedure.rollback(&ctx).await.unwrap();
assert!(procedure.dropping_regions.is_empty());
assert_eq!(ddl_context.memory_region_keeper.len(), 0);
// The physical table can be dropped again after the metadata rollback.
let retry_task = new_drop_table_task("phy_table", physical_table_id, false);
let mut retry =
DropTableProcedure::new(retry_task, ddl_context.clone(), EventContext::default());
retry.on_prepare().await.unwrap();
retry.on_delete_metadata().await.unwrap();
retry.rollback(&ctx).await.unwrap();
assert!(retry.dropping_regions.is_empty());
assert_eq!(ddl_context.memory_region_keeper.len(), 0);
// Rollback again
assert!(procedure.rollback_supported());
procedure.rollback(&ctx).await.unwrap();
@@ -2656,7 +2782,7 @@ async fn test_on_rollback() {
// Drops the logical table
ddl_context.soft_drop_enabled = false;
let task = new_drop_table_task("foo", table_ids[0], false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
procedure.on_prepare().await.unwrap();
assert!(!procedure.rollback_supported());
}
@@ -2753,7 +2879,8 @@ async fn test_memory_region_keeper_guard_dropped_on_procedure_done() {
let logical_table_id = create_logical_table(ddl_context.clone(), physical_table_id, "s").await;
let inner_test = |task: DropTableTask| async {
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure =
DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
execute_procedure_until(&mut procedure, |p| {
p.data.state == DropTableState::InvalidateTableCache
})
@@ -2805,7 +2932,8 @@ async fn test_from_json() {
let physical_table_id = create_physical_table(&ddl_context, "t").await;
let task = new_drop_table_task("t", physical_table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure =
DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
execute_procedure_until(&mut procedure, |p| p.data.state == state).await;
let data = procedure.dump().unwrap();
assert_eq!(
@@ -2834,7 +2962,7 @@ async fn test_from_json() {
let physical_table_id = create_physical_table(&ddl_context, "t").await;
let task = new_drop_table_task("t", physical_table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
execute_procedure_until_done(&mut procedure).await;
let data = procedure.dump().unwrap();
assert_eq!(
src/common/meta/src/ddl/tests/drop_view.rs
+7 -7
View File
@@ -24,7 +24,7 @@ use crate::ddl::drop_view::{DropViewProcedure, DropViewState};
use crate::ddl::test_util::create_table::test_create_table_task;
use crate::ddl::tests::create_view::{test_create_view_task, test_table_names};
use crate::key::table_route::TableRouteValue;
use crate::rpc::ddl::DropViewTask;
use crate::rpc::ddl::{DropViewTask, EventContext};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
pub(crate) fn new_drop_view_task(
@@ -63,7 +63,7 @@ async fn test_on_prepare_view_not_exists_err() {
.unwrap();
let task = new_drop_view_task("bar", view_id, false);
let mut procedure = DropViewProcedure::new(task, ddl_context);
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_eq!(err.status_code(), StatusCode::TableNotFound);
}
@@ -87,7 +87,7 @@ async fn test_on_prepare_not_view_err() {
.unwrap();
let task = new_drop_view_task(view_name, view_id, false);
let mut procedure = DropViewProcedure::new(task, ddl_context);
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context);
// It's not a view, expect error
let err = procedure.on_prepare().await.unwrap_err();
assert_eq!(err.status_code(), StatusCode::InvalidArguments);
@@ -117,12 +117,12 @@ async fn test_on_prepare_success() {
let task = new_drop_view_task("bar", view_id, true);
// Drop if exists
let mut procedure = DropViewProcedure::new(task, ddl_context.clone());
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context.clone());
procedure.on_prepare().await.unwrap();
let task = new_drop_view_task(view_name, view_id, false);
// Prepare success
let mut procedure = DropViewProcedure::new(task, ddl_context);
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context);
procedure.on_prepare().await.unwrap();
assert_eq!(DropViewState::DeleteMetadata, procedure.state());
}
@@ -161,7 +161,7 @@ async fn test_drop_view_success() {
let task = new_drop_view_task(view_name, view_id, false);
// Prepare success
let mut procedure = DropViewProcedure::new(task, ddl_context.clone());
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
assert_eq!(DropViewState::InvalidateViewCache, procedure.state());
@@ -178,7 +178,7 @@ async fn test_drop_view_success() {
// Drop again
let task = new_drop_view_task(view_name, view_id, false);
let mut procedure = DropViewProcedure::new(task, ddl_context);
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_eq!(err.status_code(), StatusCode::TableNotFound);
}
src/common/meta/src/ddl/tests/event/database.rs
+40 -8
View File
@@ -20,17 +20,17 @@ use api::v1::value::ValueData;
use api::v1::{AlterDatabaseExpr, Row, SetDatabaseOptions as PbSetDatabaseOptions, Value};
use common_event_recorder::Event;
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN,
CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN,
PROCEDURE_ERROR_COLUMN as EVENT_TABLE_PROCEDURE_ERROR_COLUMN,
PROCEDURE_ID_COLUMN as EVENT_TABLE_PROCEDURE_ID_COLUMN,
PROCEDURE_STATE_COLUMN as EVENT_TABLE_PROCEDURE_STATE_COLUMN,
PROCEDURE_TRIGGER_COLUMN as EVENT_TABLE_PROCEDURE_TRIGGER_COLUMN,
SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, jsonb_value,
SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN,
};
use common_event_recorder::testing::assert_event_contract;
use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState};
use super::test_util::assert_event_filter;
use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value};
use crate::ddl::alter_database::AlterDatabaseProcedure;
use crate::ddl::create_database::CreateDatabaseProcedure;
use crate::ddl::drop_database::DropDatabaseProcedure;
@@ -39,7 +39,7 @@ use crate::ddl::event::database::{
DatabaseDdlEvent,
};
use crate::rpc::ddl::{
AlterDatabaseKind, SetDatabaseOption, SetDatabaseOptions, UnsetDatabaseOption,
AlterDatabaseKind, EventContext, SetDatabaseOption, SetDatabaseOptions, UnsetDatabaseOption,
UnsetDatabaseOptions,
};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -50,7 +50,13 @@ fn test_create_database_submitted_event_contract() {
("password".to_string(), "do-not-record".to_string()),
("compaction.type".to_string(), "twcs".to_string()),
]);
let event = DatabaseDdlEvent::create_submitted("greptime", "metrics", true, &options);
let event = DatabaseDdlEvent::create_submitted(
"greptime",
"metrics",
true,
&options,
EventContext::default(),
);
assert_event_locator(
&event,
@@ -87,6 +93,7 @@ fn test_alter_database_set_and_unset_event_contracts() {
SetDatabaseOption::Other("secret_token".to_string(), "hidden".to_string()),
SetDatabaseOption::Ttl(std::time::Duration::from_secs(3600).into()),
])),
EventContext::default(),
);
assert_event_locator(
&set,
@@ -113,6 +120,7 @@ fn test_alter_database_set_and_unset_event_contracts() {
UnsetDatabaseOption::Ttl,
UnsetDatabaseOption::Other("compaction.type".to_string()),
])),
EventContext::default(),
);
assert_event_locator(
&unset,
@@ -132,7 +140,8 @@ fn test_alter_database_set_and_unset_event_contracts() {
#[test]
fn test_drop_database_submitted_event_contract() {
let event = DatabaseDdlEvent::drop_submitted("greptime", "metrics", true);
let event =
DatabaseDdlEvent::drop_submitted("greptime", "metrics", true, EventContext::default());
assert_event_locator(
&event,
@@ -165,7 +174,14 @@ fn test_database_lifecycle_events_have_fixed_schema_and_null_intent() {
assert_eq!(
DatabaseDdlEvent::create_lifecycle().extra_schema(),
DatabaseDdlEvent::create_submitted("c", "s", false, &HashMap::new()).extra_schema()
DatabaseDdlEvent::create_submitted(
"c",
"s",
false,
&HashMap::new(),
EventContext::default()
)
.extra_schema()
);
}
@@ -179,6 +195,7 @@ fn test_database_events_preserve_procedure_envelope_contract() {
"metrics",
false,
&HashMap::new(),
EventContext::default(),
)),
ProcedureState::Running,
EventTrigger::Submitted,
@@ -217,6 +234,7 @@ fn test_create_database_event_filter() {
false,
HashMap::new(),
None,
EventContext::default(),
new_ddl_context(Arc::new(MockDatanodeManager::new(()))),
);
@@ -237,6 +255,7 @@ fn test_alter_database_event_filter() {
)),
},
},
EventContext::default(),
new_ddl_context(Arc::new(MockDatanodeManager::new(()))),
)
.unwrap();
@@ -250,6 +269,7 @@ fn test_drop_database_event_filter() {
"greptime".to_string(),
"metrics".to_string(),
false,
EventContext::default(),
new_ddl_context(Arc::new(MockDatanodeManager::new(()))),
);
@@ -268,6 +288,7 @@ fn assert_event_locator(
&[
EVENT_TABLE_CATALOG_NAME_COLUMN.column_schema(),
EVENT_TABLE_SCHEMA_NAME_COLUMN.column_schema(),
EVENT_CONTEXT_COLUMN.column_schema(),
],
&[Row {
values: vec![
@@ -277,6 +298,11 @@ fn assert_event_locator(
Value {
value_data: schema_name.map(|value| ValueData::StringValue(value.to_string())),
},
if catalog_name.is_some() {
default_event_context_value()
} else {
Value { value_data: None }
},
],
}],
);
@@ -300,6 +326,7 @@ fn assert_procedure_event_contract(
EVENT_TABLE_PROCEDURE_TRIGGER_COLUMN.column_schema(),
EVENT_TABLE_CATALOG_NAME_COLUMN.column_schema(),
EVENT_TABLE_SCHEMA_NAME_COLUMN.column_schema(),
EVENT_CONTEXT_COLUMN.column_schema(),
],
&[Row {
values: vec![
@@ -314,13 +341,18 @@ fn assert_procedure_event_contract(
Value {
value_data: Some(ValueData::StringValue(String::new())),
},
jsonb_value(&serde_json::json!({"type": procedure_trigger})),
procedure_trigger_value(procedure_trigger),
Value {
value_data: catalog_name.map(|value| ValueData::StringValue(value.to_string())),
},
Value {
value_data: schema_name.map(|value| ValueData::StringValue(value.to_string())),
},
if catalog_name.is_some() {
default_event_context_value()
} else {
Value { value_data: None }
},
],
}],
);
src/common/meta/src/ddl/tests/event/flow.rs
+21 -5
View File
@@ -19,14 +19,14 @@ use api::v1::{ColumnSchema, Row, Value};
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_event_recorder::Event;
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, PROCEDURE_ERROR_COLUMN,
PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, jsonb_value,
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN,
PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN,
};
use common_event_recorder::testing::assert_event_contract;
use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState};
use table::table_name::TableName;
use super::test_util::assert_event_filter;
use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value};
use crate::ddl::create_flow::CreateFlowProcedure;
use crate::ddl::drop_flow::DropFlowProcedure;
use crate::ddl::event::flow::{
@@ -35,6 +35,7 @@ use crate::ddl::event::flow::{
use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
use crate::ddl::tests::create_flow::{test_create_flow_task, test_query_context};
use crate::ddl::tests::drop_flow::test_drop_flow_task;
use crate::rpc::ddl::EventContext;
use crate::test_util::{MockFlownodeManager, new_ddl_context};
#[test]
@@ -48,6 +49,7 @@ fn test_flow_submitted_event_contracts() {
expire_after: Some(300),
eval_interval_secs: Some(60),
},
EventContext::default(),
);
assert_event_contract(
&create,
@@ -58,6 +60,7 @@ fn test_flow_submitted_event_contracts() {
ValueData::StringValue("greptime".to_string()).into(),
ValueData::StringValue("metrics".to_string()).into(),
Value { value_data: None },
default_event_context_value(),
],
}],
);
@@ -72,7 +75,8 @@ fn test_flow_submitted_event_contracts() {
})
);
let drop = FlowDdlEvent::drop_submitted("greptime", "metrics", 42, true);
let drop =
FlowDdlEvent::drop_submitted("greptime", "metrics", 42, true, EventContext::default());
assert_event_contract(
&drop,
DROP_FLOW_EVENT_TYPE,
@@ -82,6 +86,7 @@ fn test_flow_submitted_event_contracts() {
ValueData::StringValue("greptime".to_string()).into(),
ValueData::StringValue("metrics".to_string()).into(),
ValueData::U32Value(42).into(),
default_event_context_value(),
],
}],
);
@@ -107,6 +112,7 @@ fn test_flow_lifecycle_events_have_fixed_schema_and_null_intent() {
Value { value_data: None },
Value { value_data: None },
Value { value_data: None },
Value { value_data: None },
],
}],
);
@@ -123,6 +129,7 @@ fn test_flow_lifecycle_events_have_fixed_schema_and_null_intent() {
Value { value_data: None },
Value { value_data: None },
ValueData::U32Value(42).into(),
Value { value_data: None },
],
}],
);
@@ -142,6 +149,7 @@ fn test_flow_events_preserve_procedure_envelope_contract() {
expire_after: None,
eval_interval_secs: None,
},
EventContext::default(),
)),
ProcedureState::Running,
EventTrigger::Submitted,
@@ -187,6 +195,7 @@ fn test_create_flow_event_filter() {
false,
),
test_query_context(),
EventContext::default(),
new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))),
);
assert_event_filter(&procedure, CREATE_FLOW_EVENT_TYPE);
@@ -196,6 +205,7 @@ fn test_create_flow_event_filter() {
fn test_drop_flow_event_filter() {
let procedure = DropFlowProcedure::new(
test_drop_flow_task("flow", 42, false),
EventContext::default(),
new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))),
);
assert_event_filter(&procedure, DROP_FLOW_EVENT_TYPE);
@@ -206,6 +216,7 @@ fn flow_schema() -> Vec<ColumnSchema> {
CATALOG_NAME_COLUMN.column_schema(),
FLOW_NAME_COLUMN.column_schema(),
FLOW_ID_COLUMN.column_schema(),
EVENT_CONTEXT_COLUMN.column_schema(),
]
}
@@ -238,7 +249,7 @@ fn assert_procedure_event_contract(
ValueData::StringValue(event.procedure_id.to_string()).into(),
ValueData::StringValue(state.to_string()).into(),
ValueData::StringValue(String::new()).into(),
jsonb_value(&serde_json::json!({"type": trigger})),
procedure_trigger_value(trigger),
optional_string(locator.catalog_name),
optional_string(locator.flow_name),
locator
@@ -246,6 +257,11 @@ fn assert_procedure_event_contract(
.map(ValueData::U32Value)
.map(Into::into)
.unwrap_or(Value { value_data: None }),
if locator.catalog_name.is_some() {
default_event_context_value()
} else {
Value { value_data: None }
},
],
}],
);
src/common/meta/src/ddl/tests/event/table.rs
+120 -39
View File
@@ -19,13 +19,13 @@ use api::v1::value::ValueData;
use api::v1::{ColumnDataType, Repartition, SemanticType, Value};
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN,
TABLE_NAME_COLUMN,
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN,
TABLE_ID_COLUMN, TABLE_NAME_COLUMN, jsonb_value,
};
use common_event_recorder::{Event, EventTypeFilter};
use common_procedure::{
ChildSubmissionOutcome, EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState,
RetryPhase,
ChildSubmissionOutcome, EventRuntimeContext, EventTrigger, Procedure, ProcedureId,
ProcedureState, RetryPhase,
};
use common_time::Timestamp;
use serde_json::{Value as JsonValue, json};
@@ -52,9 +52,9 @@ use crate::ddl::truncate_table::TruncateTableProcedure;
use crate::ddl::undrop_table::UndropTableProcedure;
use crate::key::DeserializedValueWithBytes;
use crate::key::table_info::TableInfoValue;
use crate::rpc::ddl::{DropTableTask, TruncateTableTask};
use crate::rpc::ddl::{DropTableTask, EventContext, QueryContext, TruncateTableTask};
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::{PurgeDroppedTableTask, UndropTableTask};
use crate::rpc::ddl::{PurgeDroppedTableTask, TriggerReason, UndropTableTask};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
struct EventCase {
@@ -177,7 +177,13 @@ fn later_lifecycle_events_are_uniform() {
fn create_success_events_keep_allocated_ids() {
let mut task = test_create_table_task("create_success");
task.table_info.ident.table_id = 7;
let create_table = CreateTableProcedure::new(task, test_context()).unwrap();
let create_table = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
test_context(),
)
.unwrap();
let state = ProcedureState::Done {
output: Some(Arc::new(42_u32)),
};
@@ -196,6 +202,7 @@ fn create_success_events_keep_allocated_ids() {
test_create_logical_table_task("bar"),
],
1024,
EventContext::default(),
test_context(),
);
let state = ProcedureState::Done {
@@ -234,13 +241,14 @@ fn event_cases() -> Vec<EventCase> {
TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "create"),
true,
"mito2",
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"create_if_not_exists": true,
"engine": "mito2",
}),
rows: vec![table_locator_values(Some("create"), None)],
rows: vec![submitted_table_locator_values(Some("create"), None)],
},
EventCase {
event_type: TableDdlEventType::CreateLogicalTables,
@@ -252,14 +260,15 @@ fn event_cases() -> Vec<EventCase> {
.with_physical_table_id(10),
],
2,
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"table_count": 2,
}),
rows: vec![
logical_locator_values("logical1", None, 10),
logical_locator_values("logical2", None, 10),
submitted_logical_locator_values("logical1", None, 10),
submitted_logical_locator_values("logical2", None, 10),
],
},
EventCase {
@@ -268,12 +277,13 @@ fn event_cases() -> Vec<EventCase> {
TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "alter")
.with_table_id(11),
Some("drop_columns"),
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"kind": "drop_columns",
}),
rows: vec![table_locator_values(Some("alter"), Some(11))],
rows: vec![submitted_table_locator_values(Some("alter"), Some(11))],
},
EventCase {
event_type: TableDdlEventType::AlterLogicalTables,
@@ -286,6 +296,7 @@ fn event_cases() -> Vec<EventCase> {
],
2,
["rename_table", "add_columns", "add_columns"],
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
@@ -293,8 +304,8 @@ fn event_cases() -> Vec<EventCase> {
"kinds": ["add_columns", "rename_table"],
}),
rows: vec![
logical_locator_values("logical1", None, 10),
logical_locator_values("logical2", None, 10),
submitted_logical_locator_values("logical1", None, 10),
submitted_logical_locator_values("logical2", None, 10),
],
},
EventCase {
@@ -303,26 +314,33 @@ fn event_cases() -> Vec<EventCase> {
TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "drop")
.with_table_id(12),
true,
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"drop_if_exists": true,
}),
rows: vec![table_locator_values(Some("drop"), Some(12))],
rows: vec![submitted_table_locator_values(Some("drop"), Some(12))],
},
#[cfg(feature = "enterprise")]
EventCase {
event_type: TableDdlEventType::UndropTable,
event: TableDdlEvent::undrop_table_submitted(TableDdlLocator::from_table_id(13)),
event: TableDdlEvent::undrop_table_submitted(
TableDdlLocator::from_table_id(13),
EventContext::default(),
),
payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}),
rows: vec![table_locator_values(None, Some(13))],
rows: vec![submitted_table_locator_values(None, Some(13))],
},
#[cfg(feature = "enterprise")]
EventCase {
event_type: TableDdlEventType::PurgeDroppedTable,
event: TableDdlEvent::purge_dropped_table_submitted(TableDdlLocator::from_table_id(14)),
event: TableDdlEvent::purge_dropped_table_submitted(
TableDdlLocator::from_table_id(14),
EventContext::default(),
),
payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}),
rows: vec![table_locator_values(None, Some(14))],
rows: vec![submitted_table_locator_values(None, Some(14))],
},
EventCase {
event_type: TableDdlEventType::TruncateTable,
@@ -330,29 +348,41 @@ fn event_cases() -> Vec<EventCase> {
TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "truncate")
.with_table_id(15),
4,
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"time_range_count": 4,
}),
rows: vec![table_locator_values(Some("truncate"), Some(15))],
rows: vec![submitted_table_locator_values(Some("truncate"), Some(15))],
},
]
}
fn procedure_cases() -> Vec<ProcedureCase> {
let create_table =
CreateTableProcedure::new(test_create_table_task("create"), test_context()).unwrap();
let create_table = CreateTableProcedure::new(
test_create_table_task("create"),
QueryContext::default(),
EventContext::default(),
test_context(),
)
.unwrap();
let create_logical_tables = CreateLogicalTablesProcedure::new(
vec![
test_create_logical_table_task("logical1"),
test_create_logical_table_task("logical2"),
],
41,
EventContext::default(),
test_context(),
);
let alter_table =
AlterTableProcedure::new(42, test_alter_table_task("alter"), test_context()).unwrap();
let alter_table = AlterTableProcedure::new(
42,
test_alter_table_task("alter"),
EventContext::default(),
test_context(),
)
.unwrap();
let alter_logical_tables = AlterLogicalTablesProcedure::new(
vec![
make_alter_logical_table_add_column_task(
@@ -367,6 +397,7 @@ fn procedure_cases() -> Vec<ProcedureCase> {
),
],
43,
EventContext::default(),
test_context(),
);
let drop_table = DropTableProcedure::new(
@@ -378,12 +409,20 @@ fn procedure_cases() -> Vec<ProcedureCase> {
drop_if_exists: true,
},
test_context(),
EventContext::default(),
);
#[cfg(feature = "enterprise")]
let undrop_table = UndropTableProcedure::new(UndropTableTask { table_id: 45 }, test_context());
let undrop_table = UndropTableProcedure::new(
UndropTableTask { table_id: 45 },
test_context(),
EventContext::default(),
);
#[cfg(feature = "enterprise")]
let purge_dropped_table =
PurgeDroppedTableProcedure::new(PurgeDroppedTableTask { table_id: 46 }, test_context());
let purge_dropped_table = PurgeDroppedTableProcedure::new_if_expired(
PurgeDroppedTableTask { table_id: 46 },
test_context(),
EventContext::new(TriggerReason::ScheduledGc),
);
let truncate_table = truncate_procedure(TruncateTableTask {
catalog: DEFAULT_CATALOG_NAME.to_string(),
schema: DEFAULT_SCHEMA_NAME.to_string(),
@@ -404,7 +443,7 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"create_if_not_exists": false,
"engine": "mito2",
}),
rows: vec![table_locator_values(Some("create"), None)],
rows: vec![submitted_table_locator_values(Some("create"), None)],
},
ProcedureCase {
procedure: Box::new(create_logical_tables),
@@ -414,8 +453,8 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"table_count": 2,
}),
rows: vec![
logical_locator_values("logical1", None, 41),
logical_locator_values("logical2", None, 41),
submitted_logical_locator_values("logical1", None, 41),
submitted_logical_locator_values("logical2", None, 41),
],
},
ProcedureCase {
@@ -425,7 +464,7 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"version": TABLE_DDL_PAYLOAD_VERSION,
"kind": "drop_columns",
}),
rows: vec![table_locator_values(Some("alter"), Some(42))],
rows: vec![submitted_table_locator_values(Some("alter"), Some(42))],
},
ProcedureCase {
procedure: Box::new(alter_logical_tables),
@@ -436,8 +475,8 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"kinds": ["add_columns"],
}),
rows: vec![
logical_locator_values("logical1", None, 43),
logical_locator_values("logical2", None, 43),
submitted_logical_locator_values("logical1", None, 43),
submitted_logical_locator_values("logical2", None, 43),
],
},
ProcedureCase {
@@ -447,21 +486,25 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"version": TABLE_DDL_PAYLOAD_VERSION,
"drop_if_exists": true,
}),
rows: vec![table_locator_values(Some("drop"), Some(44))],
rows: vec![submitted_table_locator_values(Some("drop"), Some(44))],
},
#[cfg(feature = "enterprise")]
ProcedureCase {
procedure: Box::new(undrop_table),
event_type: "undrop_table",
payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}),
rows: vec![table_locator_values(None, Some(45))],
rows: vec![submitted_table_locator_values(None, Some(45))],
},
#[cfg(feature = "enterprise")]
ProcedureCase {
procedure: Box::new(purge_dropped_table),
event_type: "purge_dropped_table",
payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}),
rows: vec![table_locator_values(None, Some(46))],
rows: vec![submitted_table_locator_values_with_event_context(
None,
Some(46),
EventContext::new(TriggerReason::ScheduledGc),
)],
},
ProcedureCase {
procedure: Box::new(truncate_table),
@@ -470,7 +513,7 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"version": TABLE_DDL_PAYLOAD_VERSION,
"time_range_count": 1,
}),
rows: vec![table_locator_values(Some("truncate"), Some(47))],
rows: vec![submitted_table_locator_values(Some("truncate"), Some(47))],
},
]
}
@@ -491,6 +534,7 @@ fn expected_schema(event_type: TableDdlEventType) -> Vec<(&'static str, i32)> {
ColumnDataType::Uint32 as i32,
));
}
schema.push((EVENT_CONTEXT_COLUMN.name(), ColumnDataType::Binary as i32));
schema
}
@@ -503,12 +547,14 @@ fn table_locator_values(table_name: Option<&str>, table_id: Option<u32>) -> Vec<
} else {
(Value::default(), Value::default())
};
vec![
let mut values = vec![
catalog_name,
schema_name,
table_name.map(string_value).unwrap_or_default(),
table_id.map(table_id_value).unwrap_or_default(),
]
];
values.push(Value::default());
values
}
fn logical_locator_values(
@@ -517,10 +563,44 @@ fn logical_locator_values(
physical_table_id: u32,
) -> Vec<Value> {
let mut values = table_locator_values(Some(table_name), table_id);
let event_context = values.pop().unwrap();
values.push(table_id_value(physical_table_id));
values.push(event_context);
values
}
fn submitted_table_locator_values(table_name: Option<&str>, table_id: Option<u32>) -> Vec<Value> {
submitted_table_locator_values_with_event_context(table_name, table_id, EventContext::default())
}
fn submitted_table_locator_values_with_event_context(
table_name: Option<&str>,
table_id: Option<u32>,
event_context: EventContext,
) -> Vec<Value> {
let mut values = table_locator_values(table_name, table_id);
*values.last_mut().unwrap() = event_context_value_for(event_context);
values
}
fn submitted_logical_locator_values(
table_name: &str,
table_id: Option<u32>,
physical_table_id: u32,
) -> Vec<Value> {
let mut values = logical_locator_values(table_name, table_id, physical_table_id);
*values.last_mut().unwrap() = event_context_value();
values
}
fn event_context_value() -> Value {
event_context_value_for(EventContext::default())
}
fn event_context_value_for(event_context: EventContext) -> Value {
jsonb_value(&serde_json::to_value(event_context).unwrap())
}
fn string_value(value: &str) -> Value {
ValueData::StringValue(value.to_string()).into()
}
@@ -535,6 +615,7 @@ fn truncate_procedure(task: TruncateTableTask) -> TruncateTableProcedure {
task,
DeserializedValueWithBytes::from_inner(TableInfoValue::new(table_info)),
test_context(),
EventContext::default(),
)
}
@@ -552,7 +633,7 @@ fn event_for_state(
lifecycle_state: &ProcedureState,
) -> Box<dyn Event> {
procedure
.event(&EventContext {
.event(&EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state,
trigger,
src/common/meta/src/ddl/tests/event/test_util.rs
+16 -4
View File
@@ -15,12 +15,24 @@
use std::collections::HashSet;
use std::sync::Arc;
use api::v1::Value;
use common_event_recorder::EventTypeFilter;
use common_procedure::{EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState};
use common_event_recorder::event_table::jsonb_value;
use common_procedure::{EventRuntimeContext, EventTrigger, Procedure, ProcedureId, ProcedureState};
use crate::rpc::ddl::EventContext;
pub(crate) fn default_event_context_value() -> Value {
jsonb_value(&serde_json::to_value(EventContext::default()).unwrap())
}
pub(crate) fn procedure_trigger_value(trigger: &str) -> Value {
jsonb_value(&serde_json::json!({"type": trigger}))
}
pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) {
let state = ProcedureState::Running;
let event_context = |event_type_filter| EventContext {
let runtime_context = |event_type_filter| EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
@@ -28,7 +40,7 @@ pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) {
};
let allowed = procedure
.event(&event_context(EventTypeFilter::Only(HashSet::from([
.event(&runtime_context(EventTypeFilter::Only(HashSet::from([
event_type.to_string(),
]))))
.unwrap();
@@ -37,7 +49,7 @@ pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) {
for denied in [HashSet::from(["other_event".to_string()]), HashSet::new()] {
assert!(
procedure
.event(&event_context(EventTypeFilter::Only(denied)))
.event(&runtime_context(EventTypeFilter::Only(denied)))
.is_none()
);
}
src/common/meta/src/ddl/tests/event/view.rs
+46 -14
View File
@@ -17,17 +17,18 @@ use std::sync::Arc;
use api::v1::value::ValueData;
use api::v1::{ColumnSchema, Row, Value};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN,
PROCEDURE_TRIGGER_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, VIEW_NAME_COLUMN, jsonb_value,
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN,
PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN,
VIEW_NAME_COLUMN,
};
use common_event_recorder::testing::assert_event_contract;
use common_event_recorder::{Event, EventTypeFilter};
use common_procedure::{
ChildSubmissionOutcome, EventContext, EventTrigger, Output, Procedure, ProcedureEvent,
ChildSubmissionOutcome, EventRuntimeContext, EventTrigger, Output, Procedure, ProcedureEvent,
ProcedureId, ProcedureState, RetryPhase,
};
use super::test_util::assert_event_filter;
use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value};
use crate::ddl::create_view::CreateViewProcedure;
use crate::ddl::drop_view::DropViewProcedure;
use crate::ddl::event::view::{
@@ -35,6 +36,7 @@ use crate::ddl::event::view::{
};
use crate::ddl::tests::create_view::test_create_view_task;
use crate::ddl::tests::drop_view::new_drop_view_task;
use crate::rpc::ddl::EventContext;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
#[test]
@@ -42,7 +44,7 @@ fn test_view_submitted_event_contracts() {
let mut task = test_create_view_task("v_metrics");
task.create_view.or_replace = true;
task.create_view.create_if_not_exists = true;
let create = CreateViewProcedure::new(task, test_context());
let create = CreateViewProcedure::new(task, EventContext::default(), test_context());
let event = event_for(&create, EventTrigger::Submitted);
assert_view_event_contract(
@@ -70,7 +72,11 @@ fn test_view_submitted_event_contracts() {
assert!(!payload.contains(omitted));
}
let drop = DropViewProcedure::new(new_drop_view_task("view_name", 42, true), test_context());
let drop = DropViewProcedure::new(
new_drop_view_task("view_name", 42, true),
EventContext::default(),
test_context(),
);
let event = event_for(&drop, EventTrigger::Submitted);
assert_view_event_contract(
@@ -116,8 +122,16 @@ fn test_view_lifecycle_event_contracts() {
#[test]
fn test_view_procedures_emit_lightweight_lifecycle_events() {
let create = CreateViewProcedure::new(test_create_view_task("view_name"), test_context());
let drop = DropViewProcedure::new(new_drop_view_task("view_name", 42, false), test_context());
let create = CreateViewProcedure::new(
test_create_view_task("view_name"),
EventContext::default(),
test_context(),
);
let drop = DropViewProcedure::new(
new_drop_view_task("view_name", 42, false),
EventContext::default(),
test_context(),
);
let triggers = [
EventTrigger::Recovered,
EventTrigger::ChildSubmitted {
@@ -149,7 +163,11 @@ fn test_view_procedures_emit_lightweight_lifecycle_events() {
#[test]
fn test_create_view_succeeded_output_mapping() {
let procedure = CreateViewProcedure::new(test_create_view_task("view_name"), test_context());
let procedure = CreateViewProcedure::new(
test_create_view_task("view_name"),
EventContext::default(),
test_context(),
);
let state = ProcedureState::Done {
output: Some(Arc::new(84_u32)),
};
@@ -175,14 +193,21 @@ fn test_create_view_succeeded_output_mapping() {
#[test]
fn test_create_view_event_filter() {
let procedure = CreateViewProcedure::new(test_create_view_task("view_name"), test_context());
let procedure = CreateViewProcedure::new(
test_create_view_task("view_name"),
EventContext::default(),
test_context(),
);
assert_event_filter(&procedure, CREATE_VIEW_EVENT_TYPE);
}
#[test]
fn test_drop_view_event_filter() {
let procedure =
DropViewProcedure::new(new_drop_view_task("view_name", 42, false), test_context());
let procedure = DropViewProcedure::new(
new_drop_view_task("view_name", 42, false),
EventContext::default(),
test_context(),
);
assert_event_filter(&procedure, DROP_VIEW_EVENT_TYPE);
}
@@ -201,6 +226,7 @@ fn test_view_event_procedure_envelope_contract() {
referenced_table_count: 1,
column_count: 1,
},
EventContext::default(),
)),
ProcedureState::Running,
EventTrigger::Submitted,
@@ -254,6 +280,11 @@ impl ViewEventLocator<'_> {
.map(ValueData::U32Value)
.map(Into::into)
.unwrap_or_default(),
if self.catalog_name.is_some() {
default_event_context_value()
} else {
Value { value_data: None }
},
]
}
}
@@ -264,6 +295,7 @@ fn view_schema() -> Vec<ColumnSchema> {
SCHEMA_NAME_COLUMN.column_schema(),
VIEW_NAME_COLUMN.column_schema(),
VIEW_ID_COLUMN.column_schema(),
EVENT_CONTEXT_COLUMN.column_schema(),
]
}
@@ -302,7 +334,7 @@ fn assert_procedure_event_contract(
ValueData::StringValue(event.procedure_id.to_string()).into(),
ValueData::StringValue(state.to_string()).into(),
ValueData::StringValue(String::new()).into(),
jsonb_value(&serde_json::json!({"type": trigger})),
procedure_trigger_value(trigger),
];
values.extend(locator.values());
@@ -323,7 +355,7 @@ fn event_for_state(
lifecycle_state: &ProcedureState,
) -> Box<dyn Event> {
procedure
.event(&EventContext {
.event(&EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state,
trigger,
src/common/meta/src/ddl/truncate_table.rs
+17 -5
View File
@@ -20,7 +20,7 @@ use api::v1::region::{
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::debug;
@@ -43,7 +43,7 @@ use crate::key::table_info::TableInfoValue;
use crate::key::table_name::TableNameKey;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
use crate::metrics;
use crate::rpc::ddl::TruncateTableTask;
use crate::rpc::ddl::{EventContext, TruncateTableTask};
use crate::rpc::router::{find_leader_regions, find_leaders};
pub struct TruncateTableProcedure {
@@ -89,7 +89,10 @@ impl Procedure for TruncateTableProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::TruncateTable.as_str())
@@ -101,7 +104,11 @@ impl Procedure for TruncateTableProcedure {
let task = &self.data.task;
let locator = TableDdlLocator::new(&task.catalog, &task.schema, &task.table)
.with_table_id(task.table_id);
TableDdlEvent::truncate_table_submitted(locator, task.time_ranges.len())
TableDdlEvent::truncate_table_submitted(
locator,
task.time_ranges.len(),
self.data.event_context.clone(),
)
}
_ => TableDdlEvent::lifecycle(TableDdlEventType::TruncateTable),
};
@@ -117,10 +124,11 @@ impl TruncateTableProcedure {
task: TruncateTableTask,
table_info_value: DeserializedValueWithBytes<TableInfoValue>,
context: DdlContext,
event_context: EventContext,
) -> Self {
Self {
context,
data: TruncateTableData::new(task, table_info_value),
data: TruncateTableData::new(task, table_info_value, event_context),
}
}
@@ -227,17 +235,21 @@ pub struct TruncateTableData {
state: TruncateTableState,
task: TruncateTableTask,
table_info_value: DeserializedValueWithBytes<TableInfoValue>,
#[serde(default)]
event_context: EventContext,
}
impl TruncateTableData {
pub fn new(
task: TruncateTableTask,
table_info_value: DeserializedValueWithBytes<TableInfoValue>,
event_context: EventContext,
) -> Self {
Self {
state: TruncateTableState::Prepare,
task,
table_info_value,
event_context,
}
}
src/common/meta/src/ddl/undrop_table.rs
+35 -13
View File
@@ -17,7 +17,7 @@ use api::v1::region::{
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::tracing_context::TracingContext;
@@ -43,7 +43,7 @@ use crate::instruction::CacheIdent;
use crate::key::table_name::TableNameKey;
use crate::key::table_route::TableRouteValue;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
use crate::rpc::ddl::UndropTableTask;
use crate::rpc::ddl::{EventContext, UndropTableTask};
use crate::rpc::router::{
RegionRoute, find_follower_regions, find_followers, find_leader_regions, find_leaders,
};
@@ -56,16 +56,17 @@ pub struct UndropTableProcedure {
impl UndropTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::UndropTable";
pub fn new(task: UndropTableTask, context: DdlContext) -> Self {
Self::new_with_original_table_name(task, context, None)
pub fn new(task: UndropTableTask, context: DdlContext, event_context: EventContext) -> Self {
Self::new_with_original_table_name(task, context, None, event_context)
}
pub(crate) fn new_with_original_table_name(
task: UndropTableTask,
context: DdlContext,
table_name: Option<TableName>,
event_context: EventContext,
) -> Self {
let mut data = UndropTableData::new(task);
let mut data = UndropTableData::new(task, event_context);
data.table_name = table_name;
Self { context, data }
}
@@ -426,7 +427,10 @@ impl Procedure for UndropTableProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::UndropTable.as_str())
@@ -448,7 +452,7 @@ impl Procedure for UndropTableProcedure {
})
.unwrap_or_default()
.with_table_id(self.data.task.table_id);
TableDdlEvent::undrop_table_submitted(locator)
TableDdlEvent::undrop_table_submitted(locator, self.data.event_context.clone())
}
_ => TableDdlEvent::lifecycle(TableDdlEventType::UndropTable),
};
@@ -556,10 +560,12 @@ pub struct UndropTableData {
drop_generation: Option<String>,
#[serde(default)]
tombstone_identity_loaded: bool,
#[serde(default)]
event_context: EventContext,
}
impl UndropTableData {
fn new(task: UndropTableTask) -> Self {
fn new(task: UndropTableTask, event_context: EventContext) -> Self {
Self {
state: UndropTableState::Prepare,
task,
@@ -571,6 +577,7 @@ impl UndropTableData {
retention_expires_at: None,
drop_generation: None,
tombstone_identity_loaded: false,
event_context,
}
}
@@ -635,7 +642,11 @@ mod tests {
#[test]
fn test_map_restore_metadata_error_without_table_name() {
let context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
let procedure = UndropTableProcedure::new(UndropTableTask { table_id: 42 }, context);
let procedure = UndropTableProcedure::new(
UndropTableTask { table_id: 42 },
context,
EventContext::default(),
);
let err = procedure.map_restore_metadata_error(
error::TombstoneTargetAlreadyExistsSnafu {
@@ -650,8 +661,11 @@ mod tests {
#[test]
fn test_recovered_legacy_snapshot_reloads_tombstone_identity() {
let context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
let mut procedure =
UndropTableProcedure::new(UndropTableTask { table_id: 42 }, context.clone());
let mut procedure = UndropTableProcedure::new(
UndropTableTask { table_id: 42 },
context.clone(),
EventContext::default(),
);
procedure.data.state = UndropTableState::OpenRegions;
let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
let data = data.as_object_mut().unwrap();
@@ -676,7 +690,11 @@ mod tests {
let table_id = 1024;
let region_id = RegionId::new(table_id, 1);
let mut procedure = UndropTableProcedure::new(UndropTableTask { table_id }, context);
let mut procedure = UndropTableProcedure::new(
UndropTableTask { table_id },
context,
EventContext::default(),
);
procedure.data.table_route_value = Some(TableRouteValue::physical(vec![RegionRoute {
region: Region::new_test(region_id),
leader_peer: Some(Peer::empty(1)),
@@ -715,7 +733,11 @@ mod tests {
let table_id = 1024;
let region_id = RegionId::new(table_id, 1);
let mut procedure = UndropTableProcedure::new(UndropTableTask { table_id }, context);
let mut procedure = UndropTableProcedure::new(
UndropTableTask { table_id },
context,
EventContext::default(),
);
procedure.data.table_name = Some(TableName::new(
DEFAULT_CATALOG_NAME,
DEFAULT_SCHEMA_NAME,
src/common/meta/src/ddl_manager.rs
+139 -64
View File
@@ -70,13 +70,15 @@ use crate::rpc::ddl::DdlTask::{
DropTable, DropView, PurgeDroppedTable, TruncateTable, UndropTable,
};
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::TriggerReason;
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::trigger::CreateTriggerTask;
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::trigger::DropTriggerTask;
use crate::rpc::ddl::{
AlterDatabaseTask, AlterTableTask, CommentOnTask, CreateDatabaseTask, CreateFlowTask,
CreateTableTask, CreateViewTask, DropDatabaseTask, DropFlowTask, DropTableTask, DropViewTask,
PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse,
EventContext, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse,
TruncateTableTask, UndropTableTask,
};
@@ -177,6 +179,7 @@ pub enum RepartitionSource {
#[async_trait::async_trait]
pub trait RepartitionProcedureFactory: Send + Sync {
#[allow(clippy::too_many_arguments)]
fn create(
&self,
ddl_ctx: &DdlContext,
@@ -185,6 +188,7 @@ pub trait RepartitionProcedureFactory: Send + Sync {
source: RepartitionSource,
to_exprs: Vec<String>,
timeout: Option<Duration>,
event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError>;
fn register_loaders(
@@ -338,6 +342,7 @@ impl DdlManager {
repartition: Repartition,
wait: bool,
timeout: Duration,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
@@ -373,6 +378,7 @@ impl DdlManager {
source,
into_partition_exprs,
Some(timeout),
event_context,
)
.context(CreateRepartitionProcedureSnafu)?;
self.repartition_procedure_factory
@@ -395,6 +401,7 @@ impl DdlManager {
&self,
table_id: TableId,
alter_table_task: AlterTableTask,
event_context: EventContext,
ddl_options: DdlOptions,
) -> Result<(ProcedureId, Option<Output>)> {
// make alter_table_task mutable so we can call .take() on its field
@@ -415,6 +422,7 @@ impl DdlManager {
repartition,
ddl_options.wait,
ddl_options.timeout,
event_context,
)
.await;
}
@@ -448,6 +456,7 @@ impl DdlManager {
let procedure = AlterTableProcedure::new_with_region_locks(
table_id,
alter_table_task.clone(),
event_context.clone(),
region_ids_to_lock,
context,
)?;
@@ -481,14 +490,12 @@ impl DdlManager {
&self,
create_table_task: CreateTableTask,
query_context: QueryContext,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = CreateTableProcedure::new_with_query_context(
create_table_task,
query_context,
context,
)?;
let procedure =
CreateTableProcedure::new(create_table_task, query_context, event_context, context)?;
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
@@ -500,10 +507,11 @@ impl DdlManager {
pub async fn submit_create_view_task(
&self,
create_view_task: CreateViewTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = CreateViewProcedure::new(create_view_task, context);
let procedure = CreateViewProcedure::new(create_view_task, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
@@ -516,11 +524,16 @@ impl DdlManager {
&self,
create_table_tasks: Vec<CreateTableTask>,
physical_table_id: TableId,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure =
CreateLogicalTablesProcedure::new(create_table_tasks, physical_table_id, context);
let procedure = CreateLogicalTablesProcedure::new(
create_table_tasks,
physical_table_id,
event_context,
context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
@@ -533,11 +546,16 @@ impl DdlManager {
&self,
alter_table_tasks: Vec<AlterTableTask>,
physical_table_id: TableId,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure =
AlterLogicalTablesProcedure::new(alter_table_tasks, physical_table_id, context);
let procedure = AlterLogicalTablesProcedure::new(
alter_table_tasks,
physical_table_id,
event_context,
context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
@@ -549,10 +567,11 @@ impl DdlManager {
pub async fn submit_drop_table_task(
&self,
drop_table_task: DropTableTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = DropTableProcedure::new(drop_table_task, context);
let procedure = DropTableProcedure::new(drop_table_task, context, event_context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
@@ -565,6 +584,7 @@ impl DdlManager {
pub async fn submit_undrop_table_task(
&self,
undrop_table_task: UndropTableTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
#[cfg(not(feature = "enterprise"))]
{
@@ -590,6 +610,7 @@ impl DdlManager {
undrop_table_task,
context,
Some(original_table_name),
event_context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
@@ -603,6 +624,7 @@ impl DdlManager {
pub async fn submit_purge_dropped_table_task(
&self,
purge_dropped_table_task: PurgeDroppedTableTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
#[cfg(not(feature = "enterprise"))]
{
@@ -616,7 +638,8 @@ impl DdlManager {
#[cfg(feature = "enterprise")]
{
let context = self.create_context();
let procedure = PurgeDroppedTableProcedure::new(purge_dropped_table_task, context);
let procedure =
PurgeDroppedTableProcedure::new(purge_dropped_table_task, context, event_context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
self.execute_procedure_and_wait(procedure_with_id).await
@@ -643,8 +666,11 @@ impl DdlManager {
#[cfg(feature = "enterprise")]
{
let context = self.create_context();
let procedure =
PurgeDroppedTableProcedure::new_if_expired(purge_dropped_table_task, context);
let procedure = PurgeDroppedTableProcedure::new_if_expired(
purge_dropped_table_task,
context,
EventContext::new(TriggerReason::ScheduledGc),
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
self.execute_procedure_and_wait(procedure_with_id).await
@@ -662,6 +688,7 @@ impl DdlManager {
options,
creator,
}: CreateDatabaseTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = CreateDatabaseProcedure::new(
@@ -670,6 +697,7 @@ impl DdlManager {
create_if_not_exists,
options,
creator,
event_context,
context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
@@ -686,9 +714,11 @@ impl DdlManager {
schema,
drop_if_exists,
}: DropDatabaseTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = DropDatabaseProcedure::new(catalog, schema, drop_if_exists, context);
let procedure =
DropDatabaseProcedure::new(catalog, schema, drop_if_exists, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
self.execute_procedure_and_wait(procedure_with_id).await
@@ -697,9 +727,10 @@ impl DdlManager {
pub async fn submit_alter_database(
&self,
alter_database_task: AlterDatabaseTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = AlterDatabaseProcedure::new(alter_database_task, context)?;
let procedure = AlterDatabaseProcedure::new(alter_database_task, event_context, context)?;
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
self.execute_procedure_and_wait(procedure_with_id).await
@@ -711,9 +742,11 @@ impl DdlManager {
&self,
create_flow: CreateFlowTask,
query_context: QueryContext,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = CreateFlowProcedure::new(create_flow, query_context, context);
let procedure =
CreateFlowProcedure::new(create_flow, query_context, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
self.execute_procedure_and_wait(procedure_with_id).await
@@ -724,9 +757,10 @@ impl DdlManager {
pub async fn submit_drop_flow_task(
&self,
drop_flow: DropFlowTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = DropFlowProcedure::new(drop_flow, context);
let procedure = DropFlowProcedure::new(drop_flow, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
self.execute_procedure_and_wait(procedure_with_id).await
@@ -737,9 +771,10 @@ impl DdlManager {
pub async fn submit_drop_view_task(
&self,
drop_view: DropViewTask,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = DropViewProcedure::new(drop_view, context);
let procedure = DropViewProcedure::new(drop_view, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
self.execute_procedure_and_wait(procedure_with_id).await
@@ -751,9 +786,15 @@ impl DdlManager {
&self,
truncate_table_task: TruncateTableTask,
table_info_value: DeserializedValueWithBytes<TableInfoValue>,
event_context: EventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = TruncateTableProcedure::new(truncate_table_task, table_info_value, context);
let procedure = TruncateTableProcedure::new(
truncate_table_task,
table_info_value,
context,
event_context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
@@ -822,62 +863,75 @@ impl DdlManager {
.map(TracingContext::from_w3c)
.unwrap_or_else(TracingContext::from_current_span)
.attach(tracing::info_span!("DdlManager::submit_ddl_task"));
let ddl_options = DdlOptions {
wait: request.wait,
timeout: request.timeout,
};
let SubmitDdlTaskRequest {
query_context,
wait,
timeout,
task,
} = request;
let event_context = EventContext::from_query_context(&query_context);
let ddl_options = DdlOptions { wait, timeout };
async move {
debug!("Submitting Ddl task: {:?}", request.task);
match request.task {
debug!("Submitting Ddl task: {:?}", task);
match task {
CreateTable(create_table_task) => {
handle_create_table_task(self, create_table_task, request.query_context).await
handle_create_table_task(self, create_table_task, query_context, event_context)
.await
}
DropTable(drop_table_task) => {
handle_drop_table_task(self, drop_table_task, event_context).await
}
DropTable(drop_table_task) => handle_drop_table_task(self, drop_table_task).await,
UndropTable(undrop_table_task) => {
handle_undrop_table_task(self, undrop_table_task).await
handle_undrop_table_task(self, undrop_table_task, event_context).await
}
PurgeDroppedTable(purge_dropped_table_task) => {
handle_purge_dropped_table_task(self, purge_dropped_table_task).await
handle_purge_dropped_table_task(self, purge_dropped_table_task, event_context)
.await
}
AlterTable(alter_table_task) => {
handle_alter_table_task(self, alter_table_task, ddl_options).await
handle_alter_table_task(self, alter_table_task, ddl_options, event_context)
.await
}
TruncateTable(truncate_table_task) => {
handle_truncate_table_task(self, truncate_table_task).await
handle_truncate_table_task(self, truncate_table_task, event_context).await
}
CreateLogicalTables(create_table_tasks) => {
handle_create_logical_table_tasks(self, create_table_tasks).await
handle_create_logical_table_tasks(self, create_table_tasks, event_context).await
}
AlterLogicalTables(alter_table_tasks) => {
handle_alter_logical_table_tasks(self, alter_table_tasks).await
handle_alter_logical_table_tasks(self, alter_table_tasks, event_context).await
}
DropLogicalTables(_) => todo!(),
CreateDatabase(create_database_task) => {
handle_create_database_task(self, create_database_task).await
handle_create_database_task(self, create_database_task, event_context).await
}
DropDatabase(drop_database_task) => {
handle_drop_database_task(self, drop_database_task).await
handle_drop_database_task(self, drop_database_task, event_context).await
}
AlterDatabase(alter_database_task) => {
handle_alter_database_task(self, alter_database_task).await
handle_alter_database_task(self, alter_database_task, event_context).await
}
CreateFlow(create_flow_task) => {
handle_create_flow_task(self, create_flow_task, request.query_context).await
handle_create_flow_task(self, create_flow_task, query_context, event_context)
.await
}
DropFlow(drop_flow_task) => {
handle_drop_flow_task(self, drop_flow_task, event_context).await
}
DropFlow(drop_flow_task) => handle_drop_flow_task(self, drop_flow_task).await,
CreateView(create_view_task) => {
handle_create_view_task(self, create_view_task).await
handle_create_view_task(self, create_view_task, event_context).await
}
DropView(drop_view_task) => {
handle_drop_view_task(self, drop_view_task, event_context).await
}
DropView(drop_view_task) => handle_drop_view_task(self, drop_view_task).await,
CommentOn(comment_on_task) => handle_comment_on_task(self, comment_on_task).await,
#[cfg(feature = "enterprise")]
CreateTrigger(create_trigger_task) => {
handle_create_trigger_task(self, create_trigger_task, request.query_context)
.await
handle_create_trigger_task(self, create_trigger_task, query_context).await
}
#[cfg(feature = "enterprise")]
DropTrigger(drop_trigger_task) => {
handle_drop_trigger_task(self, drop_trigger_task, request.query_context).await
handle_drop_trigger_task(self, drop_trigger_task, query_context).await
}
}
}
@@ -889,6 +943,7 @@ impl DdlManager {
async fn handle_truncate_table_task(
ddl_manager: &DdlManager,
truncate_table_task: TruncateTableTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let table_id = truncate_table_task.table_id;
let table_metadata_manager = &ddl_manager.table_metadata_manager();
@@ -913,7 +968,7 @@ async fn handle_truncate_table_task(
);
let (id, _) = ddl_manager
.submit_truncate_table_task(truncate_table_task, table_info_value)
.submit_truncate_table_task(truncate_table_task, table_info_value, event_context)
.await?;
info!("Table: {table_id} is truncated via procedure_id {id:?}");
@@ -928,6 +983,7 @@ async fn handle_alter_table_task(
ddl_manager: &DdlManager,
alter_table_task: AlterTableTask,
ddl_options: DdlOptions,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let table_ref = alter_table_task.table_ref();
@@ -960,7 +1016,7 @@ async fn handle_alter_table_task(
);
let (id, _) = ddl_manager
.submit_alter_table_task(table_id, alter_table_task, ddl_options)
.submit_alter_table_task(table_id, alter_table_task, event_context, ddl_options)
.await?;
info!("Table: {table_id} is altered via procedure_id {id:?}");
@@ -974,9 +1030,12 @@ async fn handle_alter_table_task(
async fn handle_drop_table_task(
ddl_manager: &DdlManager,
drop_table_task: DropTableTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let table_id = drop_table_task.table_id;
let (id, _) = ddl_manager.submit_drop_table_task(drop_table_task).await?;
let (id, _) = ddl_manager
.submit_drop_table_task(drop_table_task, event_context)
.await?;
info!("Table: {table_id} is dropped via procedure_id {id:?}");
@@ -989,10 +1048,11 @@ async fn handle_drop_table_task(
async fn handle_undrop_table_task(
ddl_manager: &DdlManager,
undrop_table_task: UndropTableTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let table_id = undrop_table_task.table_id;
let (id, _) = ddl_manager
.submit_undrop_table_task(undrop_table_task)
.submit_undrop_table_task(undrop_table_task, event_context)
.await?;
info!("Table: {table_id} is undropped via procedure_id {id:?}");
@@ -1006,9 +1066,10 @@ async fn handle_undrop_table_task(
async fn handle_purge_dropped_table_task(
ddl_manager: &DdlManager,
purge_dropped_table_task: PurgeDroppedTableTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_purge_dropped_table_task(purge_dropped_table_task)
.submit_purge_dropped_table_task(purge_dropped_table_task, event_context)
.await?;
info!("Dropped table is purged via procedure_id {id:?}");
@@ -1023,9 +1084,10 @@ async fn handle_create_table_task(
ddl_manager: &DdlManager,
create_table_task: CreateTableTask,
query_context: QueryContext,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, output) = ddl_manager
.submit_create_table_task(create_table_task, query_context)
.submit_create_table_task(create_table_task, query_context, event_context)
.await?;
let procedure_id = id.to_string();
@@ -1048,6 +1110,7 @@ async fn handle_create_table_task(
async fn handle_create_logical_table_tasks(
ddl_manager: &DdlManager,
create_table_tasks: Vec<CreateTableTask>,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
ensure!(
!create_table_tasks.is_empty(),
@@ -1063,7 +1126,7 @@ async fn handle_create_logical_table_tasks(
let num_logical_tables = create_table_tasks.len();
let (id, output) = ddl_manager
.submit_create_logical_table_tasks(create_table_tasks, physical_table_id)
.submit_create_logical_table_tasks(create_table_tasks, physical_table_id, event_context)
.await?;
info!(
@@ -1092,11 +1155,12 @@ async fn handle_create_logical_table_tasks(
async fn handle_create_database_task(
ddl_manager: &DdlManager,
create_database_task: CreateDatabaseTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let catalog = create_database_task.catalog.clone();
let schema = create_database_task.schema.clone();
let (id, _) = ddl_manager
.submit_create_database(create_database_task)
.submit_create_database(create_database_task, event_context)
.await?;
let procedure_id = id.to_string();
@@ -1114,9 +1178,10 @@ async fn handle_create_database_task(
async fn handle_drop_database_task(
ddl_manager: &DdlManager,
drop_database_task: DropDatabaseTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_drop_database(drop_database_task.clone())
.submit_drop_database(drop_database_task.clone(), event_context)
.await?;
let procedure_id = id.to_string();
@@ -1134,9 +1199,10 @@ async fn handle_drop_database_task(
async fn handle_alter_database_task(
ddl_manager: &DdlManager,
alter_database_task: AlterDatabaseTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_alter_database(alter_database_task.clone())
.submit_alter_database(alter_database_task.clone(), event_context)
.await?;
let procedure_id = id.to_string();
@@ -1155,9 +1221,10 @@ async fn handle_alter_database_task(
async fn handle_drop_flow_task(
ddl_manager: &DdlManager,
drop_flow_task: DropFlowTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_drop_flow_task(drop_flow_task.clone())
.submit_drop_flow_task(drop_flow_task.clone(), event_context)
.await?;
let procedure_id = id.to_string();
@@ -1199,9 +1266,10 @@ async fn handle_drop_trigger_task(
async fn handle_drop_view_task(
ddl_manager: &DdlManager,
drop_view_task: DropViewTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_drop_view_task(drop_view_task.clone())
.submit_drop_view_task(drop_view_task.clone(), event_context)
.await?;
let procedure_id = id.to_string();
@@ -1221,9 +1289,10 @@ async fn handle_create_flow_task(
ddl_manager: &DdlManager,
create_flow_task: CreateFlowTask,
query_context: QueryContext,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, output) = ddl_manager
.submit_create_flow_task(create_flow_task.clone(), query_context)
.submit_create_flow_task(create_flow_task.clone(), query_context, event_context)
.await?;
let procedure_id = id.to_string();
@@ -1280,6 +1349,7 @@ async fn handle_create_trigger_task(
async fn handle_alter_logical_table_tasks(
ddl_manager: &DdlManager,
alter_table_tasks: Vec<AlterTableTask>,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
ensure!(
!alter_table_tasks.is_empty(),
@@ -1299,7 +1369,7 @@ async fn handle_alter_logical_table_tasks(
let num_logical_tables = alter_table_tasks.len();
let (id, _) = ddl_manager
.submit_alter_logical_table_tasks(alter_table_tasks, physical_table_id)
.submit_alter_logical_table_tasks(alter_table_tasks, physical_table_id, event_context)
.await?;
info!(
@@ -1318,9 +1388,10 @@ async fn handle_alter_logical_table_tasks(
async fn handle_create_view_task(
ddl_manager: &DdlManager,
create_view_task: CreateViewTask,
event_context: EventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, output) = ddl_manager
.submit_create_view_task(create_view_task)
.submit_create_view_task(create_view_task, event_context)
.await?;
let procedure_id = id.to_string();
@@ -1398,7 +1469,7 @@ mod tests {
use crate::procedure_executor::ExecutorContext;
use crate::region_keeper::MemoryRegionKeeper;
use crate::region_registry::LeaderRegionRegistry;
use crate::rpc::ddl::{CreatorGrantIntent, UndropTableTask};
use crate::rpc::ddl::{CreatorGrantIntent, EventContext, UndropTableTask};
#[cfg(not(feature = "enterprise"))]
use crate::rpc::ddl::{DdlTask, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest};
use crate::sequence::SequenceBuilder;
@@ -1435,6 +1506,7 @@ mod tests {
_source: RepartitionSource,
_to_exprs: Vec<String>,
_timeout: Option<Duration>,
_event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError> {
unimplemented!()
}
@@ -1608,7 +1680,7 @@ mod tests {
let ddl_manager = build_soft_drop_test_ddl_manager();
let err = ddl_manager
.submit_undrop_table_task(UndropTableTask { table_id: 1024 })
.submit_undrop_table_task(UndropTableTask { table_id: 1024 }, EventContext::default())
.await
.unwrap_err();
@@ -1622,13 +1694,16 @@ mod tests {
let ddl_manager = build_soft_drop_test_ddl_manager();
let err = ddl_manager
.submit_undrop_table_task(UndropTableTask { table_id: 1024 })
.submit_undrop_table_task(UndropTableTask { table_id: 1024 }, EventContext::default())
.await
.unwrap_err();
assert!(matches!(err, crate::error::Error::Unsupported { .. }));
let err = ddl_manager
.submit_purge_dropped_table_task(PurgeDroppedTableTask { table_id: 1024 })
.submit_purge_dropped_table_task(
PurgeDroppedTableTask { table_id: 1024 },
EventContext::default(),
)
.await
.unwrap_err();
assert!(matches!(err, crate::error::Error::Unsupported { .. }));
src/common/meta/src/rpc/ddl.rs
+130 -1
View File
@@ -44,11 +44,13 @@ use base64::Engine as _;
use base64::engine::general_purpose;
use common_catalog::{format_full_flow_name, format_full_table_name};
use common_error::ext::BoxedError;
use common_session::channel_protocol;
use common_time::{DatabaseTimeToLive, Timestamp};
use prost::Message;
use serde::{Deserialize, Serialize};
use serde_with::{DefaultOnNull, serde_as};
use snafu::{OptionExt, ResultExt};
use strum::{AsRefStr, EnumString};
use table::metadata::{TableId, TableInfo};
use table::requests::validate_database_option;
use table::table_name::TableName;
@@ -66,7 +68,8 @@ use crate::key::table_name::{TableNameKey, TableNameManager};
/// Reserved query-context extension key for the frontend peer address that submitted a DDL request.
pub const ORIGIN_FRONTEND_ADDR_EXTENSION_KEY: &str = "__greptime_origin_frontend.addr";
/// Reserved query-context extension key for the trigger reason supplied by frontend.
pub const TRIGGER_REASON_EXTENSION_KEY: &str = "__greptime_event.trigger_reason";
/// Reserved query-context extension key for the authenticated database creator.
pub const CREATE_DATABASE_CREATOR_EXTENSION_KEY: &str = "__greptime_create_database.creator";
/// Internal gRPC metadata key for the authenticated database creator.
@@ -1665,6 +1668,86 @@ pub struct QueryContext {
pub sst_min_sequences: HashMap<u64, u64>,
}
impl QueryContext {
/// Returns the protocol name represented by the wire channel value.
pub fn channel_protocol(&self) -> Option<&'static str> {
channel_protocol(self.channel)
}
}
/// The stable context recorded for a procedure event.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct EventContext {
pub reason: TriggerReason,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub protocol: Option<String>,
#[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
pub extensions: serde_json::Map<String, serde_json::Value>,
}
impl EventContext {
/// Creates an event context with no additional extensions.
pub fn new(reason: TriggerReason) -> Self {
Self {
reason,
protocol: None,
extensions: Default::default(),
}
}
/// Adds the protocol that originated the operation.
pub fn with_protocol(mut self, protocol: impl Into<String>) -> Self {
self.protocol = Some(protocol.into());
self
}
/// Builds an event context from frontend query context metadata.
pub fn from_query_context(query_context: &QueryContext) -> Self {
let reason = query_context
.extensions
.get(TRIGGER_REASON_EXTENSION_KEY)
.map(|reason| TriggerReason::from_extension(reason))
.unwrap_or_default();
let context = Self::new(reason);
if let Some(protocol) = query_context.channel_protocol() {
context.with_protocol(protocol)
} else {
context
}
}
}
impl Default for EventContext {
fn default() -> Self {
Self::new(TriggerReason::default())
}
}
/// The stable classification of a procedure trigger.
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, AsRefStr, EnumString,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum TriggerReason {
Manual,
AutoCreate,
AutoAlter,
AutoRepartition,
AutoRebalance,
RegionFailover,
ScheduledGc,
#[default]
#[serde(other)]
Unknown,
}
impl TriggerReason {
pub fn from_extension(value: &str) -> Self {
value.parse().unwrap_or_default()
}
}
impl QueryContext {
/// Get the current catalog
pub fn current_catalog(&self) -> &str {
@@ -2237,4 +2320,50 @@ mod tests {
assert_eq!(pb_roundtrip.channel, pb.channel);
assert_eq!(pb_roundtrip.snapshot_seqs, pb.snapshot_seqs);
}
#[test]
fn test_trigger_reason_deserializes_unknown_value() {
let reason: TriggerReason = serde_json::from_str("\"future_reason\"").unwrap();
assert_eq!(TriggerReason::Unknown, reason);
}
#[test]
fn test_event_context_serialization() {
let context = EventContext::new(TriggerReason::Manual).with_protocol("mysql");
assert_eq!(
serde_json::json!({
"reason": "manual",
"protocol": "mysql",
}),
serde_json::to_value(context).unwrap()
);
assert_eq!(
serde_json::json!({ "reason": "manual" }),
serde_json::to_value(EventContext::new(TriggerReason::Manual)).unwrap()
);
}
#[test]
fn test_event_context_from_query_context_preserves_extensions() {
let mut query_context = QueryContext::default();
query_context.extensions.insert(
TRIGGER_REASON_EXTENSION_KEY.to_string(),
TriggerReason::AutoCreate.as_ref().to_string(),
);
query_context.channel = 4;
let event_context = EventContext::from_query_context(&query_context);
assert_eq!(event_context.reason, TriggerReason::AutoCreate);
assert_eq!(event_context.protocol.as_deref(), Some("prometheus"));
assert_eq!(
query_context
.extensions
.get(TRIGGER_REASON_EXTENSION_KEY)
.map(String::as_str),
Some("auto_create")
);
}
}
src/common/procedure/src/lib.rs
+3 -3
View File
@@ -30,8 +30,8 @@ pub use crate::error::{Error, Result};
pub use crate::event::ProcedureEvent;
pub use crate::procedure::{
BoxedProcedure, BoxedProcedureLoader, ChildSubmissionOutcome, Context, ContextProvider,
ContextProviderRef, EventContext, EventTrigger, LockKey, Output, ParseIdError, PoisonKey,
PoisonKeys, Procedure, ProcedureId, ProcedureInfo, ProcedureManager, ProcedureManagerRef,
ProcedureState, ProcedureWithId, RetryPhase, Status, StringKey,
ContextProviderRef, EventRuntimeContext, EventTrigger, LockKey, Output, ParseIdError,
PoisonKey, PoisonKeys, Procedure, ProcedureId, ProcedureInfo, ProcedureManager,
ProcedureManagerRef, ProcedureState, ProcedureWithId, RetryPhase, Status, StringKey,
};
pub use crate::watcher::Watcher;
src/common/procedure/src/local.rs
+6 -6
View File
@@ -1053,8 +1053,8 @@ mod tests {
use crate::store::state_store::ObjectStateStore;
use crate::test_util::InMemoryPoisonStore;
use crate::{
ChildSubmissionOutcome, Context, EventContext, EventTrigger, Procedure, ProcedureEvent,
Status,
ChildSubmissionOutcome, Context, EventRuntimeContext, EventTrigger, Procedure,
ProcedureEvent, Status,
};
fn new_test_manager_context() -> ManagerContext {
@@ -1296,7 +1296,7 @@ mod tests {
self.poison_keys.clone()
}
fn event(&self, _ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
Some(Box::new(TestProcedureEvent))
}
}
@@ -1354,7 +1354,7 @@ mod tests {
LockKey::default()
}
fn event(&self, _ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
Some(Box::new(TestProcedureEvent))
}
}
@@ -1397,7 +1397,7 @@ mod tests {
LockKey::default()
}
fn event(&self, _ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
Some(Box::new(TestProcedureEvent))
}
}
@@ -1424,7 +1424,7 @@ mod tests {
LockKey::default()
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
*self.captured_filter.lock().unwrap() = Some(ctx.event_type_filter.clone());
Some(Box::new(TestProcedureEvent))
}
src/common/procedure/src/local/runner.rs
+3 -3
View File
@@ -34,8 +34,8 @@ use crate::procedure::{Output, StringKey};
use crate::rwlock::OwnedKeyRwLockGuard;
use crate::store::{ProcedureMessage, ProcedureStore};
use crate::{
BoxedProcedure, ChildSubmissionOutcome, Context, Error, EventContext, EventTrigger, Procedure,
ProcedureId, ProcedureState, ProcedureWithId, RetryPhase, Status,
BoxedProcedure, ChildSubmissionOutcome, Context, Error, EventRuntimeContext, EventTrigger,
Procedure, ProcedureId, ProcedureState, ProcedureWithId, RetryPhase, Status,
};
/// A guard to cleanup procedure state.
@@ -776,7 +776,7 @@ impl Runner {
pub(crate) fn build_event(&self, trigger: EventTrigger) -> Option<ProcedureEvent> {
let recorder = self.event_recorder.as_ref()?;
let state = self.meta.state();
let context = EventContext {
let context = EventRuntimeContext {
procedure_id: self.meta.id,
lifecycle_state: &state,
trigger: trigger.clone(),
src/common/procedure/src/procedure.rs
+4 -4
View File
@@ -237,13 +237,13 @@ pub trait Procedure: Send {
/// [`Event::extra_schema`] values. The event recorder batches events by type
/// and rejects incompatible schemas; use a distinct event type for a
/// different schema.
fn event(&self, _ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
None
}
}
/// Framework-owned context supplied when a procedure builds a lifecycle event.
pub struct EventContext<'a> {
pub struct EventRuntimeContext<'a> {
/// Id of the procedure associated with the event.
pub procedure_id: ProcedureId,
/// Current framework state of the procedure.
@@ -336,7 +336,7 @@ impl<T: Procedure + ?Sized> Procedure for Box<T> {
(**self).poison_keys()
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
(**self).event(ctx)
}
}
@@ -721,7 +721,7 @@ mod tests {
#[test]
fn test_default_procedure_event_hook() {
let state = ProcedureState::Running;
let context = EventContext {
let context = EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Succeeded,
src/common/session/src/lib.rs
+48 -1
View File
@@ -15,6 +15,27 @@
use serde::{Deserialize, Serialize};
use strum::{AsRefStr, Display, EnumString};
/// Returns the protocol name represented by a wire query channel value.
pub fn channel_protocol(channel: u8) -> Option<&'static str> {
match channel {
1 => Some("mysql"),
2 => Some("postgres"),
3 => Some("httpsql"),
4 => Some("prometheus"),
5 => Some("otlp"),
6 => Some("grpc"),
7 => Some("influx"),
8 => Some("opentsdb"),
9 => Some("loki"),
10 => Some("elasticsearch"),
11 => Some("jaeger"),
12 => Some("log"),
13 => Some("promql"),
14 => Some("splunk"),
_ => None,
}
}
/// Defines the read preference for frontend route operations,
/// determining whether to read from the region leader or follower.
#[derive(
@@ -31,7 +52,33 @@ pub enum ReadPreference {
mod tests {
use std::str::FromStr;
use crate::ReadPreference;
use crate::{ReadPreference, channel_protocol};
#[test]
fn test_channel_protocol() {
let expected = [
(1, "mysql"),
(2, "postgres"),
(3, "httpsql"),
(4, "prometheus"),
(5, "otlp"),
(6, "grpc"),
(7, "influx"),
(8, "opentsdb"),
(9, "loki"),
(10, "elasticsearch"),
(11, "jaeger"),
(12, "log"),
(13, "promql"),
(14, "splunk"),
];
for (channel, protocol) in expected {
assert_eq!(Some(protocol), channel_protocol(channel));
}
assert_eq!(None, channel_protocol(0));
assert_eq!(None, channel_protocol(15));
}
#[test]
fn test_read_preference() {
src/flow/src/adapter/util.rs
+2 -1
View File
@@ -21,6 +21,7 @@ use api::v1::column_def::options_from_column_schema;
use api::v1::{ColumnDataType, ColumnDataTypeExtension, CreateTableExpr, SemanticType};
use common_error::ext::BoxedError;
use common_meta::key::table_info::TableInfoValue;
use common_meta::rpc::ddl::TriggerReason;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
use itertools::Itertools;
@@ -131,7 +132,7 @@ impl StreamingEngine {
.build(),
);
stmt_exec
.create_table_inner(&mut create_table, None, ctx)
.create_table_inner(&mut create_table, None, ctx, TriggerReason::AutoCreate)
.await
.map_err(BoxedError::new)
.context(ExternalSnafu)?;
src/frontend/src/instance/dashboard.rs
+7 -1
View File
@@ -24,6 +24,7 @@ use async_trait::async_trait;
use auth::{DASHBOARD_DELETE, DASHBOARD_QUERY, DASHBOARD_SAVE, PermissionReq};
use common_catalog::consts::{DEFAULT_PRIVATE_SCHEMA_NAME, default_engine};
use common_error::ext::BoxedError;
use common_meta::rpc::ddl::TriggerReason;
use common_query::OutputData;
use common_recordbatch::util as record_util;
use common_telemetry::info;
@@ -165,7 +166,12 @@ impl Instance {
};
self.statement_executor
.create_table_inner(&mut create_table_expr, None, ctx.clone())
.create_table_inner(
&mut create_table_expr,
None,
ctx.clone(),
TriggerReason::AutoCreate,
)
.await
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)?;
src/frontend/src/instance/grpc.rs
+8 -2
View File
@@ -31,6 +31,7 @@ use auth::{
};
use common_error::ext::BoxedError;
use common_grpc::flight::do_put::DoPutResponse;
use common_meta::rpc::ddl::TriggerReason;
use common_query::Output;
use common_query::logical_plan::add_insert_to_logical_plan;
use common_telemetry::tracing::{self};
@@ -187,7 +188,12 @@ impl GrpcQueryHandler for Instance {
operator::expr_helper::validate_create_expr(&expr)?;
let _ = self
.statement_executor
.create_table_inner(&mut expr, None, ctx.clone())
.create_table_inner(
&mut expr,
None,
ctx.clone(),
TriggerReason::Manual,
)
.await?;
Output::new_with_affected_rows(0)
}
@@ -200,7 +206,7 @@ impl GrpcQueryHandler for Instance {
}
DdlExpr::AlterTable(expr) => {
self.statement_executor
.alter_table_inner(expr, ctx.clone())
.alter_table_inner(expr, ctx.clone(), TriggerReason::Manual)
.await?
}
DdlExpr::CreateDatabase(expr) => {
src/frontend/src/instance/otlp/trace_ingest.rs
+2 -1
View File
@@ -24,6 +24,7 @@ use api::v1::{
use client::Output;
use common_error::ext::{BoxedError, ErrorExt};
use common_error::status_code::StatusCode;
use common_meta::rpc::ddl::TriggerReason;
use common_telemetry::warn;
use datatypes::prelude::ConcreteDataType;
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
@@ -1678,7 +1679,7 @@ impl Instance {
if let Err(err) = self
.statement_executor
.alter_table_inner(alter_expr, ctx.clone())
.alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
.await
{
let table = self
src/frontend/src/instance/prom_store.rs
+5 -4
View File
@@ -30,6 +30,7 @@ use auth::{
use client::OutputData;
use common_catalog::{format_full_table_name, parse_optional_catalog_and_schema_from_db_string};
use common_error::ext::BoxedError;
use common_meta::rpc::ddl::TriggerReason;
use common_query::Output;
use common_query::prelude::{GREPTIME_PHYSICAL_TABLE, greptime_value};
use common_recordbatch::RecordBatches;
@@ -342,7 +343,7 @@ impl PendingRowsSchemaAlterer for Instance {
AutoCreateTableType::Logical(_) => {
// Use the batch API for logical tables.
self.statement_executor
.create_logical_tables(&create_exprs, ctx)
.create_logical_tables(&create_exprs, ctx, TriggerReason::AutoCreate)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?;
@@ -353,7 +354,7 @@ impl PendingRowsSchemaAlterer for Instance {
expr.table_options
.insert(SST_FORMAT_KEY.to_string(), "flat".to_string());
self.statement_executor
.create_table_inner(&mut expr, None, ctx.clone())
.create_table_inner(&mut expr, None, ctx.clone(), TriggerReason::AutoCreate)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?;
@@ -420,7 +421,7 @@ impl PendingRowsSchemaAlterer for Instance {
}
self.statement_executor
.alter_logical_tables(alter_exprs, ctx)
.alter_logical_tables(alter_exprs, ctx, TriggerReason::AutoAlter)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?;
@@ -686,7 +687,7 @@ impl Instance {
fill_metric_physical_table_options(&mut create_table_expr.table_options);
self.statement_executor
.create_table_inner(&mut create_table_expr, None, ctx)
.create_table_inner(&mut create_table_expr, None, ctx, TriggerReason::AutoCreate)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?;
src/meta-srv/Cargo.toml
-1
View File
@@ -100,6 +100,5 @@ common-event-recorder = { workspace = true, features = ["testing"] }
common-meta = { workspace = true, features = ["testing"] }
common-procedure-test.workspace = true
common-wal = { workspace = true, features = ["testing"] }
session.workspace = true
tracing = "0.1"
tracing-subscriber.workspace = true
src/meta-srv/src/event/gc.rs
+47 -19
View File
@@ -21,9 +21,10 @@ use api::v1::{ColumnSchema, Row, Value};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
GC_REPORT_COLUMN, REGION_ID_COLUMN, REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas,
nullable_json,
EVENT_CONTEXT_COLUMN, GC_REPORT_COLUMN, REGION_ID_COLUMN, REGION_NUMBER_COLUMN,
TABLE_ID_COLUMN, column_schemas, nullable_json,
};
use common_meta::rpc::ddl::EventContext;
use serde::Serialize;
use snafu::ResultExt;
use store_api::storage::{GcReport, IndexVersion, RegionId};
@@ -66,6 +67,7 @@ struct BatchGcRegionRow {
#[derive(Debug)]
pub(crate) struct BatchGcEvent {
payload: Option<BatchGcPayload>,
event_context: Option<EventContext>,
// None emits a procedure-level lifecycle row with null Region dimensions.
regions: Option<Vec<BatchGcRegionRow>>,
}
@@ -75,6 +77,7 @@ impl BatchGcEvent {
regions: &[RegionId],
full_file_listing: bool,
timeout: Duration,
event_context: Option<EventContext>,
) -> Self {
Self {
payload: Some(BatchGcPayload {
@@ -83,6 +86,7 @@ impl BatchGcEvent {
full_file_listing,
timeout,
}),
event_context,
regions: None,
}
}
@@ -107,6 +111,7 @@ impl BatchGcEvent {
(!regions.is_empty()).then_some(Self {
payload: None,
event_context: None,
regions: Some(regions),
})
}
@@ -131,8 +136,22 @@ impl Event for BatchGcEvent {
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
let Some(regions) = &self.regions else {
return Ok(vec![null_row()]);
return Ok(vec![Row {
values: vec![
Value { value_data: None },
Value { value_data: None },
Value { value_data: None },
Value { value_data: None },
nullable_json(event_context.as_ref()),
],
}]);
};
regions
@@ -145,6 +164,7 @@ impl Event for BatchGcEvent {
ValueData::U32Value(region.region_id.table_id()).into(),
ValueData::U32Value(region.region_id.region_number()).into(),
nullable_json(Some(&report)),
nullable_json(None),
],
})
})
@@ -162,9 +182,11 @@ fn schema() -> Vec<ColumnSchema> {
&TABLE_ID_COLUMN,
&REGION_NUMBER_COLUMN,
&GC_REPORT_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
#[cfg(test)]
fn null_row() -> Row {
Row {
values: (0..schema().len())
@@ -221,7 +243,7 @@ mod tests {
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::sequence::SequenceBuilder;
use common_procedure::{
EventContext, EventTrigger, Procedure, ProcedureEvent, ProcedureId, ProcedureState,
EventRuntimeContext, EventTrigger, Procedure, ProcedureEvent, ProcedureId, ProcedureState,
RetryPhase,
};
use store_api::storage::FileId;
@@ -234,8 +256,12 @@ mod tests {
fn test_batch_gc_lifecycle_event_contract() {
let first = RegionId::new(1024, 1);
let second = RegionId::new(1024, 2);
let event =
BatchGcEvent::with_config(&[second, first, second], true, Duration::from_secs(10));
let event = BatchGcEvent::with_config(
&[second, first, second],
true,
Duration::from_secs(10),
None,
);
assert_event_contract(&event, BATCH_GC_EVENT_TYPE, &schema(), &[null_row()]);
assert_eq!(
@@ -330,7 +356,7 @@ mod tests {
fn test_batch_gc_event_filter() {
let procedure = batch_gc_procedure();
let running = ProcedureState::Running;
let event_context = |trigger, lifecycle_state, event_type_filter| EventContext {
let runtime_context = |trigger, lifecycle_state, event_type_filter| EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state,
trigger,
@@ -339,7 +365,7 @@ mod tests {
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Submitted,
&running,
EventTypeFilter::All,
@@ -356,7 +382,7 @@ mod tests {
};
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Succeeded,
&done,
EventTypeFilter::All,
@@ -366,7 +392,7 @@ mod tests {
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Recovered,
&running,
EventTypeFilter::All,
@@ -375,7 +401,7 @@ mod tests {
);
let retrying = procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Retrying {
phase: RetryPhase::Execute,
attempt: 1,
@@ -397,7 +423,7 @@ mod tests {
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Recovered,
&running,
EventTypeFilter::Only(HashSet::from(["another_event".to_string()])),
@@ -406,7 +432,7 @@ mod tests {
);
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Recovered,
&running,
EventTypeFilter::Only(HashSet::new()),
@@ -417,7 +443,7 @@ mod tests {
let missing = ProcedureState::Done { output: None };
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Succeeded,
&missing,
EventTypeFilter::All,
@@ -429,7 +455,7 @@ mod tests {
};
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Succeeded,
&wrong,
EventTypeFilter::All,
@@ -442,7 +468,7 @@ mod tests {
fn test_batch_gc_terminal_event_with_report() {
let mut procedure = batch_gc_procedure();
let running = ProcedureState::Running;
let event_context = |trigger| EventContext {
let runtime_context = |trigger| EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &running,
trigger,
@@ -455,7 +481,7 @@ mod tests {
procedure.set_gc_report_for_test(report);
let retrying = procedure
.event(&event_context(EventTrigger::Retrying {
.event(&runtime_context(EventTrigger::Retrying {
phase: RetryPhase::Execute,
attempt: 2,
}))
@@ -472,7 +498,7 @@ mod tests {
assert_eq!(retrying.extra_rows().unwrap(), vec![null_row()]);
for trigger in [EventTrigger::Failed, EventTrigger::Poisoned] {
let event = procedure.event(&event_context(trigger)).unwrap();
let event = procedure.event(&runtime_context(trigger)).unwrap();
assert_eq!(event.json_payload().unwrap(), serde_json::Value::Null);
assert_eq!(
event.extra_rows().unwrap(),
@@ -487,7 +513,7 @@ mod tests {
let procedure = batch_gc_procedure();
let failed_without_report = procedure
.event(&event_context(EventTrigger::Failed))
.event(&runtime_context(EventTrigger::Failed))
.unwrap();
assert_eq!(
failed_without_report.json_payload().unwrap(),
@@ -563,6 +589,7 @@ mod tests {
ValueData::U32Value(region_id.table_id()).into(),
ValueData::U32Value(region_id.region_number()).into(),
nullable_json(report.as_ref()),
nullable_json(None),
],
}
}
@@ -580,6 +607,7 @@ mod tests {
true,
Duration::from_secs(10),
HashMap::new(),
EventContext::default(),
)
}
}
src/meta-srv/src/event/region_migration.rs
+14 -4
View File
@@ -20,9 +20,10 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
REGION_ID_COLUMN, REGION_MIGRATION_DST_NODE_ID_COLUMN, REGION_MIGRATION_DST_PEER_ADDR_COLUMN,
REGION_MIGRATION_SRC_NODE_ID_COLUMN, REGION_MIGRATION_SRC_PEER_ADDR_COLUMN,
REGION_MIGRATION_TRIGGER_REASON_COLUMN, REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas,
EVENT_CONTEXT_COLUMN, REGION_ID_COLUMN, REGION_MIGRATION_DST_NODE_ID_COLUMN,
REGION_MIGRATION_DST_PEER_ADDR_COLUMN, REGION_MIGRATION_SRC_NODE_ID_COLUMN,
REGION_MIGRATION_SRC_PEER_ADDR_COLUMN, REGION_MIGRATION_TRIGGER_REASON_COLUMN,
REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas, nullable_json,
};
use serde::Serialize;
use snafu::ResultExt;
@@ -49,6 +50,7 @@ pub(crate) struct RegionMigrationEvent {
dst_peer_addr: String,
// The timeout of the region migration.
timeout: Duration,
event_context: Option<common_meta::rpc::ddl::EventContext>,
}
#[derive(Debug, Serialize)]
@@ -58,7 +60,7 @@ struct Payload {
}
impl RegionMigrationEvent {
pub fn from_persistent_ctx(ctx: &PersistentContext) -> Self {
pub fn from_persistent_ctx(ctx: &PersistentContext, include_event_context: bool) -> Self {
Self {
region_ids: ctx.region_ids.clone(),
trigger_reason: ctx.trigger_reason,
@@ -67,6 +69,7 @@ impl RegionMigrationEvent {
dst_node_id: ctx.to_peer.id,
dst_peer_addr: ctx.to_peer.addr.clone(),
timeout: ctx.timeout,
event_context: include_event_context.then(|| ctx.event_context.clone()),
}
}
}
@@ -86,6 +89,7 @@ impl Event for RegionMigrationEvent {
&REGION_MIGRATION_SRC_PEER_ADDR_COLUMN,
&REGION_MIGRATION_DST_NODE_ID_COLUMN,
&REGION_MIGRATION_DST_PEER_ADDR_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
@@ -102,6 +106,12 @@ impl Event for RegionMigrationEvent {
ValueData::StringValue(self.src_peer_addr.clone()).into(),
ValueData::U64Value(self.dst_node_id).into(),
ValueData::StringValue(self.dst_peer_addr.clone()).into(),
self.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)
.map(|value| nullable_json(value.as_ref()))?,
],
});
}
src/meta-srv/src/event/repartition.rs
+32 -7
View File
@@ -21,11 +21,11 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, PARENT_PROCEDURE_ID_COLUMN, REPARTITION_GROUP_ID_COLUMN,
SCHEMA_NAME_COLUMN, SOURCE_PARTITION_EXPR_COLUMN, SOURCE_REGION_ID_COLUMN,
SOURCE_REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN, TARGET_PARTITION_EXPR_COLUMN,
TARGET_REGION_ID_COLUMN, TARGET_REGION_NUMBER_COLUMN, column_schemas, nullable_string,
nullable_value,
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PARENT_PROCEDURE_ID_COLUMN,
REPARTITION_GROUP_ID_COLUMN, SCHEMA_NAME_COLUMN, SOURCE_PARTITION_EXPR_COLUMN,
SOURCE_REGION_ID_COLUMN, SOURCE_REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN,
TARGET_PARTITION_EXPR_COLUMN, TARGET_REGION_ID_COLUMN, TARGET_REGION_NUMBER_COLUMN,
column_schemas, nullable_json, nullable_string, nullable_value,
};
use serde::Serialize;
use snafu::ResultExt;
@@ -61,6 +61,7 @@ pub(crate) struct RepartitionEvent {
table_name: Option<String>,
table_id: Option<TableId>,
payload: Option<RepartitionSubmittedPayload>,
event_context: Option<common_meta::rpc::ddl::EventContext>,
}
impl RepartitionEvent {
@@ -90,6 +91,7 @@ impl RepartitionEvent {
target_partition_columns: intent.target_partition_columns().map(ToOwned::to_owned),
timeout: persistent_ctx.timeout,
}),
event_context: Some(persistent_ctx.event_context.clone()),
}
}
@@ -100,10 +102,17 @@ impl RepartitionEvent {
table_name: None,
table_id: None,
payload: None,
event_context: None,
}
}
fn schema() -> Vec<ColumnSchema> {
let mut schema = Self::base_schema();
schema.push(EVENT_CONTEXT_COLUMN.column_schema());
schema
}
fn base_schema() -> Vec<ColumnSchema> {
column_schemas([
&CATALOG_NAME_COLUMN,
&SCHEMA_NAME_COLUMN,
@@ -132,12 +141,19 @@ impl Event for RepartitionEvent {
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
Ok(vec![Row {
values: vec![
nullable_string(self.catalog_name.as_deref()),
nullable_string(self.schema_name.as_deref()),
nullable_string(self.table_name.as_deref()),
nullable_value(self.table_id.map(ValueData::U32Value)),
nullable_json(event_context.as_ref()),
],
}])
}
@@ -307,7 +323,7 @@ impl RepartitionGroupEvent {
}
fn schema() -> Vec<ColumnSchema> {
let mut schema = RepartitionEvent::schema();
let mut schema = RepartitionEvent::base_schema();
schema.extend(column_schemas([
&PARENT_PROCEDURE_ID_COLUMN,
&REPARTITION_GROUP_ID_COLUMN,
@@ -368,6 +384,7 @@ mod tests {
PROCEDURE_TRIGGER_COLUMN, jsonb_value,
};
use common_event_recorder::testing::assert_event_contract;
use common_meta::rpc::ddl::EventContext;
use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState};
use table::table_name::TableName;
use uuid::Uuid;
@@ -385,6 +402,7 @@ mod tests {
TableName::new("greptime", "public", "repartition_events"),
1024,
Some(Duration::from_secs(30)),
EventContext::default(),
)
}
@@ -420,6 +438,9 @@ mod tests {
ValueData::StringValue("public".to_string()).into(),
ValueData::StringValue("repartition_events".to_string()).into(),
ValueData::U32Value(1024).into(),
jsonb_value(&serde_json::json!({
"reason": "unknown",
})),
],
}],
);
@@ -674,6 +695,9 @@ mod tests {
ValueData::StringValue("public".to_string()).into(),
ValueData::StringValue("repartition_events".to_string()).into(),
ValueData::U32Value(1024).into(),
jsonb_value(&serde_json::json!({
"reason": "unknown",
})),
],
}],
);
@@ -685,11 +709,12 @@ mod tests {
&SCHEMA_NAME_COLUMN,
&TABLE_NAME_COLUMN,
&TABLE_ID_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
fn group_schema() -> Vec<ColumnSchema> {
let mut schema = parent_schema();
let mut schema = RepartitionEvent::base_schema();
schema.extend(column_schemas([
&PARENT_PROCEDURE_ID_COLUMN,
&REPARTITION_GROUP_ID_COLUMN,
src/meta-srv/src/gc/ctx.rs
+6
View File
@@ -29,6 +29,7 @@ use common_meta::key::DroppedTableName;
use common_meta::key::TableMetadataManagerRef;
use common_meta::key::table_repart::TableRepartValue;
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::rpc::ddl::EventContext;
#[cfg(feature = "enterprise")]
use common_meta::rpc::ddl::PurgeDroppedTableTask;
use common_procedure::{ProcedureManagerRef, ProcedureWithId, watcher};
@@ -67,6 +68,7 @@ pub(crate) trait SchedulerCtx: Send + Sync {
full_file_listing: bool,
timeout: Duration,
region_routes_override: Region2Peers,
event_context: EventContext,
) -> Result<GcReport>;
#[cfg(feature = "enterprise")]
@@ -254,12 +256,14 @@ impl SchedulerCtx for DefaultGcSchedulerCtx {
full_file_listing: bool,
timeout: Duration,
region_routes_override: Region2Peers,
event_context: EventContext,
) -> Result<GcReport> {
self.gc_regions_inner(
region_ids,
full_file_listing,
timeout,
region_routes_override,
event_context,
)
.await
}
@@ -306,6 +310,7 @@ impl DefaultGcSchedulerCtx {
full_file_listing: bool,
timeout: Duration,
region_routes_override: Region2Peers,
event_context: EventContext,
) -> Result<GcReport> {
debug!(
"Sending GC instruction for {} regions (full_file_listing: {})",
@@ -321,6 +326,7 @@ impl DefaultGcSchedulerCtx {
full_file_listing,
timeout,
region_routes_override,
event_context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
src/meta-srv/src/gc/handler.rs
+9 -1
View File
@@ -18,6 +18,7 @@ use std::time::Instant;
use common_catalog::consts::MITO_ENGINE;
use common_meta::datanode::{RegionManifestInfo, RegionStat};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_telemetry::tracing::Instrument as _;
use common_telemetry::{debug, error, info, warn};
use futures::StreamExt;
@@ -36,7 +37,7 @@ use crate::gc::tracker::RegionGcInfo;
use crate::metrics::METRIC_META_GC_CANDIDATE_REGIONS;
impl GcScheduler {
pub(crate) async fn trigger_gc(&self) -> Result<GcJobReport> {
pub(crate) async fn trigger_gc(&self, event_context: EventContext) -> Result<GcJobReport> {
let start_time = Instant::now();
info!("Starting GC cycle");
@@ -107,6 +108,7 @@ impl GcScheduler {
datanode_to_candidates,
dropped_assignment.force_full_listing,
dropped_assignment.region_routes_override,
event_context,
)
.instrument(common_telemetry::tracing::info_span!(
"meta_gc_dispatch_to_datanodes"
@@ -210,6 +212,7 @@ impl GcScheduler {
datanode_to_candidates: HashMap<Peer, Vec<(TableId, GcCandidate)>>,
force_full_listing_by_peer: HashMap<Peer, HashSet<RegionId>>,
region_routes_override_by_peer: HashMap<Peer, Region2Peers>,
event_context: EventContext,
) -> GcJobReport {
let mut per_datanode_reports = HashMap::new();
let mut failed_datanodes: HashMap<_, Vec<_>> = HashMap::new();
@@ -231,6 +234,7 @@ impl GcScheduler {
.get(&peer)
.cloned()
.unwrap_or_default();
let event_context = event_context.clone();
async move {
(
peer,
@@ -240,6 +244,7 @@ impl GcScheduler {
candidates,
force_full_listing,
region_routes_override,
event_context,
)
.await,
)
@@ -278,6 +283,7 @@ impl GcScheduler {
candidates: Vec<(TableId, GcCandidate)>,
force_full_listing: HashSet<RegionId>,
region_routes_override: Region2Peers,
event_context: EventContext,
) -> Result<GcReport> {
info!(
"Starting GC for datanode {} with {} candidate regions",
@@ -327,6 +333,7 @@ impl GcScheduler {
false,
self.config.mailbox_timeout,
region_routes_override.clone(),
event_context.clone(),
)
.instrument(common_telemetry::tracing::info_span!(
"meta_gc_call_datanode",
@@ -359,6 +366,7 @@ impl GcScheduler {
true,
self.config.mailbox_timeout,
region_routes_override,
event_context,
)
.instrument(common_telemetry::tracing::info_span!(
"meta_gc_call_datanode",
src/meta-srv/src/gc/mock.rs
+4
View File
@@ -30,6 +30,7 @@ use common_meta::datanode::{RegionManifestInfo, RegionStat};
use common_meta::key::table_repart::TableRepartValue;
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::{Region, RegionRoute};
use common_telemetry::debug;
use ordered_float::OrderedFloat;
@@ -74,6 +75,7 @@ pub struct MockSchedulerCtx {
pub candidates: Arc<Mutex<Option<HashMap<TableId, Vec<GcCandidate>>>>>,
pub get_table_to_region_stats_calls: Arc<Mutex<usize>>,
pub gc_regions_calls: Arc<Mutex<usize>>,
pub gc_event_contexts: Arc<Mutex<Vec<EventContext>>>,
// Error injection fields for testing
pub get_table_to_region_stats_error: Arc<Mutex<Option<crate::error::Error>>>,
pub get_table_route_error: Arc<Mutex<Option<crate::error::Error>>>,
@@ -212,8 +214,10 @@ impl SchedulerCtx for MockSchedulerCtx {
_full_file_listing: bool,
_timeout: Duration,
_region_routes_override: Region2Peers,
event_context: EventContext,
) -> Result<GcReport> {
*self.gc_regions_calls.lock().unwrap() += 1;
self.gc_event_contexts.lock().unwrap().push(event_context);
// Check per-region error injection first (for any region)
for &region_id in region_ids {
src/meta-srv/src/gc/mock/basic.rs
+70 -2
View File
@@ -17,6 +17,7 @@ use std::sync::{Arc, Mutex};
use std::time::Instant;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_telemetry::init_default_ut_logging;
use store_api::region_engine::RegionRole;
use store_api::storage::{FileId, FileRefsManifest, GcReport, RegionId};
@@ -31,7 +32,12 @@ async fn test_parallel_process_datanodes_empty() {
let env = TestEnv::new();
let report = env
.scheduler
.parallel_process_datanodes(HashMap::new(), HashMap::new(), HashMap::new())
.parallel_process_datanodes(
HashMap::new(),
HashMap::new(),
HashMap::new(),
EventContext::default(),
)
.await;
match report {
@@ -95,7 +101,12 @@ async fn test_parallel_process_datanodes_with_candidates() {
)]);
let report = scheduler
.parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new())
.parallel_process_datanodes(
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
)
.await;
match report {
@@ -176,6 +187,10 @@ async fn test_handle_tick() {
assert_eq!(*ctx.get_table_to_region_stats_calls.lock().unwrap(), 1);
assert_eq!(*ctx.gc_regions_calls.lock().unwrap(), 1);
assert_eq!(
ctx.gc_event_contexts.lock().unwrap().as_slice(),
&[EventContext::new(TriggerReason::ScheduledGc)]
);
let tracker = scheduler.region_gc_tracker.lock().await;
assert!(
@@ -184,3 +199,56 @@ async fn test_handle_tick() {
tracker
);
}
#[tokio::test]
async fn test_handle_manual_gc_without_regions_records_manual_event_context() {
init_default_ut_logging();
let table_id = 1;
let region_id = RegionId::new(table_id, 1);
let peer = Peer::new(1, "");
let candidates = HashMap::from([(table_id, vec![new_candidate(region_id, 1.0)])]);
let file_refs = FileRefsManifest {
manifest_version: HashMap::from([(region_id, 1)]),
..Default::default()
};
let ctx = Arc::new(
MockSchedulerCtx {
table_to_region_stats: Arc::new(Mutex::new(Some(HashMap::from([(
table_id,
vec![mock_region_stat(
region_id,
RegionRole::Leader,
TEST_REGION_SIZE_200MB,
10,
)],
)])))),
gc_reports: Arc::new(Mutex::new(HashMap::from([(
region_id,
GcReport::default(),
)]))),
candidates: Arc::new(Mutex::new(Some(candidates))),
file_refs: Arc::new(Mutex::new(Some(file_refs))),
..Default::default()
}
.with_table_routes(HashMap::from([(
table_id,
(table_id, vec![(region_id, peer)]),
)])),
);
let scheduler = GcScheduler {
ctx: ctx.clone(),
runtime_switch_manager: crate::gc::scheduler::new_test_runtime_switch_manager(),
receiver: GcScheduler::channel().1,
config: GcSchedulerOptions::default(),
region_gc_tracker: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
last_tracker_cleanup: Arc::new(tokio::sync::Mutex::new(Instant::now())),
};
scheduler.handle_manual_gc(None, None, None).await.unwrap();
assert_eq!(
ctx.gc_event_contexts.lock().unwrap().as_slice(),
&[EventContext::new(TriggerReason::Manual)]
);
}
src/meta-srv/src/gc/mock/concurrent.rs
+26 -4
View File
@@ -18,6 +18,7 @@ use std::time::{Duration, Instant};
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::{Region, RegionRoute};
use common_telemetry::{info, init_default_ut_logging};
use store_api::region_engine::RegionRole;
@@ -97,7 +98,12 @@ async fn test_concurrent_table_processing_limits() {
)]);
let report = scheduler
.parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new())
.parallel_process_datanodes(
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
)
.await;
// Should process all datanodes
@@ -180,7 +186,12 @@ async fn test_datanode_processes_tables_with_partial_gc_failures() {
)]);
let report = scheduler
.parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new())
.parallel_process_datanodes(
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
)
.await;
// Should have one datanode with mixed results
@@ -285,6 +296,7 @@ async fn test_region_gc_concurrency_limit() {
candidates.into_iter().map(|c| (table_id, c)).collect(),
HashSet::new(),
HashMap::new(),
EventContext::default(),
)
.await
.unwrap();
@@ -397,7 +409,12 @@ async fn test_region_gc_concurrency_with_partial_failures() {
)]);
let report = scheduler
.parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new())
.parallel_process_datanodes(
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
)
.await;
let report = match &report {
@@ -536,7 +553,12 @@ async fn test_region_gc_concurrency_with_retryable_errors() {
candidates.into_iter().map(|c| (table_id, c)).collect(),
)]);
let report = scheduler
.parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new())
.parallel_process_datanodes(
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
)
.await;
let report = match &report {
src/meta-srv/src/gc/mock/full_list.rs
+6
View File
@@ -17,6 +17,7 @@ use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_telemetry::init_default_ut_logging;
use store_api::region_engine::RegionRole;
use store_api::storage::{FileId, FileRefsManifest, GcReport, RegionId};
@@ -82,6 +83,7 @@ async fn test_full_file_listing_first_time_gc() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
)
.await
.unwrap();
@@ -155,6 +157,7 @@ async fn test_full_file_listing_interval_enforcement() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
)
.await
.unwrap();
@@ -181,6 +184,7 @@ async fn test_full_file_listing_interval_enforcement() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
)
.await
.unwrap();
@@ -256,6 +260,7 @@ async fn test_full_file_listing_no_interval_passed() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
)
.await
.unwrap();
@@ -279,6 +284,7 @@ async fn test_full_file_listing_no_interval_passed() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
)
.await
.unwrap();
src/meta-srv/src/gc/mock/misc.rs
+13 -2
View File
@@ -17,6 +17,7 @@ use std::sync::{Arc, Mutex};
use std::time::Instant;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_telemetry::init_default_ut_logging;
use store_api::storage::{FileRefsManifest, GcReport, RegionId};
@@ -71,7 +72,12 @@ async fn test_empty_file_refs_manifest() {
)]);
let report = scheduler
.parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new())
.parallel_process_datanodes(
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
)
.await;
match report {
@@ -159,7 +165,12 @@ async fn test_multiple_regions_per_table() {
)]);
let report = scheduler
.parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new())
.parallel_process_datanodes(
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
)
.await;
match report {
src/meta-srv/src/gc/procedure.rs
+38 -7
View File
@@ -24,10 +24,11 @@ use common_meta::key::table_repart::TableRepartValue;
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::lock_key::{RegionLock, TableLock};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_procedure::error::ToJsonSnafu;
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, ProcedureState, Result as ProcedureResult, Status,
Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger,
LockKey, Procedure, ProcedureState, Result as ProcedureResult, Status,
};
use common_telemetry::tracing::Instrument as _;
use common_telemetry::tracing_context::TracingContext;
@@ -199,6 +200,8 @@ pub struct BatchGcData {
/// mailbox timeout duration
timeout: Duration,
gc_report: Option<GcReport>,
#[serde(default)]
event_context: EventContext,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
@@ -216,6 +219,7 @@ pub enum State {
impl BatchGcProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::BatchGcProcedure";
#[allow(clippy::too_many_arguments)]
pub fn new(
mailbox: MailboxRef,
table_metadata_manager: TableMetadataManagerRef,
@@ -224,6 +228,7 @@ impl BatchGcProcedure {
full_file_listing: bool,
timeout: Duration,
region_routes_override: Region2Peers,
event_context: EventContext,
) -> Self {
Self {
mailbox,
@@ -239,6 +244,7 @@ impl BatchGcProcedure {
related_regions: HashMap::new(),
file_refs: FileRefsManifest::default(),
gc_report: None,
event_context,
},
}
}
@@ -269,6 +275,7 @@ impl BatchGcProcedure {
related_regions: HashMap::new(),
file_refs,
gc_report: Some(GcReport::default()),
event_context: EventContext::default(),
},
}
}
@@ -1065,16 +1072,26 @@ impl Procedure for BatchGcProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(BATCH_GC_EVENT_TYPE) {
return None;
}
let event = match &ctx.trigger {
// Keep normal GC low-noise; only terminal events with meaningful reports are recorded.
EventTrigger::Submitted
| EventTrigger::Recovered
| EventTrigger::ChildSubmitted { .. } => return None,
// Keep scheduled GC low-noise; record submitted manual requests for auditability.
EventTrigger::Submitted => (self.data.event_context.reason == TriggerReason::Manual)
.then(|| {
BatchGcEvent::with_config(
&self.data.regions,
self.data.full_file_listing,
self.data.timeout,
Some(self.data.event_context.clone()),
)
})?,
EventTrigger::Recovered | EventTrigger::ChildSubmitted { .. } => return None,
EventTrigger::Succeeded => {
let ProcedureState::Done {
output: Some(output),
@@ -1089,6 +1106,7 @@ impl Procedure for BatchGcProcedure {
&self.data.regions,
self.data.full_file_listing,
self.data.timeout,
None,
),
EventTrigger::Failed | EventTrigger::Poisoned => self
.data
@@ -1100,6 +1118,7 @@ impl Procedure for BatchGcProcedure {
&self.data.regions,
self.data.full_file_listing,
self.data.timeout,
None,
)
}),
};
@@ -1192,6 +1211,16 @@ mod tests {
assert!(report.need_retry_regions.contains(&region_id));
}
#[test]
fn test_batch_gc_data_defaults_missing_event_context() {
let mut data = serde_json::to_value(&batch_gc_procedure().data).unwrap();
data.as_object_mut().unwrap().remove("event_context");
let recovered: BatchGcData = serde_json::from_value(data).unwrap();
assert_eq!(recovered.event_context, EventContext::default());
}
#[tokio::test]
async fn test_send_gc_instructions_preserves_partial_report() {
let first_region = RegionId::new(1024, 1);
@@ -1226,6 +1255,7 @@ mod tests {
true,
Duration::from_secs(10),
HashMap::new(),
EventContext::default(),
);
procedure.data.region_routes = HashMap::from([
(first_region, (first_peer, vec![])),
@@ -1266,6 +1296,7 @@ mod tests {
true,
Duration::from_secs(10),
HashMap::new(),
EventContext::default(),
)
}
}
src/meta-srv/src/gc/scheduler.rs
+19 -3
View File
@@ -18,6 +18,7 @@ use std::time::{Duration, Instant};
use common_meta::DatanodeId;
use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_telemetry::tracing::Instrument as _;
use common_telemetry::{error, info};
use snafu::ResultExt;
@@ -216,7 +217,10 @@ impl GcScheduler {
.await;
}
let span = common_telemetry::tracing::info_span!("meta_gc_handle_tick");
let report = self.trigger_gc().instrument(span).await?;
let report = self
.trigger_gc(EventContext::new(TriggerReason::ScheduledGc))
.instrument(span)
.await?;
// Periodically clean up stale tracker entries
self.cleanup_tracker_if_needed().await?;
@@ -300,7 +304,9 @@ impl GcScheduler {
// No specific regions, use default tick behavior
let Some(regions) = region_ids else {
let report = self.trigger_gc().await?;
let report = self
.trigger_gc(EventContext::new(TriggerReason::Manual))
.await?;
info!("Finished manual gc request");
return Ok(report);
};
@@ -347,6 +353,7 @@ impl GcScheduler {
full_listing,
gc_timeout,
Region2Peers::new(),
EventContext::new(TriggerReason::Manual),
)
.await?;
combined_report.merge(report);
@@ -355,7 +362,13 @@ impl GcScheduler {
if !dropped_regions.is_empty() {
let report = self
.ctx
.gc_regions(&dropped_regions, true, gc_timeout, dropped_routes_override)
.gc_regions(
&dropped_regions,
true,
gc_timeout,
dropped_routes_override,
EventContext::new(TriggerReason::Manual),
)
.await?;
combined_report.merge(report);
}
@@ -477,6 +490,7 @@ mod tests {
_full_file_listing: bool,
_timeout: Duration,
_region_routes_override: Region2Peers,
_event_context: EventContext,
) -> Result<GcReport> {
self.gc_regions_calls.fetch_add(1, Ordering::Relaxed);
panic!("gc_regions should not be called in maintenance mode")
@@ -550,6 +564,7 @@ mod tests {
_full_file_listing: bool,
_timeout: Duration,
_region_routes_override: Region2Peers,
_event_context: EventContext,
) -> Result<GcReport> {
Ok(GcReport::default())
}
@@ -883,6 +898,7 @@ mod tests {
_full_file_listing: bool,
_timeout: Duration,
_region_routes_override: Region2Peers,
_event_context: EventContext,
) -> Result<GcReport> {
crate::error::UnexpectedSnafu {
violated: "mock gc failure".to_string(),
src/meta-srv/src/procedure/region_migration.rs
+91 -7
View File
@@ -45,11 +45,13 @@ use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
use common_meta::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock};
use common_meta::peer::Peer;
use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard};
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_procedure::error::{
Error as ProcedureError, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu,
};
use common_procedure::{
Context as ProcedureContext, EventContext, LockKey, Procedure, Status, StringKey,
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status,
StringKey,
};
use common_telemetry::{debug, error, info};
use manager::RegionMigrationProcedureGuard;
@@ -124,6 +126,8 @@ pub struct PersistentContext {
/// The trigger reason of region migration.
#[serde(default)]
pub(crate) trigger_reason: RegionMigrationTriggerReason,
#[serde(default)]
pub(crate) event_context: EventContext,
}
impl PersistentContext {
@@ -133,7 +137,7 @@ impl PersistentContext {
to_peer: Peer,
region_ids: Vec<RegionId>,
timeout: Duration,
trigger_reason: RegionMigrationTriggerReason,
event_context: EventContext,
) -> Self {
#[allow(deprecated)]
Self {
@@ -144,7 +148,37 @@ impl PersistentContext {
to_peer,
region_ids,
timeout,
trigger_reason,
trigger_reason: RegionMigrationTriggerReason::from_trigger_reason(event_context.reason),
event_context,
}
}
fn normalize_trigger_reasons(&mut self) {
if self.trigger_reason != RegionMigrationTriggerReason::Unknown {
self.event_context.reason = self.trigger_reason.to_trigger_reason();
} else if self.event_context.reason != TriggerReason::Unknown {
self.trigger_reason =
RegionMigrationTriggerReason::from_trigger_reason(self.event_context.reason);
}
}
}
impl RegionMigrationTriggerReason {
fn to_trigger_reason(self) -> TriggerReason {
match self {
Self::Manual => TriggerReason::Manual,
Self::AutoRebalance => TriggerReason::AutoRebalance,
Self::Failover => TriggerReason::RegionFailover,
Self::Unknown => TriggerReason::Unknown,
}
}
fn from_trigger_reason(reason: TriggerReason) -> Self {
match reason {
TriggerReason::Manual => Self::Manual,
TriggerReason::AutoRebalance => Self::AutoRebalance,
TriggerReason::RegionFailover => Self::Failover,
_ => Self::Unknown,
}
}
}
@@ -818,9 +852,10 @@ impl RegionMigrationProcedure {
tracker: RegionMigrationProcedureTracker,
) -> ProcedureResult<Self> {
let RegionMigrationDataOwned {
persistent_ctx,
mut persistent_ctx,
state,
} = serde_json::from_str(json).context(FromJsonSnafu)?;
persistent_ctx.normalize_trigger_reasons();
let guards = persistent_ctx
.region_ids
.iter()
@@ -964,13 +999,14 @@ impl Procedure for RegionMigrationProcedure {
LockKey::new(self.context.persistent_ctx.lock_key())
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
if !ctx.event_type_filter.allows(REGION_MIGRATION_EVENT_TYPE) {
return None;
}
Some(Box::new(RegionMigrationEvent::from_persistent_ctx(
&self.context.persistent_ctx,
matches!(ctx.trigger, EventTrigger::Submitted),
)))
}
}
@@ -1027,7 +1063,7 @@ mod tests {
let procedure = RegionMigrationProcedure::new(persistent_context, context, vec![]);
let serialized = procedure.dump().unwrap();
let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown","event_context":{"reason":"unknown"}},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
assert_eq!(expected, serialized);
}
@@ -1051,7 +1087,7 @@ mod tests {
for trigger in triggers {
let event = procedure
.event(&EventContext {
.event(&EventRuntimeContext {
procedure_id: common_procedure::ProcedureId::random(),
lifecycle_state: &state,
trigger,
@@ -1076,12 +1112,60 @@ mod tests {
region_ids: vec![RegionId::new(1024, 1)],
timeout: Duration::from_secs(10),
trigger_reason: RegionMigrationTriggerReason::default(),
event_context: EventContext::default(),
};
// NOTES: Changes it will break backward compatibility.
let serialized = r#"{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105}"#;
let deserialized: PersistentContext = serde_json::from_str(serialized).unwrap();
assert_eq!(persistent_ctx, deserialized);
assert_eq!(deserialized.event_context, EventContext::default());
}
#[test]
fn test_new_persistent_context_keeps_event_context() {
let mut event_context =
EventContext::new(TriggerReason::AutoRebalance).with_protocol("internal");
event_context
.extensions
.insert("scheduler".into(), serde_json::json!("load_based"));
let context = PersistentContext::new(
vec![("greptime".into(), "public".into())],
Peer::empty(1),
Peer::empty(2),
vec![RegionId::new(1024, 1)],
Duration::from_secs(10),
event_context.clone(),
);
assert_eq!(
context.trigger_reason,
RegionMigrationTriggerReason::AutoRebalance
);
assert_eq!(context.event_context, event_context);
}
#[test]
fn test_normalize_trigger_reasons() {
let mut context = new_persistent_context();
context.trigger_reason = RegionMigrationTriggerReason::Manual;
context.event_context = EventContext::default();
context.normalize_trigger_reasons();
assert_eq!(context.event_context.reason, TriggerReason::Manual);
context.trigger_reason = RegionMigrationTriggerReason::Unknown;
context.event_context = EventContext::new(TriggerReason::RegionFailover);
context.normalize_trigger_reasons();
assert_eq!(
context.trigger_reason,
RegionMigrationTriggerReason::Failover
);
context.trigger_reason = RegionMigrationTriggerReason::Manual;
context.event_context = EventContext::new(TriggerReason::RegionFailover);
context.normalize_trigger_reasons();
assert_eq!(context.event_context.reason, TriggerReason::Manual);
}
#[derive(Debug, Serialize, Deserialize, Default)]
src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs
+2 -2
View File
@@ -382,6 +382,7 @@ mod tests {
use common_meta::key::table_route::TableRouteValue;
use common_meta::key::test_utils::new_test_table_info;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::router::{Region, RegionRoute};
use common_meta::wal_provider::RegionWalOptions;
use store_api::storage::RegionId;
@@ -389,7 +390,6 @@ mod tests {
use super::*;
use crate::error::Error;
use crate::procedure::region_migration::manager::RegionMigrationTriggerReason;
use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context};
use crate::procedure::region_migration::{ContextFactory, PersistentContext};
use crate::procedure::test_util::{
@@ -403,7 +403,7 @@ mod tests {
Peer::empty(2),
vec![RegionId::new(1024, 1)],
Duration::from_millis(1000),
RegionMigrationTriggerReason::Manual,
EventContext::new(TriggerReason::Manual),
)
}
src/meta-srv/src/procedure/region_migration/manager.rs
+3 -2
View File
@@ -21,6 +21,7 @@ use std::time::Duration;
use common_meta::key::table_info::TableInfoValue;
use common_meta::key::table_route::TableRouteValue;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::RegionRoute;
use common_procedure::{ProcedureId, ProcedureManagerRef, ProcedureWithId, watcher};
use common_telemetry::{error, info, warn};
@@ -485,7 +486,7 @@ impl RegionMigrationManager {
task.to_peer.clone(),
task.region_ids.clone(),
task.timeout,
task.trigger_reason,
EventContext::new(task.trigger_reason.to_trigger_reason()),
),
self.context_factory.clone(),
procedure_guards,
@@ -581,7 +582,7 @@ impl RegionMigrationManager {
to_peer,
vec![region_id],
timeout,
trigger_reason,
EventContext::new(trigger_reason.to_trigger_reason()),
),
self.context_factory.clone(),
vec![guard],
src/meta-srv/src/procedure/region_migration/test_util.rs
+3 -4
View File
@@ -27,6 +27,7 @@ use common_meta::kv_backend::KvBackendRef;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::peer::Peer;
use common_meta::region_keeper::{MemoryRegionKeeper, MemoryRegionKeeperRef};
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::RegionRoute;
use common_meta::sequence::SequenceBuilder;
use common_meta::state_store::KvStateStore;
@@ -45,9 +46,7 @@ use crate::metasrv::MetasrvInfo;
use crate::procedure::region_migration::close_downgraded_region::CloseDowngradedRegion;
use crate::procedure::region_migration::downgrade_leader_region::DowngradeLeaderRegion;
use crate::procedure::region_migration::flush_leader_region::PreFlushRegion;
use crate::procedure::region_migration::manager::{
RegionMigrationProcedureTracker, RegionMigrationTriggerReason,
};
use crate::procedure::region_migration::manager::RegionMigrationProcedureTracker;
use crate::procedure::region_migration::migration_abort::RegionMigrationAbort;
use crate::procedure::region_migration::migration_end::RegionMigrationEnd;
use crate::procedure::region_migration::open_candidate_region::OpenCandidateRegion;
@@ -191,7 +190,7 @@ pub fn new_persistent_context(from: u64, to: u64, region_id: RegionId) -> Persis
Peer::empty(to),
vec![region_id],
Duration::from_secs(10),
RegionMigrationTriggerReason::default(),
EventContext::default(),
)
}
src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
+2 -2
View File
@@ -356,6 +356,7 @@ mod tests {
use common_meta::key::topic_name::TopicNameKey;
use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey, TopicRegionValue};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::router::{Region, RegionRoute};
use common_meta::wal_provider::RegionWalOptions;
use common_wal::options::KafkaWalOptions;
@@ -363,7 +364,6 @@ mod tests {
use super::*;
use crate::error::Error;
use crate::procedure::region_migration::manager::RegionMigrationTriggerReason;
use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context};
use crate::procedure::region_migration::{ContextFactory, PersistentContext};
use crate::procedure::test_util::{
@@ -377,7 +377,7 @@ mod tests {
Peer::empty(2),
vec![RegionId::new(1024, 1)],
Duration::from_millis(1000),
RegionMigrationTriggerReason::Manual,
EventContext::new(TriggerReason::Manual),
)
}
src/meta-srv/src/procedure/repartition.rs
+32 -10
View File
@@ -44,11 +44,12 @@ use common_meta::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
use common_meta::node_manager::NodeManagerRef;
use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard};
use common_meta::region_registry::LeaderRegionRegistryRef;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::{RegionRoute, operating_leader_region_roles};
use common_meta::wal_provider::RegionWalOptions;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventContext,
BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext,
EventTrigger, LockKey, Procedure, ProcedureManagerRef, Result as ProcedureResult, Status,
StringKey,
};
@@ -103,6 +104,8 @@ pub struct PersistentContext {
#[serde(default)]
/// Records table-level partition metadata updated by this repartition.
pub partition_metadata_update: Option<PartitionMetadataUpdate>,
#[serde(default)]
pub event_context: EventContext,
}
fn default_timeout() -> Duration {
@@ -110,7 +113,7 @@ fn default_timeout() -> Duration {
}
impl PersistentContext {
/// Creates a new [PersistentContext] with the given table name, table id and timeout.
/// Creates a new [PersistentContext] with the table, timeout and event context.
///
/// If the timeout is not provided, the default timeout will be used.
pub fn new(
@@ -121,6 +124,7 @@ impl PersistentContext {
}: TableName,
table_id: TableId,
timeout: Option<Duration>,
event_context: EventContext,
) -> Self {
Self {
catalog_name,
@@ -132,6 +136,7 @@ impl PersistentContext {
unknown_procedures: vec![],
timeout: timeout.unwrap_or_else(default_timeout),
partition_metadata_update: None,
event_context,
}
}
@@ -792,7 +797,10 @@ impl Procedure for RepartitionProcedure {
LockKey::new(self.context.persistent_ctx.lock_key())
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(REPARTITION_EVENT_TYPE) {
return None;
}
@@ -861,6 +869,7 @@ impl RepartitionProcedureFactory for GcDisabledRepartitionProcedureFactory {
_source: RepartitionSource,
_to_exprs: Vec<String>,
_timeout: Option<Duration>,
_event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError> {
Err(BoxedError::new(
error::InvalidArgumentsSnafu {
@@ -899,8 +908,9 @@ impl RepartitionProcedureFactory for DefaultRepartitionProcedureFactory {
source: RepartitionSource,
to_exprs: Vec<String>,
timeout: Option<Duration>,
event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError> {
let persistent_ctx = PersistentContext::new(table_name, table_id, timeout);
let persistent_ctx = PersistentContext::new(table_name, table_id, timeout, event_context);
let from = match source {
RepartitionSource::Partitioned {
exprs,
@@ -1133,6 +1143,7 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
Context::new(
@@ -1164,6 +1175,7 @@ mod tests {
},
vec![],
None,
EventContext::default(),
)
.err()
.expect("GC-disabled factory must reject repartition");
@@ -1265,7 +1277,7 @@ mod tests {
let all = Arc::new(EventTypeFilter::All);
let submitted = procedure
.event(&EventContext {
.event(&EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
@@ -1276,7 +1288,7 @@ mod tests {
assert_ne!(submitted.json_payload().unwrap(), serde_json::Value::Null);
let allowed = procedure
.event(&EventContext {
.event(&EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
@@ -1288,7 +1300,7 @@ mod tests {
assert_eq!(allowed.event_type(), REPARTITION_EVENT_TYPE);
let succeeded = procedure
.event(&EventContext {
.event(&EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Succeeded,
@@ -1297,7 +1309,7 @@ mod tests {
.unwrap();
assert_eq!(succeeded.json_payload().unwrap(), serde_json::Value::Null);
let filtered = procedure.event(&EventContext {
let filtered = procedure.event(&EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
@@ -1307,7 +1319,7 @@ mod tests {
});
assert!(filtered.is_none());
let empty = procedure.event(&EventContext {
let empty = procedure.event(&EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
@@ -1370,7 +1382,7 @@ mod tests {
}
#[test]
fn test_persistent_context_partition_metadata_update_serde_default() {
fn test_persistent_context_serde_defaults_for_new_fields() {
let json = r#"{
"catalog_name":"test_catalog",
"schema_name":"test_schema",
@@ -1382,7 +1394,10 @@ mod tests {
let persistent_ctx: PersistentContext = serde_json::from_str(json).unwrap();
assert!(persistent_ctx.failed_procedures.is_empty());
assert!(persistent_ctx.unknown_procedures.is_empty());
assert!(persistent_ctx.partition_metadata_update.is_none());
assert_eq!(persistent_ctx.event_context, EventContext::default());
}
#[tokio::test]
@@ -1459,6 +1474,7 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![with_rollback_metadata(
test_plan(table_id),
@@ -1525,6 +1541,7 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![test_plan(table_id)];
let context = Context::new(
@@ -1578,6 +1595,7 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
let failed_plan = test_plan(table_id);
let failed_plan = with_rollback_metadata(
@@ -1711,6 +1729,7 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![failed_plan, succeeded_plan.clone()];
persistent_ctx.failed_procedures = vec![ProcedureMeta {
@@ -1820,6 +1839,7 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![plan.clone()];
persistent_ctx.unknown_procedures = vec![ProcedureMeta {
@@ -1881,6 +1901,7 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![with_rollback_metadata(
test_plan(table_id),
@@ -2009,6 +2030,7 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![failed_merge_plan, succeeded_split_plan.clone()];
persistent_ctx.failed_procedures = vec![ProcedureMeta {
src/meta-srv/src/procedure/repartition/collect.rs
+3
View File
@@ -136,6 +136,7 @@ mod tests {
use common_error::mock::MockError;
use common_error::status_code::StatusCode;
use common_meta::rpc::ddl::EventContext;
use common_meta::test_util::MockDatanodeManager;
use common_procedure::{
Context as ProcedureContext, ContextProvider, Error as ProcedureError, ProcedureId,
@@ -193,6 +194,7 @@ mod tests {
table::table_name::TableName::new("test_catalog", "test_schema", "test_table"),
1024,
None,
EventContext::default(),
);
let mut ctx = crate::procedure::repartition::Context::new(
&ddl_ctx,
@@ -227,6 +229,7 @@ mod tests {
table::table_name::TableName::new("test_catalog", "test_schema", "test_table"),
1024,
None,
EventContext::default(),
);
let mut ctx = crate::procedure::repartition::Context::new(
&ddl_ctx,
src/meta-srv/src/procedure/repartition/group.rs
+13 -8
View File
@@ -38,8 +38,8 @@ use common_meta::peer::Peer;
use common_meta::rpc::router::RegionRoute;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, ProcedureId, Result as ProcedureResult, Status, StringKey,
Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger,
LockKey, Procedure, ProcedureId, Result as ProcedureResult, Status, StringKey,
};
use common_telemetry::{error, info};
use serde::{Deserialize, Serialize};
@@ -262,7 +262,10 @@ impl Procedure for RepartitionGroupProcedure {
LockKey::new(self.context.persistent_ctx.lock_key())
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(REPARTITION_GROUP_EVENT_TYPE) {
return None;
}
@@ -631,7 +634,9 @@ mod tests {
use common_event_recorder::EventTypeFilter;
use common_meta::key::TableMetadataManager;
use common_meta::kv_backend::test_util::MockKvBackendBuilder;
use common_procedure::{EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState};
use common_procedure::{
EventRuntimeContext, EventTrigger, Procedure, ProcedureId, ProcedureState,
};
use crate::error::Error;
use crate::event::repartition::REPARTITION_GROUP_EVENT_TYPE;
@@ -708,7 +713,7 @@ mod tests {
context: env.create_context(new_persistent_context(1024, vec![], vec![])),
};
let state = ProcedureState::Running;
let event_context = |event_type_filter| EventContext {
let runtime_context = |event_type_filter| EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
@@ -716,7 +721,7 @@ mod tests {
};
let allowed = procedure
.event(&event_context(EventTypeFilter::Only(HashSet::from([
.event(&runtime_context(EventTypeFilter::Only(HashSet::from([
REPARTITION_GROUP_EVENT_TYPE.to_string(),
]))))
.unwrap();
@@ -724,14 +729,14 @@ mod tests {
assert!(
procedure
.event(&event_context(EventTypeFilter::Only(HashSet::from([
.event(&runtime_context(EventTypeFilter::Only(HashSet::from([
"another_event".to_string(),
]))))
.is_none()
);
assert!(
procedure
.event(&event_context(EventTypeFilter::Only(HashSet::new())))
.event(&runtime_context(EventTypeFilter::Only(HashSet::new())))
.is_none()
);
}
src/meta-srv/src/procedure/repartition/test_util.rs
+2
View File
@@ -24,6 +24,7 @@ use common_meta::kv_backend::KvBackendRef;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::node_manager::NodeManagerRef;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::{Region, RegionRoute};
use common_meta::sequence::SequenceBuilder;
use common_meta::test_util::new_ddl_context_with_kv_backend;
@@ -224,6 +225,7 @@ pub fn new_parent_context(
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
ParentContext::new(
src/meta-srv/src/procedure/tests.rs
+11 -3
View File
@@ -29,7 +29,7 @@ use common_meta::ddl::test_util::create_table::{
};
use common_meta::key::table_route::{PhysicalTableRouteValue, TableRouteValue};
use common_meta::node_manager::NodeManagerRef;
use common_meta::rpc::ddl::CreateTableTask;
use common_meta::rpc::ddl::{CreateTableTask, EventContext, QueryContext};
use common_meta::rpc::router::{RegionRoute, find_leaders};
use common_procedure::Status;
use store_api::storage::RegionId;
@@ -91,6 +91,8 @@ fn create_table_task(table_name: Option<&str>) -> CreateTableTask {
fn test_region_request_builder() {
let mut procedure = CreateTableProcedure::new(
create_table_task(None),
QueryContext::default(),
EventContext::default(),
test_data::new_ddl_context(Arc::new(NodeClients::default())),
)
.unwrap();
@@ -188,6 +190,8 @@ async fn test_on_datanode_create_regions() {
let mut procedure = CreateTableProcedure::new(
create_table_task(None),
QueryContext::default(),
EventContext::default(),
test_data::new_ddl_context(node_manager),
)
.unwrap();
@@ -260,8 +264,12 @@ async fn test_on_datanode_create_logical_regions() {
.unwrap()
.0;
let _ = kv_backend.txn(physical_route_txn).await.unwrap();
let mut procedure =
CreateLogicalTablesProcedure::new(vec![task1, task2, task3], physical_table_id, ctx);
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task1, task2, task3],
physical_table_id,
EventContext::default(),
ctx,
);
let expected_created_regions = Arc::new(Mutex::new(HashMap::from([(1, 3), (2, 3), (3, 3)])));
src/meta-srv/src/procedure/wal_prune.rs
+14 -11
View File
@@ -25,8 +25,8 @@ use common_meta::lock_key::RemoteWalLock;
use common_meta::region_registry::LeaderRegionRegistryRef;
use common_procedure::error::ToJsonSnafu;
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, ProcedureState, Result as ProcedureResult, Status, StringKey,
Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger,
LockKey, Procedure, ProcedureState, Result as ProcedureResult, Status, StringKey,
};
use common_telemetry::{info, warn};
use manager::{WalPruneProcedureGuard, WalPruneProcedureTracker};
@@ -213,7 +213,10 @@ impl Procedure for WalPruneProcedure {
LockKey::new(vec![lock_key])
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(WAL_PRUNE_EVENT_TYPE) {
return None;
}
@@ -503,7 +506,7 @@ mod tests {
let mut procedure =
WalPruneProcedure::new(context, None, "test_topic".to_string(), 42, false);
let running = ProcedureState::Running;
let event_context = |trigger, lifecycle_state, event_type_filter| EventContext {
let runtime_context = |trigger, lifecycle_state, event_type_filter| EventRuntimeContext {
procedure_id: ProcedureId::random(),
lifecycle_state,
trigger,
@@ -513,13 +516,13 @@ mod tests {
for trigger in [EventTrigger::Submitted, EventTrigger::Recovered] {
assert!(
procedure
.event(&event_context(trigger, &running, EventTypeFilter::All))
.event(&runtime_context(trigger, &running, EventTypeFilter::All))
.is_none()
);
}
let retrying_event = procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Retrying {
phase: common_procedure::RetryPhase::Execute,
attempt: 1,
@@ -537,7 +540,7 @@ mod tests {
procedure.observed_latest_offset = Some(100);
for trigger in [EventTrigger::Failed, EventTrigger::Poisoned] {
let event = procedure
.event(&event_context(trigger, &running, EventTypeFilter::All))
.event(&runtime_context(trigger, &running, EventTypeFilter::All))
.unwrap();
assert_eq!(
event.extra_rows().unwrap()[0].values[2].value_data,
@@ -548,7 +551,7 @@ mod tests {
let done_without_output = ProcedureState::Done { output: None };
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Succeeded,
&done_without_output,
EventTypeFilter::All,
@@ -560,7 +563,7 @@ mod tests {
};
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Succeeded,
&done_with_wrong_output,
EventTypeFilter::All,
@@ -572,7 +575,7 @@ mod tests {
output: Some(Arc::new(WalPruneOutcome) as Output),
};
let event = procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Succeeded,
&done,
EventTypeFilter::All,
@@ -589,7 +592,7 @@ mod tests {
assert!(
procedure
.event(&event_context(
.event(&runtime_context(
EventTrigger::Succeeded,
&done,
EventTypeFilter::Only(HashSet::new()),
src/meta-srv/src/service/procedure.rs
-1
View File
@@ -102,7 +102,6 @@ impl procedure_service_server::ProcedureService for Metasrv {
.try_into()
.context(error::ConvertProtoDataSnafu)?;
restore_create_database_creator(&metadata, header.role, &mut task, &mut query_context)?;
let resp = self
.ddl_manager()
.submit_ddl_task(
src/operator/src/insert.rs
+17 -6
View File
@@ -36,6 +36,7 @@ use common_grpc_expr::util::ColumnExpr;
use common_meta::cache::TableFlownodeSetCacheRef;
use common_meta::node_manager::{AffectedRows, NodeManagerRef};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::TriggerReason;
use common_query::Output;
use common_query::native_histogram::{is_native_histogram_value_type, native_histogram_value_type};
use common_query::prelude::{greptime_timestamp, greptime_value};
@@ -656,7 +657,7 @@ impl Inserter {
if !alter_tables.is_empty() {
// Alter logical tables in batch.
statement_executor
.alter_logical_tables(alter_tables, ctx.clone())
.alter_logical_tables(alter_tables, ctx.clone(), TriggerReason::AutoAlter)
.await?;
}
}
@@ -677,7 +678,7 @@ impl Inserter {
}
for alter_expr in alter_tables.into_iter() {
statement_executor
.alter_table_inner(alter_expr, ctx.clone())
.alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
.await?;
}
}
@@ -786,7 +787,7 @@ impl Inserter {
}
for alter_expr in alter_tables.into_iter() {
statement_executor
.alter_table_inner(alter_expr, ctx.clone())
.alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
.await?;
}
}
@@ -872,7 +873,12 @@ impl Inserter {
// create physical table
let res = statement_executor
.create_table_inner(create_table_expr, None, ctx.clone())
.create_table_inner(
create_table_expr,
None,
ctx.clone(),
TriggerReason::AutoCreate,
)
.await;
match res {
@@ -1060,7 +1066,12 @@ impl Inserter {
info!("Table `{table_ref}` does not exist, try creating table");
}
let res = statement_executor
.create_table_inner(&mut create_table_expr, partitions, ctx.clone())
.create_table_inner(
&mut create_table_expr,
partitions,
ctx.clone(),
TriggerReason::AutoCreate,
)
.await;
let table_ref = TableReference::full(
@@ -1091,7 +1102,7 @@ impl Inserter {
statement_executor: &StatementExecutor,
) -> Result<Vec<TableRef>> {
let res = statement_executor
.create_logical_tables(&create_table_exprs, ctx.clone())
.create_logical_tables(&create_table_exprs, ctx.clone(), TriggerReason::AutoCreate)
.await;
match res {
src/operator/src/procedure.rs
+15 -4
View File
@@ -19,7 +19,7 @@ use common_error::ext::BoxedError;
use common_function::handlers::ProcedureServiceHandler;
use common_meta::key::TableMetadataManagerRef;
use common_meta::procedure_executor::{ExecutorContext, ProcedureExecutorRef};
use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest};
use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, TriggerReason};
use common_meta::rpc::procedure::{
GcRegionsRequest as MetaGcRegionsRequest, GcResponse as MetaGcResponse,
GcTableRequest as MetaGcTableRequest, ManageRegionFollowerRequest, MigrateRegionRequest,
@@ -31,7 +31,7 @@ use snafu::ResultExt;
use table::table_name::TableName;
use crate::error;
use crate::utils::to_meta_query_context;
use crate::utils::to_meta_query_context_with_trigger_reason;
/// The operator for procedures which implements [`ProcedureServiceHandler`].
#[derive(Clone)]
@@ -76,8 +76,10 @@ impl ProcedureServiceHandler for ProcedureServiceOperator {
})
.map_err(BoxedError::new)
.context(query_error::ProcedureServiceSnafu)?;
let meta_query_context =
to_meta_query_context_with_trigger_reason(query_ctx, TriggerReason::Manual);
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_ctx),
meta_query_context,
DdlTask::new_purge_dropped_table(dropped.table_id),
);
self.procedure_executor
@@ -161,7 +163,9 @@ mod tests {
use common_meta::key::test_utils::new_test_table_info_with_name;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::procedure_executor::{ProcedureExecutor, ProcedureExecutorRef};
use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse};
use common_meta::rpc::ddl::{
DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse, TRIGGER_REASON_EXTENSION_KEY,
};
use common_meta::rpc::procedure::{MigrateRegionResponse, ProcedureStateResponse};
use session::context::QueryContextBuilder;
use table::table_name::TableName;
@@ -277,6 +281,13 @@ mod tests {
);
assert_eq!(requests[0].query_context.current_catalog, "catalog");
assert_eq!(requests[0].query_context.current_schema, "schema");
assert_eq!(
Some(&TriggerReason::Manual.as_ref().to_string()),
requests[0]
.query_context
.extensions
.get(TRIGGER_REASON_EXTENSION_KEY)
);
}
#[tokio::test]
src/operator/src/statement/comment.rs
+12 -10
View File
@@ -16,7 +16,9 @@ use api::v1::CommentOnExpr;
use common_error::ext::BoxedError;
use common_meta::cache_invalidator::Context;
use common_meta::procedure_executor::ExecutorContext;
use common_meta::rpc::ddl::{CommentObjectType, CommentOnTask, DdlTask, SubmitDdlTaskRequest};
use common_meta::rpc::ddl::{
CommentObjectType, CommentOnTask, DdlTask, SubmitDdlTaskRequest, TriggerReason,
};
use common_query::Output;
use session::context::QueryContextRef;
use session::table_name::table_idents_to_full_name;
@@ -28,7 +30,7 @@ use crate::error::{
self, ExecuteDdlSnafu, ExternalSnafu, InvalidSqlSnafu, Result, TableMetadataManagerSnafu,
};
use crate::statement::StatementExecutor;
use crate::utils::to_meta_query_context;
use crate::utils::to_meta_query_context_with_trigger_reason;
impl StatementExecutor {
/// Adds a comment to a database object (table, column, or flow).
@@ -52,10 +54,10 @@ impl StatementExecutor {
.context(TableMetadataManagerSnafu)?;
let cache_idents = comment_on_task.cache_idents();
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_ctx),
DdlTask::new_comment_on(comment_on_task),
);
let meta_query_context =
to_meta_query_context_with_trigger_reason(query_ctx, TriggerReason::Manual);
let request =
SubmitDdlTaskRequest::new(meta_query_context, DdlTask::new_comment_on(comment_on_task));
self.procedure_executor
.submit_ddl_task(&ExecutorContext::default(), request)
@@ -86,10 +88,10 @@ impl StatementExecutor {
.context(TableMetadataManagerSnafu)?;
let cache_idents = comment_on_task.cache_idents();
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_ctx),
DdlTask::new_comment_on(comment_on_task),
);
let meta_query_context =
to_meta_query_context_with_trigger_reason(query_ctx, TriggerReason::Manual);
let request =
SubmitDdlTaskRequest::new(meta_query_context, DdlTask::new_comment_on(comment_on_task));
self.procedure_executor
.submit_ddl_task(&ExecutorContext::default(), request)
src/operator/src/statement/ddl.rs
+83 -38
View File
@@ -53,7 +53,7 @@ use common_meta::rpc::ddl::trigger::CreateTriggerTask;
use common_meta::rpc::ddl::trigger::DropTriggerTask;
use common_meta::rpc::ddl::{
CreateFlowTask, CreatorGrantIntent, DdlTask, DropFlowTask, DropViewTask, SubmitDdlTaskRequest,
SubmitDdlTaskResponse,
SubmitDdlTaskResponse, TriggerReason,
};
use common_query::Output;
use common_recordbatch::{RecordBatch, RecordBatches};
@@ -116,7 +116,10 @@ use crate::error::{
use crate::expr_helper::{self, RepartitionRequest, RepartitionSource};
use crate::statement::StatementExecutor;
use crate::statement::show::create_partitions_stmt;
use crate::utils::{to_meta_query_context, to_meta_query_context_with_origin_frontend};
use crate::utils::{
to_meta_query_context_with_origin_frontend_and_trigger_reason,
to_meta_query_context_with_trigger_reason,
};
#[derive(Debug, Clone, Copy)]
struct DdlSubmitOptions {
@@ -300,7 +303,7 @@ impl StatementExecutor {
}
}
self.create_table_inner(create_expr, stmt.partitions, ctx)
self.create_table_inner(create_expr, stmt.partitions, ctx, TriggerReason::Manual)
.await
}
@@ -357,7 +360,8 @@ impl StatementExecutor {
);
let create_expr = &mut expr_helper::create_to_expr(&create_stmt, &ctx)?;
self.create_table_inner(create_expr, partitions, ctx).await
self.create_table_inner(create_expr, partitions, ctx, TriggerReason::Manual)
.await
}
#[tracing::instrument(skip_all)]
@@ -369,7 +373,8 @@ impl StatementExecutor {
let create_expr =
&mut expr_helper::create_external_expr(create_expr, &ctx, &self.local_file_access)
.await?;
self.create_table_inner(create_expr, None, ctx).await
self.create_table_inner(create_expr, None, ctx, TriggerReason::Manual)
.await
}
#[tracing::instrument(skip_all)]
@@ -378,6 +383,7 @@ impl StatementExecutor {
create_table: &mut CreateTableExpr,
partitions: Option<Partitions>,
query_ctx: QueryContextRef,
trigger_reason: TriggerReason,
) -> Result<TableRef> {
ensure!(
!is_readonly_schema(&create_table.schema_name),
@@ -398,16 +404,20 @@ impl StatementExecutor {
.await?;
}
// Create logical tables
self.create_logical_tables(std::slice::from_ref(create_table), query_ctx)
.await?
.into_iter()
.next()
.context(error::UnexpectedSnafu {
violated: "expected to create logical tables",
})
self.create_logical_tables(
std::slice::from_ref(create_table),
query_ctx,
trigger_reason,
)
.await?
.into_iter()
.next()
.context(error::UnexpectedSnafu {
violated: "expected to create logical tables",
})
} else {
// Create other normal table
self.create_non_logic_table(create_table, partitions, query_ctx)
self.create_non_logic_table(create_table, partitions, query_ctx, trigger_reason)
.await
}
}
@@ -418,6 +428,7 @@ impl StatementExecutor {
create_table: &mut CreateTableExpr,
partitions: Option<Partitions>,
query_ctx: QueryContextRef,
trigger_reason: TriggerReason,
) -> Result<TableRef> {
let _timer = crate::metrics::DIST_CREATE_TABLE.start_timer();
@@ -486,6 +497,7 @@ impl StatementExecutor {
partitions,
table_info.clone(),
query_ctx,
trigger_reason,
)
.await?;
@@ -513,6 +525,7 @@ impl StatementExecutor {
&self,
create_table_exprs: &[CreateTableExpr],
query_context: QueryContextRef,
trigger_reason: TriggerReason,
) -> Result<Vec<TableRef>> {
let _timer = crate::metrics::DIST_CREATE_TABLES.start_timer();
ensure!(
@@ -543,7 +556,7 @@ impl StatementExecutor {
.collect::<Vec<_>>();
let resp = self
.create_logical_tables_procedure(tables_data, query_context.clone())
.create_logical_tables_procedure(tables_data, query_context.clone(), trigger_reason)
.await?;
let table_ids = resp.table_ids;
@@ -706,7 +719,7 @@ impl StatementExecutor {
.context(error::InvalidExprSnafu)?;
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_create_trigger(task),
);
@@ -770,7 +783,7 @@ impl StatementExecutor {
})
.context(error::InvalidExprSnafu)?;
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_create_flow(task),
);
@@ -1083,7 +1096,7 @@ impl StatementExecutor {
};
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(ctx),
to_meta_query_context_with_trigger_reason(ctx, TriggerReason::Manual),
DdlTask::new_create_view(expr, view_info.clone()),
);
@@ -1169,7 +1182,7 @@ impl StatementExecutor {
query_context: QueryContextRef,
) -> Result<SubmitDdlTaskResponse> {
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_drop_flow(expr),
);
@@ -1204,7 +1217,7 @@ impl StatementExecutor {
query_context: QueryContextRef,
) -> Result<SubmitDdlTaskResponse> {
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_drop_trigger(expr),
);
@@ -1272,7 +1285,7 @@ impl StatementExecutor {
query_context: QueryContextRef,
) -> Result<SubmitDdlTaskResponse> {
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_drop_view(expr),
);
@@ -1287,6 +1300,7 @@ impl StatementExecutor {
&self,
alter_table_exprs: Vec<AlterTableExpr>,
query_context: QueryContextRef,
trigger_reason: TriggerReason,
) -> Result<Output> {
let _timer = crate::metrics::DIST_ALTER_TABLES.start_timer();
ensure!(
@@ -1332,7 +1346,8 @@ impl StatementExecutor {
// Submit procedure for each physical table
let mut handles = Vec::with_capacity(groups.len());
for (_physical_table_id, exprs) in groups {
let fut = self.alter_logical_tables_procedure(exprs, query_context.clone());
let fut =
self.alter_logical_tables_procedure(exprs, query_context.clone(), trigger_reason);
handles.push(fut);
}
let _results = futures::future::try_join_all(handles).await?;
@@ -1514,7 +1529,8 @@ impl StatementExecutor {
}
let expr = expr_helper::to_alter_table_expr(alter_table, &query_context)?;
self.alter_table_inner(expr, query_context).await
self.alter_table_inner(expr, query_context, TriggerReason::Manual)
.await
}
#[tracing::instrument(skip_all)]
@@ -1752,8 +1768,10 @@ impl StatementExecutor {
source: Some(source),
..Default::default()
};
let meta_query_context =
to_meta_query_context_with_trigger_reason(query_context.clone(), TriggerReason::Manual);
let mut req = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context.clone()),
meta_query_context,
DdlTask::new_alter_table(AlterTableExpr {
catalog_name: request.catalog_name.clone(),
schema_name: request.schema_name.clone(),
@@ -1808,6 +1826,7 @@ impl StatementExecutor {
&self,
expr: AlterTableExpr,
query_context: QueryContextRef,
trigger_reason: TriggerReason,
) -> Result<Output> {
ensure!(
!is_readonly_schema(&expr.schema_name),
@@ -1862,12 +1881,12 @@ impl StatementExecutor {
.await
.context(TableMetadataManagerSnafu)?;
let meta_query_context =
to_meta_query_context_with_trigger_reason(query_context, trigger_reason);
let (req, invalidate_keys) = if physical_table_id == table_id {
// This is physical table
let req = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
DdlTask::new_alter_table(expr),
);
let req = SubmitDdlTaskRequest::new(meta_query_context, DdlTask::new_alter_table(expr));
let invalidate_keys = vec![
CacheIdent::TableId(table_id),
@@ -1878,7 +1897,7 @@ impl StatementExecutor {
} else {
// This is logical table
let req = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
meta_query_context,
DdlTask::new_alter_logical_tables(vec![expr]),
);
@@ -1992,6 +2011,7 @@ impl StatementExecutor {
partitions: Vec<PartitionExpr>,
table_info: TableInfo,
query_context: QueryContextRef,
trigger_reason: TriggerReason,
) -> Result<SubmitDdlTaskResponse> {
let partitions = partitions
.into_iter()
@@ -1999,7 +2019,11 @@ impl StatementExecutor {
.collect::<Result<Vec<_>>>()?;
let request = SubmitDdlTaskRequest::new(
to_meta_query_context_with_origin_frontend(query_context, &self.origin_frontend_addr),
to_meta_query_context_with_origin_frontend_and_trigger_reason(
query_context,
&self.origin_frontend_addr,
trigger_reason,
),
DdlTask::new_create_table(create_table, partitions, table_info),
);
@@ -2013,9 +2037,14 @@ impl StatementExecutor {
&self,
tables_data: Vec<(CreateTableExpr, TableInfo)>,
query_context: QueryContextRef,
trigger_reason: TriggerReason,
) -> Result<SubmitDdlTaskResponse> {
let request = SubmitDdlTaskRequest::new(
to_meta_query_context_with_origin_frontend(query_context, &self.origin_frontend_addr),
to_meta_query_context_with_origin_frontend_and_trigger_reason(
query_context,
&self.origin_frontend_addr,
trigger_reason,
),
DdlTask::new_create_logical_tables(tables_data),
);
@@ -2029,9 +2058,12 @@ impl StatementExecutor {
&self,
tables_data: Vec<AlterTableExpr>,
query_context: QueryContextRef,
trigger_reason: TriggerReason,
) -> Result<SubmitDdlTaskResponse> {
let meta_query_context =
to_meta_query_context_with_trigger_reason(query_context, trigger_reason);
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
meta_query_context,
DdlTask::new_alter_logical_tables(tables_data),
);
@@ -2049,7 +2081,7 @@ impl StatementExecutor {
query_context: QueryContextRef,
) -> Result<SubmitDdlTaskResponse> {
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_drop_table(
table_name.catalog_name.clone(),
table_name.schema_name.clone(),
@@ -2073,7 +2105,7 @@ impl StatementExecutor {
query_context: QueryContextRef,
) -> Result<SubmitDdlTaskResponse> {
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_drop_database(catalog, schema, drop_if_exists),
);
@@ -2089,7 +2121,7 @@ impl StatementExecutor {
query_context: QueryContextRef,
) -> Result<SubmitDdlTaskResponse> {
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_alter_database(alter_expr),
);
@@ -2106,8 +2138,10 @@ impl StatementExecutor {
time_ranges: Vec<(Timestamp, Timestamp)>,
query_context: QueryContextRef,
) -> Result<SubmitDdlTaskResponse> {
let meta_query_context =
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual);
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
meta_query_context,
DdlTask::new_truncate_table(
table_name.catalog_name.clone(),
table_name.schema_name.clone(),
@@ -2199,7 +2233,7 @@ impl StatementExecutor {
creator: Option<CreatorGrantIntent>,
) -> Result<SubmitDdlTaskResponse> {
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual),
DdlTask::new_create_database(catalog, database, create_if_not_exists, options, creator),
);
@@ -2734,8 +2768,10 @@ async fn execute_undrop_table(
table_name: table_name.to_string(),
})?;
let meta_query_context =
to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual);
let request = SubmitDdlTaskRequest::new(
to_meta_query_context(query_context),
meta_query_context,
DdlTask::new_undrop_table(dropped.table_id),
);
procedure_executor
@@ -2787,7 +2823,9 @@ mod test {
ExecutorContext, ProcedureExecutor, ProcedureExecutorRef,
};
#[cfg(feature = "enterprise")]
use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse};
use common_meta::rpc::ddl::{
DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse, TRIGGER_REASON_EXTENSION_KEY,
};
#[cfg(feature = "enterprise")]
use common_meta::rpc::procedure::{
MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse,
@@ -2932,6 +2970,13 @@ mod test {
let requests = procedure.requests.lock().unwrap();
assert!(matches!(&requests[0].task, DdlTask::UndropTable(task) if task.table_id == 42));
assert_eq!(
requests[0]
.query_context
.extensions
.get(TRIGGER_REASON_EXTENSION_KEY),
Some(&TriggerReason::Manual.as_ref().to_string())
);
assert_eq!(
cache.invalidations.lock().unwrap()[0],
vec![CacheIdent::TableId(42), CacheIdent::TableName(name)]
src/operator/src/utils.rs
+37 -6
View File
@@ -14,6 +14,9 @@
use std::sync::{Arc, RwLock};
use common_meta::rpc::ddl::{
ORIGIN_FRONTEND_ADDR_EXTENSION_KEY, TRIGGER_REASON_EXTENSION_KEY, TriggerReason,
};
use common_time::Timezone;
use session::context::{QueryContextBuilder, QueryContextRef};
use snafu::ResultExt;
@@ -34,13 +37,28 @@ pub fn to_meta_query_context(
}
}
pub fn to_meta_query_context_with_origin_frontend(
/// Converts a session query context and records the DDL trigger reason.
pub fn to_meta_query_context_with_trigger_reason(
query_context: QueryContextRef,
origin_frontend_addr: &str,
trigger_reason: TriggerReason,
) -> common_meta::rpc::ddl::QueryContext {
let mut meta_query_context = to_meta_query_context(query_context);
meta_query_context.extensions.insert(
common_meta::rpc::ddl::ORIGIN_FRONTEND_ADDR_EXTENSION_KEY.to_string(),
TRIGGER_REASON_EXTENSION_KEY.to_string(),
trigger_reason.as_ref().to_string(),
);
meta_query_context
}
pub fn to_meta_query_context_with_origin_frontend_and_trigger_reason(
query_context: QueryContextRef,
origin_frontend_addr: &str,
trigger_reason: TriggerReason,
) -> common_meta::rpc::ddl::QueryContext {
let mut meta_query_context =
to_meta_query_context_with_trigger_reason(query_context, trigger_reason);
meta_query_context.extensions.insert(
ORIGIN_FRONTEND_ADDR_EXTENSION_KEY.to_string(),
origin_frontend_addr.to_string(),
);
meta_query_context
@@ -69,12 +87,14 @@ mod tests {
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use common_meta::rpc::ddl::ORIGIN_FRONTEND_ADDR_EXTENSION_KEY;
use common_meta::rpc::ddl::{
ORIGIN_FRONTEND_ADDR_EXTENSION_KEY, TRIGGER_REASON_EXTENSION_KEY, TriggerReason,
};
use common_time::Timezone;
use session::context::QueryContextBuilder;
use super::{
to_meta_query_context, to_meta_query_context_with_origin_frontend,
to_meta_query_context, to_meta_query_context_with_origin_frontend_and_trigger_reason,
try_to_session_query_context,
};
@@ -112,7 +132,11 @@ mod tests {
.build(),
);
let meta_ctx = to_meta_query_context_with_origin_frontend(session_ctx, "127.0.0.1:4000");
let meta_ctx = to_meta_query_context_with_origin_frontend_and_trigger_reason(
session_ctx,
"127.0.0.1:4000",
TriggerReason::Manual,
);
assert_eq!(
meta_ctx
@@ -121,5 +145,12 @@ mod tests {
.map(String::as_str),
Some("127.0.0.1:4000")
);
assert_eq!(
meta_ctx
.extensions
.get(TRIGGER_REASON_EXTENSION_KEY)
.map(String::as_str),
Some("manual")
);
}
}
src/pipeline/src/manager/pipeline_operator.rs
+2 -1
View File
@@ -19,6 +19,7 @@ use std::time::Instant;
use api::v1::CreateTableExpr;
use catalog::{CatalogManagerRef, RegisterSystemTableRequest};
use common_catalog::consts::{DEFAULT_PRIVATE_SCHEMA_NAME, default_engine};
use common_meta::rpc::ddl::TriggerReason;
use common_telemetry::info;
use common_time::FOREVER;
use datatypes::timestamp::TimestampNanosecond;
@@ -131,7 +132,7 @@ impl PipelineOperator {
// create table
self.statement_executor
.create_table_inner(&mut expr, None, ctx.clone())
.create_table_inner(&mut expr, None, ctx.clone(), TriggerReason::AutoCreate)
.await
.context(CreateTableSnafu)?;
src/session/src/context.rs
+2 -17
View File
@@ -618,6 +618,7 @@ impl ConnInfo {
}
}
/// The protocol mapping for these wire values lives in [`common_session::channel_protocol`].
#[derive(Debug, PartialEq, Default, Clone, Copy)]
#[repr(u8)]
pub enum Channel {
@@ -680,23 +681,7 @@ impl Display for Channel {
impl AsRef<str> for Channel {
fn as_ref(&self) -> &str {
match self {
Channel::Mysql => "mysql",
Channel::Postgres => "postgres",
Channel::HttpSql => "httpsql",
Channel::Prometheus => "prometheus",
Channel::Otlp => "otlp",
Channel::Grpc => "grpc",
Channel::Influx => "influx",
Channel::Opentsdb => "opentsdb",
Channel::Loki => "loki",
Channel::Elasticsearch => "elasticsearch",
Channel::Jaeger => "jaeger",
Channel::Log => "log",
Channel::Promql => "promql",
Channel::Splunk => "splunk",
Channel::Unknown => "unknown",
}
common_session::channel_protocol(*self as u8).unwrap_or("unknown")
}
}
src/standalone/src/procedure.rs
+2
View File
@@ -20,6 +20,7 @@ use common_meta::ddl::DdlContext;
use common_meta::ddl_manager::{RepartitionProcedureFactory, RepartitionSource};
use common_meta::key::runtime_switch::RuntimeSwitchManager;
use common_meta::kv_backend::KvBackendRef;
use common_meta::rpc::ddl::EventContext;
use common_meta::state_store::KvStateStore;
use common_procedure::local::{EventRecorderHandle, LocalManager, ManagerConfig};
use common_procedure::options::ProcedureConfig;
@@ -73,6 +74,7 @@ impl RepartitionProcedureFactory for StandaloneRepartitionProcedureFactory {
_source: RepartitionSource,
_to_exprs: Vec<String>,
_timeout: Option<Duration>,
_event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError> {
Err(BoxedError::new(NoSupportRepartitionProcedureSnafu.build()))
}
tests-integration/src/tests/gc.rs
+2
View File
@@ -17,6 +17,7 @@ use std::sync::Arc;
use std::time::Duration;
use common_meta::key::TableMetadataManagerRef;
use common_meta::rpc::ddl::EventContext;
use common_procedure::ProcedureWithId;
use common_telemetry::info;
use common_test_util::recordbatch::check_output_stream;
@@ -213,6 +214,7 @@ async fn test_gc_basic(store_type: &StorageType) {
false, // full_file_listing
Duration::from_secs(10), // timeout
Default::default(),
EventContext::default(),
);
// Submit the procedure to the procedure manager
tests-integration/tests/database_ddl_event.rs
+10 -1
View File
@@ -22,7 +22,7 @@ use tests_integration::cluster::GreptimeDbClusterBuilder;
use tests_integration::standalone::GreptimeDbStandaloneBuilder;
use tests_integration::test_util::{StorageType, get_test_store_config};
use crate::event_recorder_test_util::assert_single_event;
use crate::event_recorder_test_util::{assert_single_event, find_eventually_string};
const DATABASE_NAME: &str = "database_ddl_events";
@@ -122,6 +122,15 @@ WHERE type = '{event_type}'
AND {submitted_payload_predicate}"#
);
assert_single_event(instance, &submitted).await;
let actual = find_eventually_string(
instance,
&format!(
"SELECT json_to_string(event_context) AS event_context FROM greptime_private.events WHERE type = '{event_type}' AND procedure_state = 'Running' AND json_path_match(procedure_trigger, '$.type == \"Submitted\"') ORDER BY timestamp DESC LIMIT 1"
),
"event_context",
)
.await;
assert_eq!(r#"{"reason":"manual"}"#, actual);
let lifecycle = format!(
r#"SELECT count(*) AS event_count
tests-integration/tests/flow_ddl_event.rs
+18
View File
@@ -88,6 +88,7 @@ WHERE type = '{CREATE_FLOW_EVENT_TYPE}'
),
)
.await;
assert_event_context(instance, CREATE_FLOW_EVENT_TYPE, &procedure_id).await;
assert_single_event(
instance,
&format!(
@@ -106,6 +107,22 @@ WHERE type = '{CREATE_FLOW_EVENT_TYPE}'
.await;
}
async fn assert_event_context(
instance: &Arc<frontend::instance::Instance>,
event_type: &str,
procedure_id: &str,
) {
let actual = find_eventually_string(
instance,
&format!(
"SELECT json_to_string(event_context) AS event_context FROM greptime_private.events WHERE type = '{event_type}' AND procedure_id = '{procedure_id}' AND json_path_match(procedure_trigger, '$.type == \"Submitted\"')"
),
"event_context",
)
.await;
assert_eq!(r#"{"reason":"manual"}"#, actual);
}
async fn assert_drop_events(instance: &Arc<frontend::instance::Instance>, flow: &str) {
let procedure_id = find_submitted_procedure_id(instance, DROP_FLOW_EVENT_TYPE, flow).await;
assert_single_event(
@@ -125,6 +142,7 @@ WHERE type = '{DROP_FLOW_EVENT_TYPE}'
),
)
.await;
assert_event_context(instance, DROP_FLOW_EVENT_TYPE, &procedure_id).await;
assert_single_event(
instance,
&format!(
tests-integration/tests/gc_event.rs
+17 -1
View File
@@ -15,6 +15,7 @@
use std::sync::Arc;
use std::time::Duration;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_procedure::{ProcedureId, ProcedureWithId, watcher};
use common_test_util::temp_dir::create_temp_dir;
use meta_srv::gc::{BatchGcProcedure, GcSchedulerOptions};
@@ -128,6 +129,7 @@ async fn test_batch_gc_event() {
false,
Duration::from_secs(10),
Default::default(),
EventContext::new(TriggerReason::Manual),
);
let procedure_id = ProcedureId::parse_str("00000000-0000-0000-0000-00000000bac0").unwrap();
let mut watcher = cluster
@@ -142,6 +144,19 @@ async fn test_batch_gc_event() {
watcher::wait(&mut watcher).await.unwrap();
assert_sst_count(&cluster, 1).await;
let submitted = format!(
r#"SELECT json_to_string(event_context) AS event_context
FROM greptime_private.events
WHERE type = 'batch_gc'
AND procedure_id = '{procedure_id}'
AND procedure_state = 'Running'
AND json_get_string(procedure_trigger, 'type') = 'Submitted'"#,
);
assert_eq!(
r#"{"reason":"manual"}"#,
find_eventually_string(instance, &submitted, "event_context").await
);
let succeeded = format!(
r#"SELECT json_to_string(gc_report) AS gc_report
FROM greptime_private.events
@@ -149,7 +164,8 @@ WHERE type = 'batch_gc'
AND procedure_id = '{procedure_id}'
AND procedure_state = 'Done'
AND json_path_match(procedure_trigger, '$.type == "Succeeded"')
AND json_is_null(payload)"#,
AND json_is_null(payload)
AND event_context IS NULL"#,
);
let actual_report: serde_json::Value =
serde_json::from_str(&find_eventually_string(instance, &succeeded, "gc_report").await)
tests-integration/tests/http.rs
+41 -1
View File
@@ -2566,6 +2566,15 @@ pub async fn test_prometheus_remote_write(store_type: StorageType) {
.await;
assert_eq!(res.status(), StatusCode::NO_CONTENT);
// The direct Prom remote-write handler creates tables without going through
// the generic inserter. Its submitted events must retain this origin.
wait_for_event_data(
&client,
"SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type IN ('create_table', 'create_logical_tables') AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'prometheus' AND json_get_string(event_context, 'reason') = 'auto_create'",
r#"[["{\"protocol\":\"prometheus\",\"reason\":\"auto_create\"}"]]"#,
)
.await;
let expected = "[[\"demo\"],[\"demo_metrics\"],[\"demo_metrics_with_nanos\"],[\"greptime_physical_table\"],[\"metric1\"],[\"metric2\"],[\"metric3\"],[\"mito\"],[\"multi_labels\"],[\"numbers\"],[\"phy\"],[\"phy2\"],[\"phy_ns\"]]";
validate_data("prometheus_remote_write", &client, "show tables;", expected).await;
@@ -2608,6 +2617,15 @@ pub async fn test_prometheus_remote_write(store_type: StorageType) {
assert_eq!(res.status(), StatusCode::NO_CONTENT);
// Adding a label is handled by PendingRowsSchemaAlterer, which is another
// direct DDL path outside the generic inserter.
wait_for_event_data(
&client,
"SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type IN ('alter_table', 'alter_logical_tables') AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'prometheus' AND json_get_string(event_context, 'reason') = 'auto_alter'",
r#"[["{\"protocol\":\"prometheus\",\"reason\":\"auto_alter\"}"]]"#,
)
.await;
guard.remove_all().await;
}
@@ -7122,6 +7140,15 @@ pub async fn test_otlp_traces_v1(store_type: StorageType) {
let res = send_trace_v1_req(&client, existing_int_table_name, existing_int_req, false).await;
assert_eq!(StatusCode::OK, res.status());
// Trace schema reconciliation widens the existing column through a direct
// alter-table call, so it must identify itself as an OTLP auto alteration.
wait_for_event_data(
&client,
"SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type = 'alter_table' AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'otlp' AND json_get_string(event_context, 'reason') = 'auto_alter'",
r#"[["{\"protocol\":\"otlp\",\"reason\":\"auto_alter\"}"]]"#,
)
.await;
validate_data(
"otlp_traces_v1_existing_int_widens_rows",
&client,
@@ -9872,7 +9899,20 @@ async fn validate_data(test_name: &str, client: &TestClient, sql: &str, expected
}
async fn wait_for_data(client: &TestClient, sql: &str, expected: &str) {
tokio::time::timeout(Duration::from_secs(10), async {
wait_for_data_with_timeout(client, sql, expected, Duration::from_secs(10)).await;
}
async fn wait_for_event_data(client: &TestClient, sql: &str, expected: &str) {
wait_for_data_with_timeout(client, sql, expected, Duration::from_secs(30)).await;
}
async fn wait_for_data_with_timeout(
client: &TestClient,
sql: &str,
expected: &str,
timeout: Duration,
) {
tokio::time::timeout(timeout, async {
let encoded_sql = encode(sql);
loop {
let res = client
tests-integration/tests/region_migration.rs
+10 -6
View File
@@ -1347,6 +1347,10 @@ async fn check_region_migration_events_system_table(
Expr::cust("json_get_string(procedure_trigger, 'type')"),
Alias::new("procedure_trigger"),
)
.expr_as(
Expr::cust("json_to_string(event_context)"),
Alias::new("event_context"),
)
.from((RegionMigrationEvents::Schema, RegionMigrationEvents::Table))
.and_where(Expr::col(RegionMigrationEvents::EventType).eq(REGION_MIGRATION_EVENT_TYPE))
.and_where(Expr::col(RegionMigrationEvents::ProcedureId).eq(procedure_id))
@@ -1362,11 +1366,11 @@ async fn check_region_migration_events_system_table(
.remove(0);
let expected = "\
+---------------------------------+-----------------+-------------------+
| region_migration_trigger_reason | procedure_state | procedure_trigger |
+---------------------------------+-----------------+-------------------+
| Manual | Running | Submitted |
| Manual | Done | Succeeded |
+---------------------------------+-----------------+-------------------+";
+---------------------------------+-----------------+-------------------+---------------------+
| region_migration_trigger_reason | procedure_state | procedure_trigger | event_context |
+---------------------------------+-----------------+-------------------+---------------------+
| Manual | Running | Submitted | {\"reason\":\"manual\"} |
| Manual | Done | Succeeded | |
+---------------------------------+-----------------+-------------------+---------------------+";
check_output_stream(result.unwrap().data, expected).await;
}
tests-integration/tests/repartition.rs
+2
View File
@@ -19,6 +19,7 @@ use std::time::Duration;
use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_error::root_source;
use common_meta::key::table_name::TableNameKey;
use common_meta::rpc::ddl::EventContext;
use common_procedure::{ProcedureWithId, watcher};
use common_query::Output;
use common_telemetry::info;
@@ -816,6 +817,7 @@ async fn trigger_table_gc(metasrv: &Arc<Metasrv>, table_name: &str) {
false, // full_file_listing
Duration::from_secs(10), // timeout
Default::default(),
EventContext::default(),
);
// Submit the procedure to the procedure manager
tests-integration/tests/table_ddl_event.rs
+331 -13
View File
@@ -16,7 +16,15 @@ use std::sync::Arc;
#[cfg(feature = "enterprise")]
use std::time::Duration;
use api::v1::ddl_request::Expr as DdlExpr;
use api::v1::greptime_request::Request;
use api::v1::value::ValueData;
use api::v1::{
AddColumn, AddColumns, AlterTableExpr, ColumnDataType, ColumnDef, CreateTableExpr, DdlRequest,
Row, RowInsertRequest, RowInsertRequests, Rows, SemanticType, alter_table_expr,
};
use client::OutputData;
use common_catalog::consts::MITO_ENGINE;
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, PAYLOAD_COLUMN, PHYSICAL_TABLE_ID_COLUMN, PROCEDURE_ID_COLUMN,
PROCEDURE_TRIGGER_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN, TYPE_COLUMN,
@@ -25,8 +33,9 @@ use common_test_util::temp_dir::create_temp_dir;
use frontend::instance::Instance;
use meta_srv::gc::GcSchedulerOptions;
use mito2::gc::GcConfig;
use servers::query_handler::grpc::GrpcQueryHandler;
use servers::query_handler::sql::SqlQueryHandler;
use session::context::QueryContext;
use session::context::{Channel, QueryContext};
use tests_integration::cluster::GreptimeDbClusterBuilder;
use crate::event_recorder_test_util::{
@@ -37,6 +46,9 @@ const EVENTS_TABLE: &str = "greptime_private.events";
const TABLE: &str = "table_ddl_events";
const PHYSICAL_TABLE: &str = "table_ddl_events_phy";
const LOGICAL_TABLE: &str = "table_ddl_events_logical";
const AUTO_TABLE: &str = "table_ddl_events_auto";
const AUTO_INFLUX_TABLE: &str = "table_ddl_events_auto_influx";
const GRPC_TABLE: &str = "table_ddl_events_grpc";
#[tokio::test(flavor = "multi_thread")]
async fn test_table_ddl_procedure_events() {
@@ -70,17 +82,142 @@ async fn test_table_ddl_procedure_events() {
.await;
let frontend = cluster.fe_instance().clone();
auto_create_table(&frontend, AUTO_TABLE, Channel::Prometheus).await;
let auto_create_procedure_id =
submitted_procedure_id(&frontend, "create_table", AUTO_TABLE).await;
assert_event_context(
&frontend,
"create_table",
&auto_create_procedure_id,
"auto_create",
"prometheus",
)
.await;
auto_create_table(&frontend, AUTO_INFLUX_TABLE, Channel::Influx).await;
let auto_influx_create_procedure_id =
submitted_procedure_id(&frontend, "create_table", AUTO_INFLUX_TABLE).await;
assert_event_context(
&frontend,
"create_table",
&auto_influx_create_procedure_id,
"auto_create",
"influx",
)
.await;
// gRPC DDL bypasses the SQL statement executor, so it must set its own
// manual trigger reason before submitting the table procedure.
GrpcQueryHandler::do_query(
frontend.as_ref(),
Request::Ddl(DdlRequest {
expr: Some(DdlExpr::CreateTable(CreateTableExpr {
catalog_name: "greptime".to_string(),
schema_name: "public".to_string(),
table_name: GRPC_TABLE.to_string(),
column_defs: vec![
ColumnDef {
name: "host".to_string(),
data_type: ColumnDataType::String as i32,
is_nullable: true,
semantic_type: SemanticType::Tag as i32,
..Default::default()
},
ColumnDef {
name: "ts".to_string(),
data_type: ColumnDataType::TimestampMillisecond as i32,
is_nullable: false,
semantic_type: SemanticType::Timestamp as i32,
..Default::default()
},
],
time_index: "ts".to_string(),
engine: MITO_ENGINE.to_string(),
..Default::default()
})),
}),
Arc::new(QueryContext::with_channel(
"greptime",
"public",
Channel::Grpc,
)),
)
.await
.unwrap();
let grpc_create_procedure_id =
submitted_procedure_id(&frontend, "create_table", GRPC_TABLE).await;
assert_event_context(
&frontend,
"create_table",
&grpc_create_procedure_id,
"manual",
"grpc",
)
.await;
GrpcQueryHandler::do_query(
frontend.as_ref(),
Request::Ddl(DdlRequest {
expr: Some(DdlExpr::AlterTable(AlterTableExpr {
catalog_name: "greptime".to_string(),
schema_name: "public".to_string(),
table_name: GRPC_TABLE.to_string(),
kind: Some(alter_table_expr::Kind::AddColumns(AddColumns {
add_columns: vec![AddColumn {
column_def: Some(ColumnDef {
name: "value".to_string(),
data_type: ColumnDataType::Float64 as i32,
is_nullable: true,
semantic_type: SemanticType::Field as i32,
..Default::default()
}),
..Default::default()
}],
})),
})),
}),
Arc::new(QueryContext::with_channel(
"greptime",
"public",
Channel::Grpc,
)),
)
.await
.unwrap();
let grpc_alter_procedure_id =
submitted_procedure_id(&frontend, "alter_table", GRPC_TABLE).await;
assert_event_context(
&frontend,
"alter_table",
&grpc_alter_procedure_id,
"manual",
"grpc",
)
.await;
// Act / Assert: Create Table retains its rich submitted row and records the
// created table ID when it completes.
run_sql(
run_sql_with_context(
&frontend,
&format!(
"CREATE TABLE {TABLE} (host STRING PRIMARY KEY, ts TIMESTAMP TIME INDEX, val DOUBLE)"
),
Arc::new(QueryContext::with_channel(
"greptime",
"public",
Channel::HttpSql,
)),
)
.await;
let table_id = find_table_id(&frontend, TABLE).await;
let create_table_procedure_id = submitted_procedure_id(&frontend, "create_table", TABLE).await;
assert_event_context(
&frontend,
"create_table",
&create_table_procedure_id,
"manual",
"httpsql",
)
.await;
assert_named_submitted_event(
&frontend,
"create_table",
@@ -110,12 +247,25 @@ async fn test_table_ddl_procedure_events() {
// Act / Assert: Alter and Truncate have a rich submitted row and lightweight
// terminal lifecycle row.
run_sql(
run_sql_with_context(
&frontend,
&format!("ALTER TABLE {TABLE} ADD COLUMN extra STRING"),
Arc::new(QueryContext::with_channel(
"greptime",
"public",
Channel::HttpSql,
)),
)
.await;
let alter_table_procedure_id = submitted_procedure_id(&frontend, "alter_table", TABLE).await;
assert_event_context(
&frontend,
"alter_table",
&alter_table_procedure_id,
"manual",
"httpsql",
)
.await;
assert_named_submitted_event(
&frontend,
"alter_table",
@@ -140,9 +290,26 @@ async fn test_table_ddl_procedure_events() {
&format!("INSERT INTO {TABLE} VALUES ('a', 0, 1, 'b')"),
)
.await;
run_sql(&frontend, &format!("TRUNCATE TABLE {TABLE}")).await;
run_sql_with_context(
&frontend,
&format!("TRUNCATE TABLE {TABLE}"),
Arc::new(QueryContext::with_channel(
"greptime",
"public",
Channel::HttpSql,
)),
)
.await;
let truncate_table_procedure_id =
submitted_procedure_id(&frontend, "truncate_table", TABLE).await;
assert_event_context(
&frontend,
"truncate_table",
&truncate_table_procedure_id,
"manual",
"httpsql",
)
.await;
assert_named_submitted_event(
&frontend,
"truncate_table",
@@ -165,8 +332,25 @@ async fn test_table_ddl_procedure_events() {
// Act / Assert: Drop records the table name, while Undrop and Purge use the
// table ID as their submitted locator.
run_sql(&frontend, &format!("DROP TABLE {TABLE}")).await;
run_sql_with_context(
&frontend,
&format!("DROP TABLE {TABLE}"),
Arc::new(QueryContext::with_channel(
"greptime",
"public",
Channel::HttpSql,
)),
)
.await;
let drop_table_procedure_id = submitted_procedure_id(&frontend, "drop_table", TABLE).await;
assert_event_context(
&frontend,
"drop_table",
&drop_table_procedure_id,
"manual",
"httpsql",
)
.await;
assert_named_submitted_event(
&frontend,
"drop_table",
@@ -191,10 +375,23 @@ async fn test_table_ddl_procedure_events() {
let (undrop_table_procedure_id, _) = cluster
.metasrv
.ddl_manager()
.submit_undrop_table_task(common_meta::rpc::ddl::UndropTableTask { table_id })
.submit_undrop_table_task(
common_meta::rpc::ddl::UndropTableTask { table_id },
common_meta::rpc::ddl::EventContext::new(
common_meta::rpc::ddl::TriggerReason::Manual,
),
)
.await
.unwrap();
let undrop_table_procedure_id = undrop_table_procedure_id.to_string();
assert_event_context(
&frontend,
"undrop_table",
&undrop_table_procedure_id,
"manual",
"unknown",
)
.await;
assert_id_submitted_event(
&frontend,
"undrop_table",
@@ -218,12 +415,23 @@ async fn test_table_ddl_procedure_events() {
let (purge_table_procedure_id, _) = cluster
.metasrv
.ddl_manager()
.submit_purge_dropped_table_task(common_meta::rpc::ddl::PurgeDroppedTableTask {
table_id,
})
.submit_purge_dropped_table_task(
common_meta::rpc::ddl::PurgeDroppedTableTask { table_id },
common_meta::rpc::ddl::EventContext::new(
common_meta::rpc::ddl::TriggerReason::Manual,
),
)
.await
.unwrap();
let purge_table_procedure_id = purge_table_procedure_id.to_string();
assert_event_context(
&frontend,
"purge_dropped_table",
&purge_table_procedure_id,
"manual",
"unknown",
)
.await;
assert_id_submitted_event(
&frontend,
"purge_dropped_table",
@@ -255,16 +463,29 @@ async fn test_table_ddl_procedure_events() {
)
.await;
let physical_table_id = find_table_id(&frontend, PHYSICAL_TABLE).await;
run_sql(
run_sql_with_context(
&frontend,
&format!(
"CREATE TABLE {LOGICAL_TABLE} (ts TIMESTAMP TIME INDEX, val DOUBLE, host STRING PRIMARY KEY) ENGINE=metric WITH (\"on_physical_table\" = \"{PHYSICAL_TABLE}\")"
),
Arc::new(QueryContext::with_channel(
"greptime",
"public",
Channel::HttpSql,
)),
)
.await;
let logical_table_id = find_table_id(&frontend, LOGICAL_TABLE).await;
let create_logical_tables_procedure_id =
submitted_procedure_id(&frontend, "create_logical_tables", LOGICAL_TABLE).await;
assert_event_context(
&frontend,
"create_logical_tables",
&create_logical_tables_procedure_id,
"manual",
"httpsql",
)
.await;
// Assert: logical Create emits one submitted row per logical table and
// preserves the logical and physical table IDs on its terminal row.
@@ -298,13 +519,26 @@ async fn test_table_ddl_procedure_events() {
// Act / Assert: logical Alter preserves the one-row-per-logical-table submitted
// contract and emits a lightweight terminal row.
run_sql(
run_sql_with_context(
&frontend,
&format!("ALTER TABLE {LOGICAL_TABLE} ADD COLUMN rack STRING PRIMARY KEY"),
Arc::new(QueryContext::with_channel(
"greptime",
"public",
Channel::HttpSql,
)),
)
.await;
let alter_logical_tables_procedure_id =
submitted_procedure_id(&frontend, "alter_logical_tables", LOGICAL_TABLE).await;
assert_event_context(
&frontend,
"alter_logical_tables",
&alter_logical_tables_procedure_id,
"manual",
"httpsql",
)
.await;
assert_named_submitted_event(
&frontend,
"alter_logical_tables",
@@ -327,14 +561,98 @@ async fn test_table_ddl_procedure_events() {
}
async fn run_sql(instance: &Arc<Instance>, sql: &str) {
let output = instance
.do_query(sql, QueryContext::arc())
run_sql_with_context(instance, sql, QueryContext::arc()).await;
}
async fn auto_create_table(instance: &Arc<Instance>, table_name: &str, channel: Channel) {
instance
.handle_row_inserts(
RowInsertRequests {
inserts: vec![RowInsertRequest {
table_name: table_name.to_string(),
rows: Some(Rows {
schema: vec![
api::v1::ColumnSchema {
column_name: "host".to_string(),
datatype: ColumnDataType::String as i32,
semantic_type: SemanticType::Tag as i32,
datatype_extension: None,
options: None,
},
api::v1::ColumnSchema {
column_name: "ts".to_string(),
datatype: ColumnDataType::TimestampMillisecond as i32,
semantic_type: SemanticType::Timestamp as i32,
datatype_extension: None,
options: None,
},
api::v1::ColumnSchema {
column_name: "val".to_string(),
datatype: ColumnDataType::Float64 as i32,
semantic_type: SemanticType::Field as i32,
datatype_extension: None,
options: None,
},
],
rows: vec![Row {
values: vec![
ValueData::StringValue("host".to_string()).into(),
ValueData::TimestampMillisecondValue(0).into(),
ValueData::F64Value(1.0).into(),
],
}],
}),
}],
},
Arc::new(QueryContext::with_channel("greptime", "public", channel)),
false,
false,
)
.await
.unwrap();
}
async fn run_sql_with_context(
instance: &Arc<Instance>,
sql: &str,
query_context: Arc<QueryContext>,
) {
let output = SqlQueryHandler::do_query(instance.as_ref(), sql, query_context)
.await
.remove(0)
.unwrap();
assert!(matches!(output.data, OutputData::AffectedRows(_)), "{sql}");
}
async fn assert_event_context(
instance: &Arc<Instance>,
event_type: &str,
procedure_id: &str,
reason: &str,
protocol: &str,
) {
let event_filter = format!(
"FROM {EVENTS_TABLE} WHERE {} = '{event_type}' AND {} = '{procedure_id}' AND json_get_string({}, 'type') = 'Submitted'",
TYPE_COLUMN.name(),
PROCEDURE_ID_COLUMN.name(),
PROCEDURE_TRIGGER_COLUMN.name(),
);
assert_single_event(
instance,
&format!("SELECT count(*) AS event_count {event_filter}"),
)
.await;
let actual = find_eventually_string(
instance,
&format!("SELECT json_to_string(event_context) AS event_context {event_filter}"),
"event_context",
)
.await;
let expected = format!(r#"{{"protocol":"{protocol}","reason":"{reason}"}}"#);
assert_eq!(expected, actual);
}
async fn find_table_id(instance: &Arc<Instance>, table_name: &str) -> u32 {
find_eventually_u32(
instance,
tests-integration/tests/view_ddl_event.rs
+18
View File
@@ -124,6 +124,7 @@ WHERE type = '{CREATE_VIEW_EVENT_TYPE}'
),
)
.await;
assert_event_context(instance, CREATE_VIEW_EVENT_TYPE, &procedure_id).await;
assert_single_event(
instance,
&format!(
@@ -143,6 +144,22 @@ WHERE type = '{CREATE_VIEW_EVENT_TYPE}'
.await;
}
async fn assert_event_context(
instance: &Arc<frontend::instance::Instance>,
event_type: &str,
procedure_id: &str,
) {
let actual = find_eventually_string(
instance,
&format!(
"SELECT json_to_string(event_context) AS event_context FROM greptime_private.events WHERE type = '{event_type}' AND procedure_id = '{procedure_id}' AND json_path_match(procedure_trigger, '$.type == \"Submitted\"')"
),
"event_context",
)
.await;
assert_eq!(r#"{"reason":"manual"}"#, actual);
}
async fn assert_drop_events(instance: &Arc<frontend::instance::Instance>, view: &str) {
let procedure_id = find_submitted_procedure_id(instance, DROP_VIEW_EVENT_TYPE, view).await;
assert_single_event(
@@ -163,6 +180,7 @@ WHERE type = '{DROP_VIEW_EVENT_TYPE}'
),
)
.await;
assert_event_context(instance, DROP_VIEW_EVENT_TYPE, &procedure_id).await;
assert_single_event(
instance,
&format!(