rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-08-24 23:18:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(procedure): centralize event context handling (#8834)

Browse Source 
* refactor(procedure): centralize event context handling

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(meta): simplify migration trigger reason handling

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor(meta): avoid cloning event context

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>
(cherry picked from commit 72f6cf09bf)
This commit is contained in:
Weny Xu
2026-08-11 17:05:26 +08:00
committed by discord9
parent e80f539077
commit 5d059c84bd
86 changed files with 1143 additions and 1654 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -2420,6 +2420,7 @@ dependencies = [
"serde_json",
"snafu 0.8.6",
"store-api",
"strum 0.27.1",
"tokio",
"tokio-util",
"toml 0.8.23",
src/common/event-recorder/Cargo.toml
+1
View File
@@ -23,6 +23,7 @@ serde.workspace = true
serde_json.workspace = true
snafu.workspace = true
store-api.workspace = true
strum.workspace = true
tokio.workspace = true
tokio-util.workspace = true
src/common/event-recorder/src/context.rs
+104
View File
@@ -0,0 +1,104 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use serde::{Deserialize, Serialize};
use strum::{AsRefStr, EnumString};
/// Stable context recorded for a procedure event.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PersistentEventContext {
pub reason: TriggerReason,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub protocol: Option<String>,
#[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
pub extensions: serde_json::Map<String, serde_json::Value>,
}
impl PersistentEventContext {
/// Creates an event context with no additional extensions.
pub fn new(reason: TriggerReason) -> Self {
Self {
reason,
protocol: None,
extensions: Default::default(),
}
}
/// Adds the protocol that originated the operation.
pub fn with_protocol(mut self, protocol: impl Into<String>) -> Self {
self.protocol = Some(protocol.into());
self
}
}
impl Default for PersistentEventContext {
fn default() -> Self {
Self::new(TriggerReason::default())
}
}
/// Stable classification of a procedure trigger.
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, AsRefStr, EnumString,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum TriggerReason {
Manual,
AutoCreate,
AutoAlter,
AutoRepartition,
AutoRebalance,
RegionFailover,
ScheduledGc,
#[default]
#[serde(other)]
Unknown,
}
impl TriggerReason {
pub fn from_extension(value: &str) -> Self {
value.parse().unwrap_or_default()
}
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
#[test]
fn test_event_context_serialization() {
let context = PersistentEventContext::new(TriggerReason::Manual).with_protocol("mysql");
assert_eq!(
json!({
"reason": "manual",
"protocol": "mysql",
}),
serde_json::to_value(context).unwrap()
);
assert_eq!(
json!({ "reason": "manual" }),
serde_json::to_value(PersistentEventContext::new(TriggerReason::Manual)).unwrap()
);
}
#[test]
fn test_trigger_reason_deserializes_unknown_value() {
let reason: TriggerReason = serde_json::from_str("\"future_reason\"").unwrap();
assert_eq!(TriggerReason::Unknown, reason);
}
}
src/common/event-recorder/src/lib.rs
+2
View File
@@ -14,6 +14,7 @@
#![feature(duration_constructors)]
pub mod context;
pub mod error;
pub mod event_table;
pub mod recorder;
@@ -21,4 +22,5 @@ pub mod recorder;
#[cfg(any(test, feature = "testing"))]
pub mod testing;
pub use context::{PersistentEventContext, TriggerReason};
pub use recorder::*;
src/common/meta/src/ddl/alter_database.rs
+6 -17
View File
@@ -15,7 +15,7 @@
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Status,
};
use common_telemetry::tracing::info;
use serde::{Deserialize, Serialize};
@@ -32,7 +32,7 @@ use crate::key::DeserializedValueWithBytes;
use crate::key::schema_name::{SchemaName, SchemaNameKey, SchemaNameValue};
use crate::lock_key::{CatalogLock, SchemaLock};
use crate::rpc::ddl::UnsetDatabaseOption::{self};
use crate::rpc::ddl::{AlterDatabaseKind, AlterDatabaseTask, EventContext, SetDatabaseOption};
use crate::rpc::ddl::{AlterDatabaseKind, AlterDatabaseTask, SetDatabaseOption};
pub struct AlterDatabaseProcedure {
pub context: DdlContext,
@@ -73,14 +73,10 @@ fn build_new_schema_value(
impl AlterDatabaseProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterDatabase";
pub fn new(
task: AlterDatabaseTask,
event_context: EventContext,
context: DdlContext,
) -> Result<Self> {
pub fn new(task: AlterDatabaseTask, context: DdlContext) -> Result<Self> {
Ok(Self {
context,
data: AlterDatabaseData::new(task, event_context)?,
data: AlterDatabaseData::new(task)?,
})
}
@@ -180,10 +176,7 @@ impl Procedure for AlterDatabaseProcedure {
LockKey::new(lock_key)
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(ALTER_DATABASE_EVENT_TYPE) {
return None;
}
@@ -193,7 +186,6 @@ impl Procedure for AlterDatabaseProcedure {
self.data.catalog(),
self.data.schema(),
&self.data.kind,
self.data.event_context.clone(),
)
} else {
DatabaseDdlEvent::alter_lifecycle(self.data.catalog(), self.data.schema())
@@ -217,19 +209,16 @@ pub struct AlterDatabaseData {
catalog_name: String,
schema_name: String,
schema_value: Option<DeserializedValueWithBytes<SchemaNameValue>>,
#[serde(default)]
event_context: EventContext,
}
impl AlterDatabaseData {
pub fn new(task: AlterDatabaseTask, event_context: EventContext) -> Result<Self> {
pub fn new(task: AlterDatabaseTask) -> Result<Self> {
Ok(Self {
state: AlterDatabaseState::Prepare,
kind: AlterDatabaseKind::try_from(task.alter_expr.kind.unwrap())?,
catalog_name: task.alter_expr.catalog_name,
schema_name: task.alter_expr.schema_name,
schema_value: None,
event_context,
})
}
src/common/meta/src/ddl/alter_logical_tables.rs
+3 -11
View File
@@ -20,7 +20,7 @@ use api::region::RegionResponse;
use async_trait::async_trait;
use common_catalog::format_full_table_name;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{Context, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status};
use common_procedure::{Context, EventContext, EventTrigger, LockKey, Procedure, Status};
use common_telemetry::{debug, error, info, warn};
pub use executor::make_alter_region_request;
use serde::{Deserialize, Serialize};
@@ -47,7 +47,7 @@ use crate::key::table_info::TableInfoValue;
use crate::key::table_route::PhysicalTableRouteValue;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
use crate::metrics;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::router::RegionRoute;
pub struct AlterLogicalTablesProcedure {
@@ -88,7 +88,6 @@ impl AlterLogicalTablesProcedure {
pub fn new(
tasks: Vec<AlterTableTask>,
physical_table_id: TableId,
event_context: EventContext,
context: DdlContext,
) -> Self {
Self {
@@ -101,7 +100,6 @@ impl AlterLogicalTablesProcedure {
physical_table_info: None,
physical_columns: vec![],
table_cache_keys_to_invalidate: vec![],
event_context,
},
physical_table_route: None,
}
@@ -322,10 +320,7 @@ impl Procedure for AlterLogicalTablesProcedure {
LockKey::new(lock_key)
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::AlterLogicalTables.as_str())
@@ -358,7 +353,6 @@ impl Procedure for AlterLogicalTablesProcedure {
locators,
self.data.tasks.len(),
kinds,
self.data.event_context.clone(),
)))
}
}
@@ -375,8 +369,6 @@ pub struct AlterTablesData {
physical_table_info: Option<DeserializedValueWithBytes<TableInfoValue>>,
physical_columns: Vec<ColumnMetadata>,
table_cache_keys_to_invalidate: Vec<CacheIdent>,
#[serde(default)]
event_context: EventContext,
}
impl AlterTablesData {
src/common/meta/src/ddl/alter_table.rs
+8 -25
View File
@@ -27,7 +27,7 @@ use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE};
use common_error::ext::BoxedError;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, ContextProvider, Error as ProcedureError, EventRuntimeContext,
Context as ProcedureContext, ContextProvider, Error as ProcedureError, EventContext,
EventTrigger, LockKey, PoisonKey, PoisonKeys, Procedure, ProcedureId, Status, StringKey,
};
use common_telemetry::{error, info, warn};
@@ -58,7 +58,7 @@ use crate::key::{DeserializedValueWithBytes, RegionDistribution};
use crate::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock, TableNameLock};
use crate::metrics;
use crate::poison_key::table_poison_key;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::router::{RegionRoute, find_leaders, region_distribution};
/// The alter table procedure
@@ -97,24 +97,18 @@ fn build_executor_from_alter_expr(alter_data: &AlterTableData) -> AlterTableExec
impl AlterTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterTable";
pub fn new(
table_id: TableId,
task: AlterTableTask,
event_context: EventContext,
context: DdlContext,
) -> Result<Self> {
Self::new_with_region_locks(table_id, task, event_context, vec![], context)
pub fn new(table_id: TableId, task: AlterTableTask, context: DdlContext) -> Result<Self> {
Self::new_with_region_locks(table_id, task, vec![], context)
}
pub(crate) fn new_with_region_locks(
table_id: TableId,
task: AlterTableTask,
event_context: EventContext,
region_locks: Vec<RegionId>,
context: DdlContext,
) -> Result<Self> {
task.validate()?;
let data = AlterTableData::new(task, table_id, event_context, region_locks);
let data = AlterTableData::new(task, table_id, region_locks);
let executor = build_executor_from_alter_expr(&data);
Ok(Self {
context,
@@ -569,10 +563,7 @@ impl Procedure for AlterTableProcedure {
PoisonKeys::new(vec![self.table_poison_key()])
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::AlterTable.as_str())
@@ -591,7 +582,7 @@ impl Procedure for AlterTableProcedure {
.kind
.as_ref()
.and_then(alter_table_kind_name);
TableDdlEvent::alter_table_submitted(locator, kind, self.data.event_context.clone())
TableDdlEvent::alter_table_submitted(locator, kind)
}
_ => TableDdlEvent::lifecycle(TableDdlEventType::AlterTable, [locator]),
};
@@ -619,8 +610,6 @@ pub struct AlterTableData {
task: AlterTableTask,
table_id: TableId,
#[serde(default)]
event_context: EventContext,
#[serde(default)]
column_metadatas: Vec<ColumnMetadata>,
/// Table info value before alteration.
table_info_value: Option<DeserializedValueWithBytes<TableInfoValue>>,
@@ -632,17 +621,11 @@ pub struct AlterTableData {
}
impl AlterTableData {
pub fn new(
task: AlterTableTask,
table_id: TableId,
event_context: EventContext,
region_locks: Vec<RegionId>,
) -> Self {
pub fn new(task: AlterTableTask, table_id: TableId, region_locks: Vec<RegionId>) -> Self {
Self {
state: AlterTableState::Prepare,
task,
table_id,
event_context,
column_metadatas: vec![],
table_info_value: None,
region_distribution: None,
src/common/meta/src/ddl/alter_table/region_request.rs
+3 -5
View File
@@ -144,7 +144,7 @@ mod tests {
};
use crate::key::table_route::TableRouteValue;
use crate::peer::Peer;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::router::{Region, RegionRoute};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -240,8 +240,7 @@ mod tests {
},
};
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context).unwrap();
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap();
procedure.on_prepare().await.unwrap();
let alter_kind = procedure.make_region_alter_kind().unwrap();
let Some(Body::Alter(alter_region_request)) =
@@ -297,8 +296,7 @@ mod tests {
},
};
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context).unwrap();
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap();
procedure.on_prepare().await.unwrap();
let alter_kind = procedure.make_region_alter_kind().unwrap();
let Some(Body::Alter(alter_region_request)) =
src/common/meta/src/ddl/create_database.rs
+4 -15
View File
@@ -19,8 +19,8 @@ use async_trait::async_trait;
use common_error::ext::{BoxedError, ErrorExt};
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureId, Status,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureId,
Status,
};
use serde::{Deserialize, Serialize};
use serde_with::{DefaultOnNull, serde_as};
@@ -34,7 +34,7 @@ use crate::error::{self, Result};
use crate::instruction::{CacheIdent, UserCacheIdent};
use crate::key::schema_name::{SchemaNameKey, SchemaNameValue};
use crate::lock_key::{CatalogLock, SchemaLock};
use crate::rpc::ddl::{CreatorGrantIntent, EventContext};
use crate::rpc::ddl::CreatorGrantIntent;
/// Describes the creator-access result of an atomic create.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -93,7 +93,6 @@ impl CreateDatabaseProcedure {
create_if_not_exists: bool,
options: HashMap<String, String>,
creator: Option<CreatorGrantIntent>,
event_context: EventContext,
context: DdlContext,
) -> Self {
Self {
@@ -105,7 +104,6 @@ impl CreateDatabaseProcedure {
create_if_not_exists,
options,
creator,
event_context,
},
}
}
@@ -280,10 +278,7 @@ impl Procedure for CreateDatabaseProcedure {
LockKey::new(lock_key)
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(CREATE_DATABASE_EVENT_TYPE) {
return None;
}
@@ -294,7 +289,6 @@ impl Procedure for CreateDatabaseProcedure {
&self.data.schema,
self.data.create_if_not_exists,
&self.data.options,
self.data.event_context.clone(),
)
} else {
DatabaseDdlEvent::create_lifecycle(&self.data.catalog, &self.data.schema)
@@ -327,8 +321,6 @@ pub struct CreateDatabaseData {
/// Authenticated creator whose access is ensured, absent for legacy schema-only requests.
#[serde(default)]
pub creator: Option<CreatorGrantIntent>,
#[serde(default)]
pub event_context: EventContext,
}
#[cfg(test)]
@@ -423,7 +415,6 @@ mod tests {
username: "alice".to_string(),
created_at_ns: 1,
}),
EventContext::default(),
context,
);
(procedure, committer)
@@ -583,7 +574,6 @@ mod tests {
false,
HashMap::new(),
None,
EventContext::default(),
context,
);
procedure.data.state = CreateDatabaseState::CreateMetadata;
@@ -619,7 +609,6 @@ mod tests {
username: "alice".to_string(),
created_at_ns: 1,
}),
EventContext::default(),
context,
);
procedure.data.state = CreateDatabaseState::CreateMetadata;
src/common/meta/src/ddl/create_flow.rs
+5 -17
View File
@@ -24,8 +24,8 @@ use async_trait::async_trait;
use common_catalog::format_full_flow_name;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureState, Result as ProcedureResult, Status,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState,
Result as ProcedureResult, Status,
};
use common_telemetry::info;
use common_telemetry::tracing_context::TracingContext;
@@ -50,7 +50,7 @@ use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId};
use crate::lock_key::{CatalogLock, FlowNameLock};
use crate::metrics;
use crate::peer::Peer;
use crate::rpc::ddl::{CreateFlowTask, EventContext, FlowQueryContext, QueryContext};
use crate::rpc::ddl::{CreateFlowTask, FlowQueryContext, QueryContext};
/// The procedure of flow creation.
pub struct CreateFlowProcedure {
@@ -62,12 +62,7 @@ impl CreateFlowProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateFlow";
/// Returns a new [CreateFlowProcedure].
pub fn new(
task: CreateFlowTask,
query_context: QueryContext,
event_context: EventContext,
context: DdlContext,
) -> Self {
pub fn new(task: CreateFlowTask, query_context: QueryContext, context: DdlContext) -> Self {
Self {
context,
data: CreateFlowData {
@@ -81,7 +76,6 @@ impl CreateFlowProcedure {
prev_flow_info_value: None,
did_replace: false,
flow_type: None,
event_context,
},
}
}
@@ -410,10 +404,7 @@ impl Procedure for CreateFlowProcedure {
])
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(CREATE_FLOW_EVENT_TYPE) {
return None;
}
@@ -428,7 +419,6 @@ impl Procedure for CreateFlowProcedure {
expire_after: self.data.task.expire_after,
eval_interval_secs: self.data.task.eval_interval_secs,
},
self.data.event_context.clone(),
),
EventTrigger::Succeeded => {
let flow_id = match ctx.lifecycle_state {
@@ -718,8 +708,6 @@ pub struct CreateFlowData {
#[serde(default)]
pub(crate) did_replace: bool,
pub(crate) flow_type: Option<FlowType>,
#[serde(default)]
pub(crate) event_context: EventContext,
}
impl CreateFlowData {
src/common/meta/src/ddl/create_logical_tables.rs
+4 -9
View File
@@ -24,8 +24,8 @@ use common_catalog::consts::METRIC_ENGINE;
use common_event_recorder::Event;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureState, Status,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState,
Status,
};
use common_telemetry::{debug, error, warn};
use futures::future;
@@ -48,7 +48,7 @@ use crate::error::Result;
use crate::key::table_route::TableRouteValue;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
use crate::metrics;
use crate::rpc::ddl::{CreateTableTask, EventContext};
use crate::rpc::ddl::CreateTableTask;
use crate::rpc::router::{RegionRoute, find_leaders};
pub struct CreateLogicalTablesProcedure {
@@ -62,7 +62,6 @@ impl CreateLogicalTablesProcedure {
pub fn new(
tasks: Vec<CreateTableTask>,
physical_table_id: TableId,
event_context: EventContext,
context: DdlContext,
) -> Self {
Self {
@@ -75,7 +74,6 @@ impl CreateLogicalTablesProcedure {
physical_region_numbers: vec![],
physical_columns: vec![],
physical_partition_columns: vec![],
event_context,
},
}
}
@@ -273,7 +271,7 @@ impl Procedure for CreateLogicalTablesProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::CreateLogicalTables.as_str())
@@ -284,7 +282,6 @@ impl Procedure for CreateLogicalTablesProcedure {
EventTrigger::Submitted => TableDdlEvent::create_logical_tables_submitted(
self.event_locators(),
self.data.tasks.len(),
self.data.event_context.clone(),
),
EventTrigger::Succeeded => match ctx.lifecycle_state {
ProcedureState::Done {
@@ -329,8 +326,6 @@ pub struct CreateTablesData {
physical_region_numbers: Vec<RegionNumber>,
physical_columns: Vec<ColumnMetadata>,
physical_partition_columns: Vec<String>,
#[serde(default)]
event_context: EventContext,
}
impl CreateTablesData {
src/common/meta/src/ddl/create_table.rs
+11 -23
View File
@@ -23,8 +23,8 @@ use common_procedure::error::{
};
use common_procedure::local::DynamicKeyLockGuard;
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureId, ProcedureState, Status,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureId,
ProcedureState, Status,
};
use common_telemetry::info;
use serde::{Deserialize, Serialize};
@@ -47,7 +47,7 @@ use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock};
use crate::metrics;
use crate::peer::PeerAllocContext;
use crate::region_keeper::OperatingRegionGuard;
use crate::rpc::ddl::{CreateTableTask, EventContext, QueryContext};
use crate::rpc::ddl::{CreateTableTask, QueryContext};
use crate::rpc::router::{RegionRoute, operating_leader_region_roles};
use crate::wal_provider::{
RegionWalOptions, acquire_remote_wal_read_locks, optional_region_wal_options_serde,
@@ -84,17 +84,20 @@ fn build_executor_from_create_table_data(
impl CreateTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateTable";
pub fn new(
pub fn new(task: CreateTableTask, context: DdlContext) -> Result<Self> {
Self::new_with_query_context(task, QueryContext::default(), context)
}
pub fn new_with_query_context(
task: CreateTableTask,
query_context: QueryContext,
event_context: EventContext,
context: DdlContext,
) -> Result<Self> {
let executor = build_executor_from_create_table_data(&task.create_table)?;
Ok(Self {
context,
data: CreateTableData::new_with_query_context(task, event_context, query_context),
data: CreateTableData::new(task, query_context),
opening_regions: vec![],
executor,
remote_wal_lock_guards: vec![],
@@ -396,10 +399,7 @@ impl Procedure for CreateTableProcedure {
])
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::CreateTable.as_str())
@@ -415,7 +415,6 @@ impl Procedure for CreateTableProcedure {
locator,
create_table.create_if_not_exists,
&create_table.engine,
self.data.event_context.clone(),
)
}
EventTrigger::Succeeded => match ctx.lifecycle_state {
@@ -456,8 +455,6 @@ pub struct CreateTableData {
#[serde(default)]
pub query_context: QueryContext,
#[serde(default)]
pub event_context: EventContext,
#[serde(default)]
pub column_metadatas: Vec<ColumnMetadata>,
/// None stands for not allocated yet.
pub(crate) table_route: Option<PhysicalTableRouteValue>,
@@ -468,21 +465,12 @@ pub struct CreateTableData {
}
impl CreateTableData {
pub fn new(task: CreateTableTask, event_context: EventContext) -> Self {
Self::new_with_query_context(task, event_context, QueryContext::default())
}
pub fn new_with_query_context(
task: CreateTableTask,
event_context: EventContext,
query_context: QueryContext,
) -> Self {
pub fn new(task: CreateTableTask, query_context: QueryContext) -> Self {
CreateTableData {
state: CreateTableState::Prepare,
column_metadatas: vec![],
task,
query_context,
event_context,
table_route: None,
region_wal_options: None,
}
src/common/meta/src/ddl/create_view.rs
+5 -9
View File
@@ -16,8 +16,8 @@ use async_trait::async_trait;
use common_event_recorder::Event;
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
ProcedureState, Status,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState,
Status,
};
use common_telemetry::info;
use serde::{Deserialize, Serialize};
@@ -35,7 +35,7 @@ use crate::instruction::CacheIdent;
use crate::key::table_name::TableNameKey;
use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock};
use crate::metrics;
use crate::rpc::ddl::{CreateViewTask, EventContext};
use crate::rpc::ddl::CreateViewTask;
// The procedure to execute `[CreateViewTask]`.
pub struct CreateViewProcedure {
@@ -46,14 +46,13 @@ pub struct CreateViewProcedure {
impl CreateViewProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateView";
pub fn new(task: CreateViewTask, event_context: EventContext, context: DdlContext) -> Self {
pub fn new(task: CreateViewTask, context: DdlContext) -> Self {
Self {
context,
data: CreateViewData {
state: CreateViewState::Prepare,
task,
need_update: false,
event_context,
},
}
}
@@ -272,7 +271,7 @@ impl Procedure for CreateViewProcedure {
])
}
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
if !ctx.event_type_filter.allows(CREATE_VIEW_EVENT_TYPE) {
return None;
}
@@ -290,7 +289,6 @@ impl Procedure for CreateViewProcedure {
referenced_table_count: self.data.task.table_names().len(),
column_count: self.data.task.columns().len(),
},
self.data.event_context.clone(),
)
}
EventTrigger::Succeeded => match ctx.lifecycle_state {
@@ -344,8 +342,6 @@ pub struct CreateViewData {
pub task: CreateViewTask,
/// Whether to update the view info.
pub need_update: bool,
#[serde(default)]
pub event_context: EventContext,
}
impl CreateViewData {
src/common/meta/src/ddl/drop_database.rs
+3 -22
View File
@@ -23,7 +23,7 @@ use std::fmt::Debug;
use common_error::ext::BoxedError;
use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use futures::stream::BoxStream;
@@ -38,13 +38,11 @@ use crate::ddl::utils::map_to_procedure_error;
use crate::error::Result;
use crate::key::table_name::TableNameValue;
use crate::lock_key::{CatalogLock, SchemaLock};
use crate::rpc::ddl::EventContext;
pub struct DropDatabaseProcedure {
/// The context of procedure runtime.
runtime_context: DdlContext,
context: DropDatabaseContext,
event_context: EventContext,
state: Box<dyn State>,
}
@@ -87,16 +85,9 @@ pub(crate) trait State: Send + Debug {
impl DropDatabaseProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::DropDatabase";
pub fn new(
catalog: String,
schema: String,
drop_if_exists: bool,
event_context: EventContext,
context: DdlContext,
) -> Self {
pub fn new(catalog: String, schema: String, drop_if_exists: bool, context: DdlContext) -> Self {
Self {
runtime_context: context,
event_context,
context: DropDatabaseContext {
catalog,
schema,
@@ -113,13 +104,11 @@ impl DropDatabaseProcedure {
catalog,
schema,
drop_if_exists,
event_context,
state,
} = serde_json::from_str(json).context(FromJsonSnafu)?;
Ok(Self {
runtime_context,
event_context,
context: DropDatabaseContext {
catalog,
schema,
@@ -170,7 +159,6 @@ impl Procedure for DropDatabaseProcedure {
catalog: &self.context.catalog,
schema: &self.context.schema,
drop_if_exists: self.context.drop_if_exists,
event_context: &self.event_context,
state: self.state.as_ref(),
};
@@ -186,10 +174,7 @@ impl Procedure for DropDatabaseProcedure {
LockKey::new(lock_key)
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(DROP_DATABASE_EVENT_TYPE) {
return None;
}
@@ -199,7 +184,6 @@ impl Procedure for DropDatabaseProcedure {
&self.context.catalog,
&self.context.schema,
self.context.drop_if_exists,
self.event_context.clone(),
)
} else {
DatabaseDdlEvent::drop_lifecycle(&self.context.catalog, &self.context.schema)
@@ -215,7 +199,6 @@ struct DropDatabaseData<'a> {
// The schema name
schema: &'a str,
drop_if_exists: bool,
event_context: &'a EventContext,
state: &'a dyn State,
}
@@ -226,7 +209,5 @@ struct DropDatabaseOwnedData {
// The schema name
schema: String,
drop_if_exists: bool,
#[serde(default)]
event_context: EventContext,
state: Box<dyn State>,
}
src/common/meta/src/ddl/drop_flow.rs
+4 -11
View File
@@ -21,7 +21,7 @@ use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::info;
@@ -41,7 +41,7 @@ use crate::key::flow::flow_info::FlowInfoValue;
use crate::key::flow::flow_route::FlowRouteValue;
use crate::lock_key::{CatalogLock, FlowLock};
use crate::metrics;
use crate::rpc::ddl::{DropFlowTask, EventContext};
use crate::rpc::ddl::DropFlowTask;
/// The procedure for dropping a flow.
pub struct DropFlowProcedure {
@@ -54,7 +54,7 @@ pub struct DropFlowProcedure {
impl DropFlowProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::DropFlow";
pub fn new(task: DropFlowTask, event_context: EventContext, context: DdlContext) -> Self {
pub fn new(task: DropFlowTask, context: DdlContext) -> Self {
Self {
context,
data: DropFlowData {
@@ -62,7 +62,6 @@ impl DropFlowProcedure {
task,
flow_info_value: None,
flow_route_values: vec![],
event_context,
},
}
}
@@ -223,10 +222,7 @@ impl Procedure for DropFlowProcedure {
LockKey::new(lock_key)
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(DROP_FLOW_EVENT_TYPE) {
return None;
}
@@ -237,7 +233,6 @@ impl Procedure for DropFlowProcedure {
&self.data.task.flow_name,
self.data.task.flow_id,
self.data.task.drop_if_exists,
self.data.event_context.clone(),
),
_ => FlowDdlEvent::drop_lifecycle(
&self.data.task.catalog_name,
@@ -257,8 +252,6 @@ pub(crate) struct DropFlowData {
task: DropFlowTask,
pub(crate) flow_info_value: Option<FlowInfoValue>,
pub(crate) flow_route_values: Vec<FlowRouteValue>,
#[serde(default)]
pub(crate) event_context: EventContext,
}
/// The state of drop flow
src/common/meta/src/ddl/drop_table.rs
+8 -15
View File
@@ -22,8 +22,8 @@ use common_error::ext::BoxedError;
use common_event_recorder::Event;
use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger,
LockKey, Procedure, Result as ProcedureResult, Status,
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, Result as ProcedureResult, Status,
};
use common_telemetry::info;
use common_telemetry::tracing::warn;
@@ -48,7 +48,7 @@ use crate::key::table_route::TableRouteValue;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
use crate::metrics;
use crate::region_keeper::OperatingRegionGuard;
use crate::rpc::ddl::{DropTableTask, EventContext};
use crate::rpc::ddl::DropTableTask;
use crate::rpc::router::{RegionRoute, operating_leader_region_roles};
#[cfg(feature = "enterprise")]
@@ -74,12 +74,11 @@ pub struct DropTableProcedure {
impl DropTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::DropTable";
pub fn new(task: DropTableTask, context: DdlContext, event_context: EventContext) -> Self {
pub fn new(task: DropTableTask, context: DdlContext) -> Self {
let data = DropTableData::new(
task,
cfg!(feature = "enterprise") && context.soft_drop_enabled,
context.soft_drop_retention,
event_context,
);
let executor = data.build_executor();
Self {
@@ -414,7 +413,7 @@ impl Procedure for DropTableProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::DropTable.as_str())
@@ -425,11 +424,9 @@ impl Procedure for DropTableProcedure {
let locator = TableDdlLocator::new(&task.catalog, &task.schema, &task.table)
.with_table_id(task.table_id);
let event = match &ctx.trigger {
EventTrigger::Submitted => TableDdlEvent::drop_table_submitted(
locator,
task.drop_if_exists,
self.data.event_context.clone(),
),
EventTrigger::Submitted => {
TableDdlEvent::drop_table_submitted(locator, task.drop_if_exists)
}
_ => TableDdlEvent::lifecycle(TableDdlEventType::DropTable, [locator]),
};
@@ -483,8 +480,6 @@ pub struct DropTableData {
pub soft_drop_retention_millis: Option<i64>,
#[serde(default)]
pub drop_generation: Option<String>,
#[serde(default)]
pub event_context: EventContext,
}
impl DropTableData {
@@ -492,7 +487,6 @@ impl DropTableData {
task: DropTableTask,
soft_drop_enabled: bool,
soft_drop_retention: Option<std::time::Duration>,
event_context: EventContext,
) -> Self {
Self {
state: DropTableState::Prepare,
@@ -506,7 +500,6 @@ impl DropTableData {
retention_expires_at: None,
soft_drop_retention_millis: soft_drop_retention_millis(soft_drop_retention),
drop_generation: None,
event_context,
}
}
src/common/meta/src/ddl/drop_view.rs
+4 -8
View File
@@ -16,7 +16,7 @@ use async_trait::async_trait;
use common_event_recorder::Event;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::info;
@@ -35,7 +35,7 @@ use crate::instruction::CacheIdent;
use crate::key::table_name::TableNameKey;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
use crate::metrics;
use crate::rpc::ddl::{DropViewTask, EventContext};
use crate::rpc::ddl::DropViewTask;
/// The procedure for dropping a view.
pub struct DropViewProcedure {
@@ -48,13 +48,12 @@ pub struct DropViewProcedure {
impl DropViewProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::DropView";
pub fn new(task: DropViewTask, event_context: EventContext, context: DdlContext) -> Self {
pub fn new(task: DropViewTask, context: DdlContext) -> Self {
Self {
context,
data: DropViewData {
state: DropViewState::Prepare,
task,
event_context,
},
}
}
@@ -214,7 +213,7 @@ impl Procedure for DropViewProcedure {
LockKey::new(lock_key)
}
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
if !ctx.event_type_filter.allows(DROP_VIEW_EVENT_TYPE) {
return None;
}
@@ -228,7 +227,6 @@ impl Procedure for DropViewProcedure {
table_ref.table,
self.data.view_id(),
self.data.task.drop_if_exists,
self.data.event_context.clone(),
)
}
_ => {
@@ -251,8 +249,6 @@ impl Procedure for DropViewProcedure {
pub(crate) struct DropViewData {
state: DropViewState,
task: DropViewTask,
#[serde(default)]
event_context: EventContext,
}
impl DropViewData {
src/common/meta/src/ddl/event/database.rs
+3 -22
View File
@@ -19,14 +19,13 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN,
SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, column_schemas, nullable_json,
nullable_string,
CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN,
SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, column_schemas, nullable_string,
};
use serde::Serialize;
use snafu::ResultExt;
use crate::rpc::ddl::{AlterDatabaseKind, EventContext, SetDatabaseOption, UnsetDatabaseOption};
use crate::rpc::ddl::{AlterDatabaseKind, SetDatabaseOption, UnsetDatabaseOption};
pub(crate) const CREATE_DATABASE_EVENT_TYPE: &str = "create_database";
pub(crate) const ALTER_DATABASE_EVENT_TYPE: &str = "alter_database";
@@ -40,7 +39,6 @@ pub(crate) struct DatabaseDdlEvent {
catalog_name: Option<String>,
schema_name: Option<String>,
payload: Option<DatabaseDdlPayload>,
event_context: Option<EventContext>,
}
#[derive(Debug, Serialize)]
@@ -90,7 +88,6 @@ impl DatabaseDdlEvent {
schema_name: &str,
create_if_not_exists: bool,
options: &HashMap<String, String>,
event_context: EventContext,
) -> Self {
let mut options = options.iter().collect::<Vec<_>>();
options.sort_unstable_by_key(|(left, _)| *left);
@@ -110,7 +107,6 @@ impl DatabaseDdlEvent {
create_if_not_exists,
options,
}),
event_context,
)
}
@@ -118,7 +114,6 @@ impl DatabaseDdlEvent {
catalog_name: &str,
schema_name: &str,
kind: &AlterDatabaseKind,
event_context: EventContext,
) -> Self {
let intent = match kind {
AlterDatabaseKind::SetDatabaseOptions(options) => AlterDatabaseIntent::Set {
@@ -157,7 +152,6 @@ impl DatabaseDdlEvent {
version: PAYLOAD_VERSION,
intent,
}),
event_context,
)
}
@@ -165,7 +159,6 @@ impl DatabaseDdlEvent {
catalog_name: &str,
schema_name: &str,
drop_if_exists: bool,
event_context: EventContext,
) -> Self {
Self::submitted(
DROP_DATABASE_EVENT_TYPE,
@@ -175,7 +168,6 @@ impl DatabaseDdlEvent {
version: PAYLOAD_VERSION,
drop_if_exists,
}),
event_context,
)
}
@@ -196,14 +188,12 @@ impl DatabaseDdlEvent {
catalog_name: &str,
schema_name: &str,
payload: DatabaseDdlPayload,
event_context: EventContext,
) -> Self {
Self {
event_type,
catalog_name: Some(catalog_name.to_string()),
schema_name: Some(schema_name.to_string()),
payload: Some(payload),
event_context: Some(event_context),
}
}
@@ -213,7 +203,6 @@ impl DatabaseDdlEvent {
catalog_name: Some(catalog_name.to_string()),
schema_name: Some(schema_name.to_string()),
payload: None,
event_context: None,
}
}
}
@@ -234,22 +223,14 @@ impl Event for DatabaseDdlEvent {
column_schemas([
&EVENT_TABLE_CATALOG_NAME_COLUMN,
&EVENT_TABLE_SCHEMA_NAME_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
Ok(vec![Row {
values: vec![
nullable_string(self.catalog_name.as_deref()),
nullable_string(self.schema_name.as_deref()),
nullable_json(event_context.as_ref()),
],
}])
}
src/common/meta/src/ddl/event/flow.rs
+5 -23
View File
@@ -19,14 +19,12 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, column_schemas,
nullable_json, nullable_string, nullable_value,
CATALOG_NAME_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, column_schemas, nullable_string,
nullable_value,
};
use serde::Serialize;
use snafu::ResultExt;
use crate::rpc::ddl::EventContext;
pub(crate) const CREATE_FLOW_EVENT_TYPE: &str = "create_flow";
pub(crate) const DROP_FLOW_EVENT_TYPE: &str = "drop_flow";
@@ -71,15 +69,14 @@ pub(crate) struct FlowDdlEvent {
flow_name: Option<String>,
flow_id: Option<u32>,
payload: Option<FlowDdlPayload>,
event_context: Option<EventContext>,
}
impl FlowDdlEvent {
/// Builds the bounded event emitted when creating a Flow is submitted.
pub(crate) fn create_submitted(
catalog_name: &str,
flow_name: &str,
intent: CreateFlowEventIntent,
event_context: EventContext,
) -> Self {
Self {
event_type: CREATE_FLOW_EVENT_TYPE,
@@ -93,16 +90,15 @@ impl FlowDdlEvent {
expire_after: intent.expire_after,
eval_interval_secs: intent.eval_interval_secs,
})),
event_context: Some(event_context),
}
}
/// Builds the bounded event emitted when dropping a Flow is submitted.
pub(crate) fn drop_submitted(
catalog_name: &str,
flow_name: &str,
flow_id: u32,
drop_if_exists: bool,
event_context: EventContext,
) -> Self {
Self {
event_type: DROP_FLOW_EVENT_TYPE,
@@ -113,7 +109,6 @@ impl FlowDdlEvent {
version: PAYLOAD_VERSION,
drop_if_exists,
})),
event_context: Some(event_context),
}
}
@@ -149,7 +144,6 @@ impl FlowDdlEvent {
flow_name: Some(flow_name.to_string()),
flow_id: None,
payload: None,
event_context: None,
}
}
}
@@ -167,27 +161,15 @@ impl Event for FlowDdlEvent {
}
fn extra_schema(&self) -> Vec<ColumnSchema> {
column_schemas([
&CATALOG_NAME_COLUMN,
&FLOW_NAME_COLUMN,
&FLOW_ID_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
column_schemas([&CATALOG_NAME_COLUMN, &FLOW_NAME_COLUMN, &FLOW_ID_COLUMN])
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
Ok(vec![Row {
values: vec![
nullable_string(self.catalog_name.as_deref()),
nullable_string(self.flow_name.as_deref()),
nullable_value(self.flow_id.map(ValueData::U32Value)),
nullable_json(event_context.as_ref()),
],
}])
}
src/common/meta/src/ddl/event/table.rs
+10 -47
View File
@@ -21,17 +21,14 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN,
TABLE_ID_COLUMN, TABLE_NAME_COLUMN, column_schemas, nullable_json, nullable_string,
nullable_value,
CATALOG_NAME_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN,
TABLE_NAME_COLUMN, column_schemas, nullable_string, nullable_value,
};
use serde::Serialize;
use serde_json::Value as JsonValue;
use snafu::ResultExt;
use store_api::storage::TableId;
use crate::rpc::ddl::EventContext;
/// Current version of table DDL event payloads.
pub(crate) const TABLE_DDL_PAYLOAD_VERSION: u8 = 1;
@@ -215,7 +212,6 @@ pub(crate) struct TableDdlEvent {
event_type: TableDdlEventType,
locators: Vec<TableDdlLocator>,
payload: Option<TableDdlPayload>,
event_context: Option<EventContext>,
}
impl TableDdlEvent {
@@ -224,7 +220,6 @@ impl TableDdlEvent {
locator: TableDdlLocator,
create_if_not_exists: bool,
engine: &str,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::CreateTable,
@@ -234,7 +229,6 @@ impl TableDdlEvent {
create_if_not_exists,
engine: engine.to_string(),
}),
Some(event_context),
)
}
@@ -242,7 +236,6 @@ impl TableDdlEvent {
pub(crate) fn create_logical_tables_submitted(
locators: impl IntoIterator<Item = TableDdlLocator>,
table_count: usize,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::CreateLogicalTables,
@@ -251,7 +244,6 @@ impl TableDdlEvent {
version: TABLE_DDL_PAYLOAD_VERSION,
table_count,
}),
Some(event_context),
)
}
@@ -259,7 +251,6 @@ impl TableDdlEvent {
pub(crate) fn alter_table_submitted(
locator: TableDdlLocator,
kind: Option<&'static str>,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::AlterTable,
@@ -268,7 +259,6 @@ impl TableDdlEvent {
version: TABLE_DDL_PAYLOAD_VERSION,
kind,
}),
Some(event_context),
)
}
@@ -277,7 +267,6 @@ impl TableDdlEvent {
locators: impl IntoIterator<Item = TableDdlLocator>,
table_count: usize,
kinds: impl IntoIterator<Item = &'static str>,
event_context: EventContext,
) -> Self {
let kinds = kinds
.into_iter()
@@ -292,16 +281,11 @@ impl TableDdlEvent {
table_count,
kinds,
}),
Some(event_context),
)
}
/// Builds the bounded event emitted when dropping a table is submitted.
pub(crate) fn drop_table_submitted(
locator: TableDdlLocator,
drop_if_exists: bool,
event_context: EventContext,
) -> Self {
pub(crate) fn drop_table_submitted(locator: TableDdlLocator, drop_if_exists: bool) -> Self {
Self::submitted(
TableDdlEventType::DropTable,
[locator],
@@ -309,39 +293,30 @@ impl TableDdlEvent {
version: TABLE_DDL_PAYLOAD_VERSION,
drop_if_exists,
}),
Some(event_context),
)
}
/// Builds the bounded event emitted when restoring a dropped table is submitted.
#[cfg(feature = "enterprise")]
pub(crate) fn undrop_table_submitted(
locator: TableDdlLocator,
event_context: EventContext,
) -> Self {
pub(crate) fn undrop_table_submitted(locator: TableDdlLocator) -> Self {
Self::submitted(
TableDdlEventType::UndropTable,
[locator],
TableDdlPayload::UndropTable(UndropTablePayload {
version: TABLE_DDL_PAYLOAD_VERSION,
}),
Some(event_context),
)
}
/// Builds the bounded event emitted when purging a dropped table is submitted.
#[cfg(feature = "enterprise")]
pub(crate) fn purge_dropped_table_submitted(
locator: TableDdlLocator,
event_context: EventContext,
) -> Self {
pub(crate) fn purge_dropped_table_submitted(locator: TableDdlLocator) -> Self {
Self::submitted(
TableDdlEventType::PurgeDroppedTable,
[locator],
TableDdlPayload::PurgeDroppedTable(PurgeDroppedTablePayload {
version: TABLE_DDL_PAYLOAD_VERSION,
}),
Some(event_context),
)
}
@@ -349,7 +324,6 @@ impl TableDdlEvent {
pub(crate) fn truncate_table_submitted(
locator: TableDdlLocator,
time_range_count: usize,
event_context: EventContext,
) -> Self {
Self::submitted(
TableDdlEventType::TruncateTable,
@@ -358,7 +332,6 @@ impl TableDdlEvent {
version: TABLE_DDL_PAYLOAD_VERSION,
time_range_count,
}),
Some(event_context),
)
}
@@ -371,7 +344,6 @@ impl TableDdlEvent {
event_type,
locators: locators.into_iter().collect(),
payload: None,
event_context: None,
}
}
@@ -394,13 +366,11 @@ impl TableDdlEvent {
event_type: TableDdlEventType,
locators: impl IntoIterator<Item = TableDdlLocator>,
payload: TableDdlPayload,
event_context: Option<EventContext>,
) -> Self {
Self {
event_type,
locators: locators.into_iter().collect(),
payload: Some(payload),
event_context,
}
}
@@ -413,7 +383,7 @@ impl TableDdlEvent {
])
}
fn locator_row(&self, locator: &TableDdlLocator) -> Result<Row> {
fn locator_row(&self, locator: &TableDdlLocator) -> Row {
let mut values = vec![
nullable_string(locator.catalog_name.as_deref()),
nullable_string(locator.schema_name.as_deref()),
@@ -423,14 +393,7 @@ impl TableDdlEvent {
if self.event_type.has_physical_table_id() {
values.push(nullable_table_id(locator.physical_table_id));
}
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
values.push(nullable_json(event_context.as_ref()));
Ok(Row { values })
Row { values }
}
}
@@ -451,15 +414,15 @@ impl Event for TableDdlEvent {
if self.event_type.has_physical_table_id() {
schema.push(PHYSICAL_TABLE_ID_COLUMN.column_schema());
}
schema.push(EVENT_CONTEXT_COLUMN.column_schema());
schema
}
fn extra_rows(&self) -> Result<Vec<Row>> {
self.locators
Ok(self
.locators
.iter()
.map(|locator| self.locator_row(locator))
.collect()
.collect())
}
fn as_any(&self) -> &dyn Any {
src/common/meta/src/ddl/event/view.rs
+4 -21
View File
@@ -19,14 +19,12 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN,
VIEW_NAME_COLUMN, column_schemas, nullable_json, nullable_string, nullable_value,
CATALOG_NAME_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, VIEW_NAME_COLUMN, column_schemas,
nullable_string, nullable_value,
};
use serde::Serialize;
use snafu::ResultExt;
use crate::rpc::ddl::EventContext;
pub(crate) const CREATE_VIEW_EVENT_TYPE: &str = "create_view";
pub(crate) const DROP_VIEW_EVENT_TYPE: &str = "drop_view";
@@ -64,7 +62,6 @@ pub(crate) struct ViewDdlEvent {
view_name: Option<String>,
view_id: Option<u32>,
payload: Option<ViewDdlPayload>,
event_context: Option<EventContext>,
}
#[derive(Debug, Serialize)]
@@ -75,12 +72,12 @@ enum ViewDdlPayload {
}
impl ViewDdlEvent {
/// Builds the bounded event emitted when creating a View is submitted.
pub(crate) fn create_submitted(
catalog_name: &str,
schema_name: &str,
view_name: &str,
intent: CreateViewEventIntent,
event_context: EventContext,
) -> Self {
Self::submitted(
CREATE_VIEW_EVENT_TYPE,
@@ -95,17 +92,16 @@ impl ViewDdlEvent {
referenced_table_count: intent.referenced_table_count,
column_count: intent.column_count,
}),
event_context,
)
}
/// Builds the bounded event emitted when dropping a View is submitted.
pub(crate) fn drop_submitted(
catalog_name: &str,
schema_name: &str,
view_name: &str,
view_id: u32,
drop_if_exists: bool,
event_context: EventContext,
) -> Self {
Self::submitted(
DROP_VIEW_EVENT_TYPE,
@@ -117,7 +113,6 @@ impl ViewDdlEvent {
version: PAYLOAD_VERSION,
drop_if_exists,
}),
event_context,
)
}
@@ -162,7 +157,6 @@ impl ViewDdlEvent {
view_name: &str,
view_id: Option<u32>,
payload: ViewDdlPayload,
event_context: EventContext,
) -> Self {
Self {
event_type,
@@ -171,7 +165,6 @@ impl ViewDdlEvent {
view_name: Some(view_name.to_string()),
view_id,
payload: Some(payload),
event_context: Some(event_context),
}
}
@@ -188,7 +181,6 @@ impl ViewDdlEvent {
view_name: Some(view_name.to_string()),
view_id: None,
payload: None,
event_context: None,
}
}
@@ -206,7 +198,6 @@ impl ViewDdlEvent {
view_name: Some(view_name.to_string()),
view_id: Some(view_id),
payload: None,
event_context: None,
}
}
}
@@ -229,24 +220,16 @@ impl Event for ViewDdlEvent {
&SCHEMA_NAME_COLUMN,
&VIEW_NAME_COLUMN,
&VIEW_ID_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
Ok(vec![Row {
values: vec![
nullable_string(self.catalog_name.as_deref()),
nullable_string(self.schema_name.as_deref()),
nullable_string(self.view_name.as_deref()),
nullable_value(self.view_id.map(ValueData::U32Value)),
nullable_json(event_context.as_ref()),
],
}])
}
src/common/meta/src/ddl/purge_dropped_table.rs
+11 -25
View File
@@ -14,7 +14,7 @@ use std::collections::HashMap;
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::info;
@@ -38,7 +38,7 @@ use crate::error::{self, Result};
use crate::key::DroppedTableMetadata;
use crate::key::table_route::TableRouteValue;
use crate::lock_key::TableLock;
use crate::rpc::ddl::{EventContext, PurgeDroppedTableTask};
use crate::rpc::ddl::PurgeDroppedTableTask;
use crate::rpc::router::RegionRoute;
pub struct PurgeDroppedTableProcedure {
@@ -50,25 +50,17 @@ impl PurgeDroppedTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::PurgeDroppedTable";
pub const EXPIRED_TYPE_NAME: &'static str = "metasrv-procedure::PurgeExpiredDroppedTable";
pub fn new(
task: PurgeDroppedTableTask,
context: DdlContext,
event_context: EventContext,
) -> Self {
pub fn new(task: PurgeDroppedTableTask, context: DdlContext) -> Self {
Self {
context,
data: PurgeDroppedTableData::new(task, event_context),
data: PurgeDroppedTableData::new(task),
}
}
pub fn new_if_expired(
task: PurgeDroppedTableTask,
context: DdlContext,
event_context: EventContext,
) -> Self {
pub fn new_if_expired(task: PurgeDroppedTableTask, context: DdlContext) -> Self {
Self {
context,
data: PurgeDroppedTableData::new_if_expired(task, event_context),
data: PurgeDroppedTableData::new_if_expired(task),
}
}
@@ -278,10 +270,7 @@ impl Procedure for PurgeDroppedTableProcedure {
LockKey::new(vec![TableLock::Write(self.data.task.table_id).into()])
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::PurgeDroppedTable.as_str())
@@ -290,7 +279,7 @@ impl Procedure for PurgeDroppedTableProcedure {
}
let locator = self.event_locator();
let event = if ctx.trigger == EventTrigger::Submitted {
TableDdlEvent::purge_dropped_table_submitted(locator, self.data.event_context.clone())
TableDdlEvent::purge_dropped_table_submitted(locator)
} else {
TableDdlEvent::lifecycle(TableDdlEventType::PurgeDroppedTable, [locator])
};
@@ -316,12 +305,10 @@ pub struct PurgeDroppedTableData {
drop_generation_loaded: bool,
#[serde(default)]
purging_claimed: bool,
#[serde(default)]
event_context: EventContext,
}
impl PurgeDroppedTableData {
fn new(task: PurgeDroppedTableTask, event_context: EventContext) -> Self {
fn new(task: PurgeDroppedTableTask) -> Self {
Self {
state: PurgeDroppedTableState::Prepare,
task,
@@ -334,14 +321,13 @@ impl PurgeDroppedTableData {
drop_generation: None,
drop_generation_loaded: false,
purging_claimed: false,
event_context,
}
}
fn new_if_expired(task: PurgeDroppedTableTask, event_context: EventContext) -> Self {
fn new_if_expired(task: PurgeDroppedTableTask) -> Self {
Self {
check_expired: true,
..Self::new(task, event_context)
..Self::new(task)
}
}
src/common/meta/src/ddl/test_util.rs
+2 -7
View File
@@ -45,7 +45,7 @@ use crate::key::node_address::{NodeAddressKey, NodeAddressValue};
use crate::key::table_route::TableRouteValue;
use crate::key::{MetadataKey, MetadataValue};
use crate::peer::Peer;
use crate::rpc::ddl::{CreateTableTask, EventContext};
use crate::rpc::ddl::CreateTableTask;
use crate::rpc::store::PutRequest;
pub async fn create_physical_table_metadata(
@@ -104,12 +104,7 @@ pub async fn create_logical_table(
table_name: &str,
) -> TableId {
let tasks = vec![test_create_logical_table_task(table_name)];
let mut procedure = CreateLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
src/common/meta/src/ddl/tests/alter_logical_tables.rs
+13 -39
View File
@@ -43,7 +43,7 @@ use crate::error::Error::{AlterLogicalTablesInvalidArguments, TableNotFound};
use crate::error::Result;
use crate::key::table_name::TableNameKey;
use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue};
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::router::{Region, RegionRoute};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -162,12 +162,7 @@ async fn test_on_prepare_check_schema() {
),
];
let physical_table_id = 1024u32;
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, AlterLogicalTablesInvalidArguments { .. });
}
@@ -182,12 +177,7 @@ async fn test_on_prepare_check_alter_kind() {
"new_table1",
)];
let physical_table_id = 1024u32;
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, AlterLogicalTablesInvalidArguments { .. });
}
@@ -207,8 +197,7 @@ async fn test_on_prepare_different_physical_table() {
make_alter_logical_table_add_column_task(None, "table2", vec!["column2".to_string()]),
];
let mut procedure =
AlterLogicalTablesProcedure::new(tasks, phy1_id, EventContext::default(), ddl_context);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy1_id, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, AlterLogicalTablesInvalidArguments { .. });
}
@@ -229,8 +218,7 @@ async fn test_on_prepare_logical_table_not_exists() {
make_alter_logical_table_add_column_task(None, "table2", vec!["column2".to_string()]),
];
let mut procedure =
AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, TableNotFound { .. });
}
@@ -253,8 +241,7 @@ async fn test_on_prepare() {
make_alter_logical_table_add_column_task(None, "table3", vec!["column3".to_string()]),
];
let mut procedure =
AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context);
let result = procedure.on_prepare().await;
assert_matches!(
result,
@@ -290,12 +277,7 @@ async fn test_on_update_metadata() {
make_alter_logical_table_add_column_task(None, "table3", vec!["new_col".to_string()]),
];
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
phy_id,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone());
let mut status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -308,6 +290,7 @@ async fn test_on_update_metadata() {
let ctx = common_procedure::Context {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
// on_submit_alter_region_requests
status = procedure.execute(&ctx).await.unwrap();
@@ -379,12 +362,7 @@ async fn test_on_part_duplicate_alter_request() {
make_alter_logical_table_add_column_task(None, "table2", vec!["col_0".to_string()]),
];
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
phy_id,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone());
let mut status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -397,6 +375,7 @@ async fn test_on_part_duplicate_alter_request() {
let ctx = common_procedure::Context {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
// on_submit_alter_region_requests
status = procedure.execute(&ctx).await.unwrap();
@@ -469,12 +448,7 @@ async fn test_on_part_duplicate_alter_request() {
),
];
let mut procedure = AlterLogicalTablesProcedure::new(
tasks,
phy_id,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone());
let mut status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -487,6 +461,7 @@ async fn test_on_part_duplicate_alter_request() {
let ctx = common_procedure::Context {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
// on_submit_alter_region_requests
status = procedure.execute(&ctx).await.unwrap();
@@ -645,8 +620,7 @@ async fn test_on_submit_alter_region_request() {
make_alter_logical_table_add_column_task(None, "table2", vec!["mew_col".to_string()]),
];
let mut procedure =
AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context);
let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context);
procedure.on_prepare().await.unwrap();
procedure.on_submit_alter_region_requests().await.unwrap();
let mut results = Vec::new();
src/common/meta/src/ddl/tests/alter_table.rs
+16 -56
View File
@@ -59,7 +59,7 @@ use crate::lock_key::RegionLock;
use crate::node_manager::NodeManagerRef;
use crate::peer::Peer;
use crate::poison_key::table_poison_key;
use crate::rpc::ddl::{AlterTableTask, EventContext};
use crate::rpc::ddl::AlterTableTask;
use crate::rpc::router::{Region, RegionRoute};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -121,8 +121,7 @@ async fn test_on_prepare_table_exists_err() {
.unwrap();
let task = test_rename_alter_table_task("non-exists", "foo");
let mut procedure =
AlterTableProcedure::new(1024, task, EventContext::default(), ddl_context).unwrap();
let mut procedure = AlterTableProcedure::new(1024, task, ddl_context).unwrap();
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err.status_code(), StatusCode::TableAlreadyExists);
}
@@ -132,8 +131,7 @@ async fn test_on_prepare_table_not_exists_err() {
let node_manager = Arc::new(MockDatanodeManager::new(()));
let ddl_context = new_ddl_context(node_manager);
let task = test_rename_alter_table_task("non-exists", "foo");
let mut procedure =
AlterTableProcedure::new(1024, task, EventContext::default(), ddl_context).unwrap();
let mut procedure = AlterTableProcedure::new(1024, task, ddl_context).unwrap();
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err.status_code(), StatusCode::TableNotFound);
}
@@ -247,13 +245,8 @@ async fn test_on_submit_alter_request() {
let alter_table_task = test_alter_table_task(table_name);
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure = AlterTableProcedure::new(
table_id,
alter_table_task,
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -306,13 +299,8 @@ async fn test_on_submit_alter_request_without_sync_request() {
let alter_table_task = test_alter_table_task(table_name);
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure = AlterTableProcedure::new(
table_id,
alter_table_task,
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -369,13 +357,7 @@ async fn test_on_submit_alter_request_with_outdated_request() {
};
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure = AlterTableProcedure::new(
table_id,
alter_table_task,
EventContext::default(),
ddl_context,
)
.unwrap();
let mut procedure = AlterTableProcedure::new(table_id, alter_table_task, ddl_context).unwrap();
procedure.on_prepare().await.unwrap();
let err = procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -404,9 +386,7 @@ async fn test_on_update_metadata_rename() {
.unwrap();
let task = test_rename_alter_table_task(table_name, new_table_name);
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone())
.unwrap();
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
procedure.on_update_metadata().await.unwrap();
@@ -486,9 +466,7 @@ async fn test_on_update_metadata_add_columns() {
};
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone())
.unwrap();
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -585,9 +563,7 @@ async fn test_on_update_table_options() {
};
let procedure_id = ProcedureId::random();
let provider = Arc::new(MockContextProvider::default());
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone())
.unwrap();
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
procedure
.submit_alter_region_requests(procedure_id, provider.as_ref())
@@ -658,7 +634,6 @@ async fn test_skip_wal_rejects_mixed_table_options() {
})),
},
},
EventContext::default(),
ddl_context,
)
.unwrap();
@@ -687,13 +662,8 @@ async fn test_skip_wal_rejects_logical_table() {
},
};
let mut procedure = AlterTableProcedure::new(
logical_table_id,
task,
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure =
AlterTableProcedure::new(logical_table_id, task, ddl_context.clone()).unwrap();
let error = procedure.on_prepare().await.unwrap_err();
assert_matches!(error, Error::Unsupported { .. });
@@ -739,9 +709,7 @@ async fn test_skip_wal_rejects_file_engine_table() {
},
};
let mut procedure =
AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone())
.unwrap();
let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
let error = procedure.on_prepare().await.unwrap_err();
assert_matches!(error, Error::Unsupported { .. });
@@ -778,7 +746,6 @@ fn test_skip_wal_holds_region_locks() {
let procedure = AlterTableProcedure::new_with_region_locks(
table_id,
task,
EventContext::default(),
region_ids.clone(),
context.clone(),
)
@@ -830,7 +797,6 @@ async fn test_skip_wal_detects_region_route_change() {
let mut procedure = AlterTableProcedure::new_with_region_locks(
table_id,
alter_task,
EventContext::default(),
stale_region_locks,
ddl_context.clone(),
)
@@ -950,7 +916,6 @@ async fn test_skip_wal_updates_metadata_before_all_replicas() {
let mut procedure = AlterTableProcedure::new_with_region_locks(
table_id,
alter_task,
EventContext::default(),
region_locks,
ddl_context.clone(),
)
@@ -1032,13 +997,8 @@ async fn prepare_alter_table_procedure(
},
};
let procedure_id = ProcedureId::random();
let mut procedure = AlterTableProcedure::new(
table_id,
alter_table_task,
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure =
AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
(procedure, procedure_id)
}
src/common/meta/src/ddl/tests/create_flow.rs
+16 -71
View File
@@ -37,7 +37,7 @@ use crate::key::flow::flow_info::{
FlowInfoValue, FlowMissedTickPolicy, FlowScheduleConfig, FlowStatus,
};
use crate::key::table_route::TableRouteValue;
use crate::rpc::ddl::{CreateFlowTask, EventContext, FlowQueryContext, QueryContext};
use crate::rpc::ddl::{CreateFlowTask, FlowQueryContext, QueryContext};
use crate::test_util::{MockFlownodeManager, new_ddl_context};
pub(crate) fn test_query_context() -> QueryContext {
@@ -120,8 +120,7 @@ async fn test_create_flow_source_table_not_found() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let query_ctx = test_query_context();
let mut procedure =
CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
assert!(
@@ -144,12 +143,7 @@ async fn test_create_pending_flow_source_table_not_found_with_defer() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
let status = procedure.on_prepare().await.unwrap();
assert_matches!(status, Status::Executing { persist: true, .. });
assert_eq!(procedure.data.unresolved_source_table_names.len(), 1);
@@ -189,8 +183,7 @@ async fn test_create_pending_flow_source_table_not_found_with_defer_false() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let query_ctx = test_query_context();
let mut procedure =
CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
assert!(
@@ -240,12 +233,7 @@ async fn test_create_pending_flow_records_partial_source_resolution() {
);
enable_defer_on_missing_source(&mut task);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
let status = procedure.on_prepare().await.unwrap();
assert_matches!(status, Status::Executing { persist: true, .. });
assert_eq!(procedure.data.source_table_ids, vec![existing_table_id]);
@@ -440,7 +428,6 @@ fn test_resolved_schedule_defaults_in_create_request() {
// Construct CreateFlowData and verify CreateRequest carries internal key.
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateFlows,
task,
flow_id: Some(1024),
@@ -454,7 +441,6 @@ fn test_resolved_schedule_defaults_in_create_request() {
};
let data2 = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateMetadata,
task: data.task.clone(),
flow_id: Some(1024),
@@ -594,8 +580,7 @@ async fn test_create_flow_rejects_unknown_option_in_meta_task() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let query_ctx = test_query_context();
let mut procedure =
CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unexpected { .. });
@@ -616,7 +601,6 @@ fn test_create_request_strips_defer_on_missing_source_runtime_option() {
enable_defer_on_missing_source(&mut task);
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateFlows,
task,
flow_id: Some(1024),
@@ -666,12 +650,7 @@ pub(crate) async fn create_test_flow(
false,
);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
task.clone(),
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context.clone());
let output = execute_procedure_until_done(&mut procedure).await.unwrap();
let flow_id = output.downcast_ref::<FlowId>().unwrap();
@@ -692,12 +671,7 @@ pub(crate) async fn create_test_pending_flow(
);
enable_defer_on_missing_source(&mut task);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
let output = execute_procedure_until_done(&mut procedure).await.unwrap();
let flow_id = output.downcast_ref::<FlowId>().unwrap();
@@ -744,12 +718,7 @@ async fn test_create_flow() {
true,
);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
task.clone(),
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context.clone());
let output = execute_procedure_until_done(&mut procedure).await.unwrap();
let flow_id = output.downcast_ref::<FlowId>().unwrap();
assert_eq!(*flow_id, 1024);
@@ -757,12 +726,7 @@ async fn test_create_flow() {
// Creates again
let task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false);
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
task.clone(),
query_ctx,
EventContext::default(),
ddl_context,
);
let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowAlreadyExists { .. });
}
@@ -819,12 +783,7 @@ async fn test_replace_pending_flow_with_active_flow_is_unsupported() {
);
replace_task.or_replace = true;
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
replace_task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
assert!(
@@ -882,12 +841,7 @@ async fn test_replace_active_flow_with_pending_flow_is_unsupported() {
enable_defer_on_missing_source(&mut replace_task);
replace_task.or_replace = true;
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
replace_task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
assert!(
@@ -947,12 +901,7 @@ async fn test_replace_pending_flow_with_pending_flow_updates_metadata() {
enable_defer_on_missing_source(&mut replace_task);
replace_task.or_replace = true;
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(
replace_task,
query_ctx,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
let output = execute_procedure_until_done(&mut procedure).await.unwrap();
let replaced_flow_id = *output.downcast_ref::<FlowId>().unwrap();
assert_eq!(replaced_flow_id, original_flow_id);
@@ -1004,8 +953,7 @@ async fn test_create_flow_same_source_and_sink_table() {
// Try to create a flow with same source and sink table - should fail
let task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false);
let query_ctx = test_query_context();
let mut procedure =
CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::Unsupported { .. });
@@ -1038,7 +986,7 @@ fn test_create_flow_lock_key_does_not_lock_sink_table_name() {
let task = create_test_flow_task_for_serialization();
let query_ctx = test_query_context();
let ddl_context = new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)));
let procedure = CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let lock_keys = procedure.lock_key().get_keys();
@@ -1113,7 +1061,6 @@ fn test_create_flow_data_new_format_serialization() {
};
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::Prepare,
task: create_test_flow_task_for_serialization(),
flow_id: None,
@@ -1182,7 +1129,7 @@ fn test_create_flow_procedure_strips_scheduled_time_extension() {
.extensions
.insert("flow.other".to_string(), "kept".to_string());
let procedure = CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context);
let procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
assert!(
!procedure
@@ -1215,7 +1162,6 @@ fn test_flow_info_conversion_with_flow_context() {
};
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateMetadata,
task: create_test_flow_task_for_serialization(),
flow_id: Some(123),
@@ -1258,7 +1204,6 @@ fn test_flow_info_conversion_strips_scheduled_time_extension() {
};
let data = CreateFlowData {
event_context: EventContext::default(),
state: CreateFlowState::CreateMetadata,
task: create_test_flow_task_for_serialization(),
flow_id: Some(123),
src/common/meta/src/ddl/tests/create_logical_tables.rs
+9 -30
View File
@@ -39,7 +39,6 @@ use crate::ddl::test_util::{
};
use crate::error::{Error, Result};
use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue};
use crate::rpc::ddl::EventContext;
use crate::rpc::router::{Region, RegionRoute};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -97,12 +96,7 @@ async fn test_on_prepare_physical_table_not_found() {
let ddl_context = new_ddl_context(node_manager);
let tasks = vec![test_create_logical_table_task("foo")];
let physical_table_id = 1024u32;
let mut procedure = CreateLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableRouteNotFound { .. });
}
@@ -132,12 +126,7 @@ async fn test_on_prepare() {
// The create logical table procedure.
let tasks = vec![test_create_logical_table_task("foo")];
let physical_table_id = table_id;
let mut procedure = CreateLogicalTablesProcedure::new(
tasks,
physical_table_id,
EventContext::default(),
ddl_context,
);
let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -183,12 +172,8 @@ async fn test_on_prepare_logical_table_exists_err() {
.unwrap();
// The create logical table procedure.
let physical_table_id = table_id;
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task],
physical_table_id,
EventContext::default(),
ddl_context,
);
let mut procedure =
CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
@@ -231,12 +216,8 @@ async fn test_on_prepare_with_create_if_table_exists() {
let physical_table_id = table_id;
// Sets `create_if_not_exists`
task.create_table.create_if_not_exists = true;
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task],
physical_table_id,
EventContext::default(),
ddl_context,
);
let mut procedure =
CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context);
let status = procedure.on_prepare().await.unwrap();
let output = status.downcast_output_ref::<Vec<u32>>().unwrap();
assert_eq!(*output, vec![8192]);
@@ -283,7 +264,6 @@ async fn test_on_prepare_part_logical_tables_exist() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task, non_exist_task],
physical_table_id,
EventContext::default(),
ddl_context,
);
let status = procedure.on_prepare().await.unwrap();
@@ -331,7 +311,6 @@ async fn test_on_create_metadata() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task, yet_another_task],
physical_table_id,
EventContext::default(),
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
@@ -345,6 +324,7 @@ async fn test_on_create_metadata() {
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
procedure.execute(&ctx).await.unwrap();
// Triggers procedure to create table metadata
@@ -425,7 +405,6 @@ async fn test_on_create_metadata_part_logical_tables_exist() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task, non_exist_task],
physical_table_id,
EventContext::default(),
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
@@ -439,6 +418,7 @@ async fn test_on_create_metadata_part_logical_tables_exist() {
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
procedure.execute(&ctx).await.unwrap();
// Triggers procedure to create table metadata
@@ -498,7 +478,6 @@ async fn test_on_create_metadata_err() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task.clone(), yet_another_task],
physical_table_id,
EventContext::default(),
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
@@ -512,6 +491,7 @@ async fn test_on_create_metadata_err() {
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
procedure.execute(&ctx).await.unwrap();
// Creates logical table metadata(different with the task)
@@ -559,7 +539,6 @@ async fn test_on_submit_create_request() {
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task, yet_another_task],
physical_table_id,
EventContext::default(),
ddl_context,
);
procedure.on_prepare().await.unwrap();
src/common/meta/src/ddl/tests/create_table.rs
+14 -64
View File
@@ -48,7 +48,7 @@ use crate::ddl::test_util::{assert_column_name, get_raw_table_info, put_datanode
use crate::error::{Error, Result};
use crate::key::table_route::TableRouteValue;
use crate::kv_backend::memory::MemoryKvBackend;
use crate::rpc::ddl::{CreateTableTask, EventContext, QueryContext};
use crate::rpc::ddl::CreateTableTask;
use crate::test_util::{MockDatanodeManager, new_ddl_context, new_ddl_context_with_kv_backend};
fn create_request_handler(_peer: Peer, request: RegionRequest) -> Result<RegionResponse> {
@@ -189,13 +189,7 @@ async fn test_on_prepare_table_exists_err() {
)
.await
.unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
@@ -218,13 +212,7 @@ async fn test_on_prepare_with_create_if_table_exists() {
)
.await
.unwrap();
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
let status = procedure.on_prepare().await.unwrap();
assert_matches!(status, Status::Done { output: Some(..) });
let table_id = *status.downcast_output_ref::<u32>().unwrap();
@@ -237,13 +225,7 @@ async fn test_on_prepare_without_create_if_table_exists() {
let ddl_context = new_ddl_context(node_manager);
let mut task = test_create_table_task("foo");
task.create_table.create_if_not_exists = true;
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -262,17 +244,12 @@ async fn test_on_datanode_create_regions_should_retry() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
assert!(!task.create_table.create_if_not_exists);
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
let error = procedure.execute(&ctx).await.unwrap_err();
assert!(error.is_retry_later());
@@ -285,17 +262,12 @@ async fn test_on_datanode_create_regions_should_not_retry() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
assert!(!task.create_table.create_if_not_exists);
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context,
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
let error = procedure.execute(&ctx).await.unwrap_err();
assert!(!error.is_retry_later());
@@ -308,13 +280,7 @@ async fn test_on_datanode_create_regions_remaps_addresses_when_retrying() {
let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
let table_route = procedure.data.table_route.as_mut().unwrap();
@@ -335,17 +301,12 @@ async fn test_on_create_metadata_error() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
assert!(!task.create_table.create_if_not_exists);
let mut procedure = CreateTableProcedure::new(
task.clone(),
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task.clone(), ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
procedure.execute(&ctx).await.unwrap();
let mut task = task;
@@ -374,17 +335,12 @@ async fn test_on_create_metadata() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_table_task("foo");
assert!(!task.create_table.create_if_not_exists);
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
procedure.execute(&ctx).await.unwrap();
// Triggers procedure to create table metadata
@@ -408,13 +364,7 @@ async fn test_memory_region_keeper_guard_dropped_on_procedure_done() {
let ddl_context = new_ddl_context_with_kv_backend(node_manager, kv_backend);
let task = test_create_table_task("foo");
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap();
execute_procedure_until(&mut procedure, |p| {
p.data.state == CreateTableState::CreateMetadata
src/common/meta/src/ddl/tests/create_view.rs
+14 -20
View File
@@ -29,7 +29,7 @@ use crate::ddl::create_view::CreateViewProcedure;
use crate::ddl::test_util::datanode_handler::NaiveDatanodeHandler;
use crate::ddl::tests::create_table::test_create_table_task;
use crate::error::Error;
use crate::rpc::ddl::{CreateViewTask, EventContext, QueryContext};
use crate::rpc::ddl::CreateViewTask;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
pub(crate) fn test_table_names() -> HashSet<table::table_name::TableName> {
@@ -112,7 +112,7 @@ async fn test_on_prepare_view_exists_err() {
)
.await
.unwrap();
let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = CreateViewProcedure::new(task, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::ViewAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
@@ -138,7 +138,7 @@ async fn test_on_prepare_with_create_if_view_exists() {
)
.await
.unwrap();
let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = CreateViewProcedure::new(task, ddl_context);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(status, Status::Done { output: Some(..) });
let table_id = *status.downcast_output_ref::<u32>().unwrap();
@@ -151,7 +151,7 @@ async fn test_on_prepare_without_create_if_table_exists() {
let ddl_context = new_ddl_context(node_manager);
let mut task = test_create_view_task("foo");
task.create_view.create_if_not_exists = true;
let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = CreateViewProcedure::new(task, ddl_context);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
@@ -170,11 +170,12 @@ async fn test_on_create_metadata() {
let ddl_context = new_ddl_context(node_manager);
let task = test_create_view_task("foo");
assert!(!task.create_view.create_if_not_exists);
let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = CreateViewProcedure::new(task, ddl_context);
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
// Triggers procedure to create view metadata
let status = procedure.execute(&ctx).await.unwrap();
@@ -189,12 +190,12 @@ async fn test_replace_view_metadata() {
let ddl_context = new_ddl_context(node_manager.clone());
let task = test_create_view_task("foo");
assert!(!task.create_view.create_if_not_exists);
let mut procedure =
CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone());
let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone());
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
// Triggers procedure to create view metadata
let status = procedure.execute(&ctx).await.unwrap();
@@ -215,8 +216,7 @@ async fn test_replace_view_metadata() {
let mut task = test_create_view_task("foo");
// The view already exists, prepare should fail
{
let mut procedure =
CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone());
let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone());
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::ViewAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
@@ -227,12 +227,12 @@ async fn test_replace_view_metadata() {
task.create_view.logical_plan = vec![4, 5, 6];
task.create_view.definition = "new_definition".to_string();
let mut procedure =
CreateViewProcedure::new(task, EventContext::default(), ddl_context.clone());
let mut procedure = CreateViewProcedure::new(task, ddl_context.clone());
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
// Triggers procedure to replace view metadata, but the view_id is unchanged.
let status = procedure.execute(&ctx).await.unwrap();
@@ -262,17 +262,12 @@ async fn test_replace_table() {
{
// Create a `foo` table.
let task = test_create_table_task("foo");
let mut procedure = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap();
procedure.on_prepare().await.unwrap();
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
procedure.execute(&ctx).await.unwrap();
procedure.execute(&ctx).await.unwrap();
@@ -281,8 +276,7 @@ async fn test_replace_table() {
// Try to replace a view named `foo` too.
let mut task = test_create_view_task("foo");
task.create_view.or_replace = true;
let mut procedure =
CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone());
let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone());
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableAlreadyExists { .. });
assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
src/common/meta/src/ddl/tests/drop_database.rs
+2 -4
View File
@@ -26,7 +26,6 @@ use crate::ddl::drop_database::executor::DropDatabaseExecutor;
use crate::ddl::test_util::datanode_handler::{NaiveDatanodeHandler, RetryErrorDatanodeHandler};
use crate::ddl::test_util::{create_logical_table, create_physical_table};
use crate::key::schema_name::SchemaNameKey;
use crate::rpc::ddl::EventContext;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
#[tokio::test]
@@ -55,13 +54,13 @@ async fn test_drop_database_with_logical_tables() {
DEFAULT_CATALOG_NAME.to_string(),
DEFAULT_SCHEMA_NAME.to_string(),
false,
EventContext::default(),
ddl_context.clone(),
);
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
while !procedure.execute(&ctx).await.unwrap().is_done() {
@@ -104,13 +103,13 @@ async fn test_drop_database_retryable_error() {
DEFAULT_CATALOG_NAME.to_string(),
DEFAULT_SCHEMA_NAME.to_string(),
false,
EventContext::default(),
ddl_context.clone(),
);
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
loop {
@@ -149,7 +148,6 @@ async fn test_drop_database_recover() {
DEFAULT_CATALOG_NAME.to_string(),
DEFAULT_SCHEMA_NAME.to_string(),
false,
EventContext::default(),
ddl_context.clone(),
);
let num_operating_regions = 1;
src/common/meta/src/ddl/tests/drop_flow.rs
+7 -7
View File
@@ -26,7 +26,7 @@ use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
use crate::ddl::tests::create_flow::{create_test_flow, create_test_pending_flow};
use crate::error;
use crate::key::table_route::TableRouteValue;
use crate::rpc::ddl::{DropFlowTask, EventContext};
use crate::rpc::ddl::DropFlowTask;
use crate::test_util::{MockFlownodeManager, new_ddl_context};
pub(crate) fn test_drop_flow_task(
@@ -48,7 +48,7 @@ async fn test_drop_flow_not_found() {
let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
let ddl_context = new_ddl_context(node_manager);
let task = test_drop_flow_task("my_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = DropFlowProcedure::new(task, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowNotFound { .. });
}
@@ -81,17 +81,17 @@ async fn test_drop_flow() {
create_test_flow(&ddl_context, "my_flow", source_table_names, sink_table_name).await;
// Drops the flows
let task = test_drop_flow_task("my_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone());
let mut procedure = DropFlowProcedure::new(task, ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
// Drops if not exists
let task = test_drop_flow_task("my_flow", flow_id, true);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone());
let mut procedure = DropFlowProcedure::new(task, ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
// Drops again
let task = test_drop_flow_task("my_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = DropFlowProcedure::new(task, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowNotFound { .. });
}
@@ -129,11 +129,11 @@ async fn test_drop_pending_flow_without_routes() {
assert!(flow_info.flownode_ids().is_empty());
let task = test_drop_flow_task("drop_pending_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone());
let mut procedure = DropFlowProcedure::new(task, ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
let task = test_drop_flow_task("drop_pending_flow", flow_id, false);
let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = DropFlowProcedure::new(task, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowNotFound { .. });
}
src/common/meta/src/ddl/tests/drop_table.rs
+54 -171
View File
@@ -65,9 +65,9 @@ use crate::key::table_route::TableRouteValue;
use crate::kv_backend::KvBackend;
use crate::kv_backend::memory::MemoryKvBackend;
use crate::peer::Peer;
use crate::rpc::ddl::{DropTableTask, EventContext};
use crate::rpc::ddl::DropTableTask;
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::{PurgeDroppedTableTask, QueryContext, UndropTableTask};
use crate::rpc::ddl::{PurgeDroppedTableTask, UndropTableTask};
use crate::rpc::router::{Region, RegionRoute};
#[cfg(feature = "enterprise")]
use crate::rpc::store::{BatchDeleteRequest, PutRequest};
@@ -128,11 +128,7 @@ async fn undrop_at_restore_metadata(
table_id: TableId,
) -> UndropTableProcedure {
create_dropped_table(context, table_id, Some(1), Some(i64::MAX)).await;
let mut undrop = UndropTableProcedure::new(
new_undrop_table_task(table_id),
context.clone(),
EventContext::default(),
);
let mut undrop = UndropTableProcedure::new(new_undrop_table_task(table_id), context.clone());
let procedure_context = new_test_procedure_context();
undrop.execute(&procedure_context).await.unwrap();
undrop.execute(&procedure_context).await.unwrap();
@@ -161,7 +157,6 @@ fn test_old_drop_table_json_defaults_to_hard_drop() {
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
runtime_context.clone(),
EventContext::default(),
);
let mut old_data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
old_data
@@ -169,7 +164,6 @@ fn test_old_drop_table_json_defaults_to_hard_drop() {
.unwrap()
.remove("soft_drop_enabled");
old_data.as_object_mut().unwrap().remove("dropped_at");
old_data.as_object_mut().unwrap().remove("event_context");
let recovered = DropTableProcedure::from_json(&old_data.to_string(), runtime_context).unwrap();
let recovered_data: serde_json::Value =
@@ -177,20 +171,12 @@ fn test_old_drop_table_json_defaults_to_hard_drop() {
assert_eq!(recovered_data["soft_drop_enabled"], false);
assert_eq!(recovered_data["dropped_at"], serde_json::Value::Null);
assert_eq!(
recovered_data["event_context"],
serde_json::json!({"reason": "unknown"})
);
}
#[test]
fn test_drop_table_lock_key_includes_table_name() {
let context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
context,
EventContext::default(),
);
let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context);
let keys = procedure
.lock_key()
@@ -219,11 +205,7 @@ fn test_new_drop_table_procedure_disables_soft_drop_without_enterprise() {
let mut context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
context.soft_drop_enabled = true;
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
context,
EventContext::default(),
);
let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context);
assert!(!procedure.data.soft_drop_enabled);
}
@@ -234,11 +216,7 @@ fn test_new_drop_table_procedure_preserves_soft_drop_with_enterprise() {
let mut context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
context.soft_drop_enabled = true;
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
context,
EventContext::default(),
);
let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context);
assert!(procedure.data.soft_drop_enabled);
}
@@ -273,7 +251,6 @@ async fn test_recovered_soft_drop_preserves_persisted_mode() {
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
soft_context.clone(),
EventContext::default(),
);
let hard_context = new_ddl_context_with_kv_backend(node_manager, kv_backend);
@@ -313,13 +290,9 @@ async fn test_legacy_soft_drop_prepare_preserves_persisted_mode() {
.unwrap();
let mut persisted: serde_json::Value = serde_json::from_str(
&DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
context.clone(),
EventContext::default(),
)
.dump()
.unwrap(),
&DropTableProcedure::new(new_drop_table_task("foo", table_id, false), context.clone())
.dump()
.unwrap(),
)
.unwrap();
persisted["soft_drop_enabled"] = true.into();
@@ -349,11 +322,7 @@ async fn test_disabled_soft_drop_creates_hard_drop_procedure() {
context.soft_drop_enabled = false;
context.soft_drop_retention = Some(Duration::from_millis(100));
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", 1024, false),
context,
EventContext::default(),
);
let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context);
assert!(!procedure.data.soft_drop_enabled);
assert_eq!(None, procedure.data.dropped_at);
@@ -387,7 +356,6 @@ async fn test_recovered_hard_drop_ignores_soft_runtime_context() {
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
hard_context.clone(),
EventContext::default(),
);
let mut soft_context = new_ddl_context_with_kv_backend(node_manager, kv_backend);
@@ -429,7 +397,7 @@ async fn test_on_prepare_table_not_exists_err() {
.unwrap();
let task = new_drop_table_task("bar", table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context);
assert_eq!(procedure.data.dropped_at, None);
assert_eq!(procedure.data.retention_expires_at, None);
let err = procedure.on_prepare().await.unwrap_err();
@@ -459,7 +427,6 @@ async fn test_soft_drop_prepare_assigns_stable_deadline_and_recovers_it() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
ddl_context.clone(),
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
@@ -496,7 +463,6 @@ async fn test_soft_drop_prepare_rejects_deadline_overflow_before_metadata_delete
let mut procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
ddl_context.clone(),
EventContext::default(),
);
let error = procedure.on_prepare().await.unwrap_err();
@@ -536,13 +502,13 @@ async fn test_on_prepare_table() {
let task = new_drop_table_task("bar", table_id, true);
// Drop if exists
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
procedure.on_prepare().await.unwrap();
assert!(!procedure.rollback_supported());
let task = new_drop_table_task(table_name, table_id, false);
// Drop table
let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context);
procedure.on_prepare().await.unwrap();
}
@@ -593,7 +559,7 @@ async fn test_on_datanode_drop_regions() {
let task = new_drop_table_task(table_name, table_id, false);
// Drop table
let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context);
procedure.on_prepare().await.unwrap();
procedure.on_datanode_drop_regions(false).await.unwrap();
@@ -662,7 +628,7 @@ async fn test_on_datanode_drop_regions_remaps_addresses_when_retrying() {
.unwrap();
let task = new_drop_table_task(table_name, table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
procedure.on_prepare().await.unwrap();
put_datanode_address(&ddl_context, 1, "new-leader").await;
@@ -721,7 +687,7 @@ async fn test_soft_drop_closes_regions_and_keeps_tombstone() {
.unwrap();
let task = new_drop_table_task(table_name, table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
@@ -835,7 +801,7 @@ async fn test_soft_drop_keeps_metadata_live_until_regions_close() {
.unwrap();
let task = new_drop_table_task(table_name, table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
put_datanode_address(&ddl_context, 1, "new-leader").await;
@@ -922,7 +888,6 @@ async fn test_soft_drop_timestamp_is_stable_across_retry_and_recovery() {
let procedure = DropTableProcedure::new(
new_drop_table_task("foo", table_id, false),
ddl_context.clone(),
EventContext::default(),
);
assert_eq!(procedure.data.dropped_at, None);
let mut procedure = procedure;
@@ -993,7 +958,7 @@ async fn test_hard_drop_keeps_delete_tombstone_flow() {
.unwrap();
let task = new_drop_table_task(table_name, table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
execute_procedure_until(&mut procedure, |p| {
p.data.state == DropTableState::DeleteTombstone
@@ -1036,20 +1001,13 @@ async fn test_create_table_succeeds_while_tombstone_exists() {
.unwrap();
let drop_task = new_drop_table_task(table_name, dropped_table_id, false);
let mut drop_procedure =
DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default());
let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone());
execute_procedure_until_done(&mut drop_procedure).await;
let mut create_task = test_create_table_task(table_name, 1025);
create_task.create_table.table_id = None;
create_task.table_info.ident.table_id = 0;
let mut create_procedure = CreateTableProcedure::new(
create_task,
QueryContext::default(),
EventContext::default(),
ddl_context.clone(),
)
.unwrap();
let mut create_procedure = CreateTableProcedure::new(create_task, ddl_context.clone()).unwrap();
execute_procedure_until_done(&mut create_procedure).await;
let live_table = ddl_context
@@ -1094,8 +1052,7 @@ async fn test_hard_drop_recreated_table_fails_when_soft_tombstone_exists() {
.unwrap();
let drop_task = new_drop_table_task(table_name, original_table_id, false);
let mut drop_procedure =
DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default());
let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone());
execute_procedure_until_done(&mut drop_procedure).await;
ddl_context.soft_drop_enabled = false;
@@ -1112,7 +1069,6 @@ async fn test_hard_drop_recreated_table_fails_when_soft_tombstone_exists() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, recreated_table_id, false),
ddl_context,
EventContext::default(),
);
let err = procedure.on_prepare().await.unwrap_err();
@@ -1142,8 +1098,7 @@ async fn test_hard_drop_recreated_table_ignores_previous_orphan_tombstone() {
.unwrap();
let drop_task = new_drop_table_task(table_name, original_table_id, false);
let mut drop_procedure =
DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default());
let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone());
execute_procedure_until(&mut drop_procedure, |p| {
p.data.state == DropTableState::DeleteTombstone
})
@@ -1162,7 +1117,6 @@ async fn test_hard_drop_recreated_table_ignores_previous_orphan_tombstone() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, recreated_table_id, false),
ddl_context,
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
@@ -1202,7 +1156,6 @@ async fn test_undrop_table_restores_metadata_and_reopens_regions() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
assert!(
@@ -1219,11 +1172,8 @@ async fn test_undrop_table_restores_metadata_and_reopens_regions() {
while rx.try_recv().is_ok() {}
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
let live_table = ddl_context
@@ -1314,16 +1264,12 @@ async fn test_undrop_table_opens_regions_before_restoring_live_metadata() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
while rx.try_recv().is_ok() {}
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
procedure.execute(&ctx).await.unwrap();
@@ -1387,17 +1333,13 @@ async fn test_undrop_logical_table_skips_datanode_open() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, logical_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
while rx.try_recv().is_ok() {}
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(logical_table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(logical_table_id), ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
let live_table = ddl_context
@@ -1428,7 +1370,6 @@ async fn test_soft_drop_metric_logical_table_falls_back_to_hard_drop() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task("foo", logical_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
assert!(!procedure.data.soft_drop_enabled);
@@ -1472,7 +1413,6 @@ async fn test_soft_drop_metric_physical_table_remains_enabled() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut procedure).await;
@@ -1517,7 +1457,6 @@ async fn test_soft_drop_file_engine_table_falls_back_to_hard_drop() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
assert!(!procedure.data.soft_drop_enabled);
@@ -1554,7 +1493,6 @@ async fn test_file_engine_fallback_is_resolved_before_tombstone_conflict() {
let mut procedure = DropTableProcedure::new(
new_drop_table_task(table_name, recreated_table_id, false),
ddl_context,
EventContext::default(),
);
procedure.on_prepare().await.unwrap();
@@ -1570,11 +1508,8 @@ async fn test_undrop_metric_logical_table_fails() {
let logical_table_id =
create_metric_logical_table_tombstone(&ddl_context, physical_table_id, "foo").await;
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(logical_table_id),
ddl_context,
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(logical_table_id), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_eq!(err.status_code(), StatusCode::Unsupported);
@@ -1592,7 +1527,6 @@ async fn test_purge_metric_logical_table_fails() {
let mut procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(logical_table_id),
ddl_context,
EventContext::default(),
);
let err = procedure
.execute(&new_test_procedure_context())
@@ -1624,7 +1558,6 @@ async fn test_undrop_table_fails_when_live_name_exists() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, dropped_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
ddl_context
@@ -1637,11 +1570,8 @@ async fn test_undrop_table_fails_when_live_name_exists() {
.await
.unwrap();
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(dropped_table_id),
ddl_context,
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, Error::TableAlreadyExists { .. });
@@ -1669,15 +1599,11 @@ async fn test_undrop_table_fails_when_live_name_is_created_after_prepare() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, dropped_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(dropped_table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context.clone());
procedure.on_prepare().await.unwrap();
ddl_context
.table_metadata_manager
@@ -1741,17 +1667,13 @@ async fn test_undrop_table_closes_opened_regions_when_restore_metadata_races_wit
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, dropped_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
while rx.try_recv().is_ok() {}
detector_controller.clear().await;
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(dropped_table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context.clone());
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
procedure.execute(&ctx).await.unwrap();
@@ -1845,7 +1767,6 @@ async fn test_undrop_table_lock_key_includes_original_table_name_before_prepare(
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
@@ -1860,7 +1781,6 @@ async fn test_undrop_table_lock_key_includes_original_table_name_before_prepare(
new_undrop_table_task(table_id),
ddl_context,
Some(original_table_name),
EventContext::default(),
);
let keys = procedure
@@ -1928,12 +1848,10 @@ async fn test_undrop_rejects_stale_id_after_name_tombstone_is_consumed() {
new_undrop_table_task(original_table_id),
ddl_context.clone(),
Some(original_table_name.clone()),
EventContext::default(),
);
let mut hard_drop = DropTableProcedure::new(
new_drop_table_task(table_name, recreated_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut hard_drop).await;
assert!(
@@ -1988,15 +1906,11 @@ async fn test_undrop_table_replayed_restore_metadata_is_idempotent() {
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
procedure.execute(&ctx).await.unwrap();
@@ -2100,7 +2014,6 @@ async fn test_purge_dropped_table_cleans_regions_offline_and_deletes_tombstone()
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
while rx.try_recv().is_ok() {}
@@ -2109,7 +2022,6 @@ async fn test_purge_dropped_table_cleans_regions_offline_and_deletes_tombstone()
let mut procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut procedure).await;
@@ -2188,7 +2100,6 @@ async fn test_automatic_purge_rechecks_unexpired_tombstone() {
let procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
data["check_expired"] = true.into();
@@ -2245,7 +2156,6 @@ async fn test_recovered_automatic_purge_rechecks_unexpired_tombstone() {
let procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
data["check_expired"] = true.into();
@@ -2323,7 +2233,6 @@ async fn test_recovered_automatic_purge_obeys_runtime_disable() {
let procedure = PurgeDroppedTableProcedure::new_if_expired(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut recovered =
PurgeDroppedTableProcedure::from_json(&procedure.dump().unwrap(), ddl_context.clone())
@@ -2355,7 +2264,6 @@ async fn test_recovered_automatic_purge_finishes_after_cleanup_is_claimed() {
let mut procedure = PurgeDroppedTableProcedure::new_if_expired(
new_purge_dropped_table_task(table_id),
enabled_context.clone(),
EventContext::default(),
);
procedure
.execute(&new_test_procedure_context())
@@ -2419,7 +2327,6 @@ async fn test_recovered_automatic_purge_ignores_foreign_generation_claim() {
let mut procedure = PurgeDroppedTableProcedure::new_if_expired(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
procedure
.execute(&new_test_procedure_context())
@@ -2462,11 +2369,8 @@ async fn test_undrop_rejects_table_claimed_for_purge() {
.await
.unwrap();
let mut procedure = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
let error = procedure.on_prepare().await.unwrap_err();
assert_eq!(StatusCode::TableNotFound, error.status_code());
@@ -2488,11 +2392,8 @@ async fn test_recovered_undrop_rejects_tombstone_deleted_by_purge() {
let table_id = 1024;
create_dropped_table(&ddl_context, table_id, Some(1), Some(i64::MAX)).await;
let mut undrop = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut undrop =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
undrop.on_prepare().await.unwrap();
let open_regions_state = undrop.dump().unwrap();
undrop.execute(&new_test_procedure_context()).await.unwrap();
@@ -2501,7 +2402,6 @@ async fn test_recovered_undrop_rejects_tombstone_deleted_by_purge() {
let mut purge = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut purge).await;
@@ -2533,11 +2433,8 @@ async fn test_recovered_undrop_rejects_replacement_generation() {
.await
.unwrap();
let mut undrop = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut undrop =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
undrop.on_prepare().await.unwrap();
let persisted = undrop.dump().unwrap();
kv_backend
@@ -2603,11 +2500,8 @@ async fn test_legacy_unmarked_tombstone_can_be_undropped_and_purged() {
assert_eq!(dropped_tables.len(), 1);
assert_eq!(dropped_tables[0].dropped_at, None);
let mut undrop = UndropTableProcedure::new(
new_undrop_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
let mut undrop =
UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone());
execute_procedure_until_done(&mut undrop).await;
ddl_context
@@ -2624,7 +2518,6 @@ async fn test_legacy_unmarked_tombstone_can_be_undropped_and_purged() {
let mut purge = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut purge).await;
assert!(
@@ -2667,7 +2560,6 @@ async fn test_purge_dropped_table_by_id_selects_tombstone_when_live_table_exists
let mut drop_procedure = DropTableProcedure::new(
new_drop_table_task(table_name, dropped_table_id, false),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut drop_procedure).await;
ddl_context
@@ -2684,7 +2576,6 @@ async fn test_purge_dropped_table_by_id_selects_tombstone_when_live_table_exists
let mut procedure = PurgeDroppedTableProcedure::new(
new_purge_dropped_table_task(dropped_table_id),
ddl_context.clone(),
EventContext::default(),
);
execute_procedure_until_done(&mut procedure).await;
@@ -2735,12 +2626,8 @@ async fn test_on_rollback() {
let physical_table_id = table_id;
// Creates the logical table metadata.
let task = test_create_logical_table_task("foo");
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task],
physical_table_id,
EventContext::default(),
ddl_context.clone(),
);
let mut procedure =
CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context.clone());
procedure.on_prepare().await.unwrap();
let ctx = new_test_procedure_context();
procedure.execute(&ctx).await.unwrap();
@@ -2753,8 +2640,7 @@ async fn test_on_rollback() {
// Drops the physical table
{
let task = new_drop_table_task("phy_table", physical_table_id, false);
let mut procedure =
DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
procedure.on_prepare().await.unwrap();
assert!(procedure.rollback_supported());
procedure.on_delete_metadata().await.unwrap();
@@ -2765,8 +2651,7 @@ async fn test_on_rollback() {
// The physical table can be dropped again after the metadata rollback.
let retry_task = new_drop_table_task("phy_table", physical_table_id, false);
let mut retry =
DropTableProcedure::new(retry_task, ddl_context.clone(), EventContext::default());
let mut retry = DropTableProcedure::new(retry_task, ddl_context.clone());
retry.on_prepare().await.unwrap();
retry.on_delete_metadata().await.unwrap();
retry.rollback(&ctx).await.unwrap();
@@ -2782,7 +2667,7 @@ async fn test_on_rollback() {
// Drops the logical table
ddl_context.soft_drop_enabled = false;
let task = new_drop_table_task("foo", table_ids[0], false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
procedure.on_prepare().await.unwrap();
assert!(!procedure.rollback_supported());
}
@@ -2879,8 +2764,7 @@ async fn test_memory_region_keeper_guard_dropped_on_procedure_done() {
let logical_table_id = create_logical_table(ddl_context.clone(), physical_table_id, "s").await;
let inner_test = |task: DropTableTask| async {
let mut procedure =
DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
execute_procedure_until(&mut procedure, |p| {
p.data.state == DropTableState::InvalidateTableCache
})
@@ -2932,8 +2816,7 @@ async fn test_from_json() {
let physical_table_id = create_physical_table(&ddl_context, "t").await;
let task = new_drop_table_task("t", physical_table_id, false);
let mut procedure =
DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
execute_procedure_until(&mut procedure, |p| p.data.state == state).await;
let data = procedure.dump().unwrap();
assert_eq!(
@@ -2962,7 +2845,7 @@ async fn test_from_json() {
let physical_table_id = create_physical_table(&ddl_context, "t").await;
let task = new_drop_table_task("t", physical_table_id, false);
let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default());
let mut procedure = DropTableProcedure::new(task, ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
let data = procedure.dump().unwrap();
assert_eq!(
src/common/meta/src/ddl/tests/drop_view.rs
+7 -7
View File
@@ -24,7 +24,7 @@ use crate::ddl::drop_view::{DropViewProcedure, DropViewState};
use crate::ddl::test_util::create_table::test_create_table_task;
use crate::ddl::tests::create_view::{test_create_view_task, test_table_names};
use crate::key::table_route::TableRouteValue;
use crate::rpc::ddl::{DropViewTask, EventContext};
use crate::rpc::ddl::DropViewTask;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
pub(crate) fn new_drop_view_task(
@@ -63,7 +63,7 @@ async fn test_on_prepare_view_not_exists_err() {
.unwrap();
let task = new_drop_view_task("bar", view_id, false);
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = DropViewProcedure::new(task, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_eq!(err.status_code(), StatusCode::TableNotFound);
}
@@ -87,7 +87,7 @@ async fn test_on_prepare_not_view_err() {
.unwrap();
let task = new_drop_view_task(view_name, view_id, false);
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = DropViewProcedure::new(task, ddl_context);
// It's not a view, expect error
let err = procedure.on_prepare().await.unwrap_err();
assert_eq!(err.status_code(), StatusCode::InvalidArguments);
@@ -117,12 +117,12 @@ async fn test_on_prepare_success() {
let task = new_drop_view_task("bar", view_id, true);
// Drop if exists
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context.clone());
let mut procedure = DropViewProcedure::new(task, ddl_context.clone());
procedure.on_prepare().await.unwrap();
let task = new_drop_view_task(view_name, view_id, false);
// Prepare success
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = DropViewProcedure::new(task, ddl_context);
procedure.on_prepare().await.unwrap();
assert_eq!(DropViewState::DeleteMetadata, procedure.state());
}
@@ -161,7 +161,7 @@ async fn test_drop_view_success() {
let task = new_drop_view_task(view_name, view_id, false);
// Prepare success
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context.clone());
let mut procedure = DropViewProcedure::new(task, ddl_context.clone());
execute_procedure_until_done(&mut procedure).await;
assert_eq!(DropViewState::InvalidateViewCache, procedure.state());
@@ -178,7 +178,7 @@ async fn test_drop_view_success() {
// Drop again
let task = new_drop_view_task(view_name, view_id, false);
let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context);
let mut procedure = DropViewProcedure::new(task, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
assert_eq!(err.status_code(), StatusCode::TableNotFound);
}
src/common/meta/src/ddl/tests/event/database.rs
+8 -40
View File
@@ -25,12 +25,12 @@ use common_event_recorder::event_table::{
PROCEDURE_ID_COLUMN as EVENT_TABLE_PROCEDURE_ID_COLUMN,
PROCEDURE_STATE_COLUMN as EVENT_TABLE_PROCEDURE_STATE_COLUMN,
PROCEDURE_TRIGGER_COLUMN as EVENT_TABLE_PROCEDURE_TRIGGER_COLUMN,
SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN,
SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, jsonb_value,
};
use common_event_recorder::testing::assert_event_contract;
use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState};
use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value};
use super::test_util::assert_event_filter;
use crate::ddl::alter_database::AlterDatabaseProcedure;
use crate::ddl::create_database::CreateDatabaseProcedure;
use crate::ddl::drop_database::DropDatabaseProcedure;
@@ -39,7 +39,7 @@ use crate::ddl::event::database::{
DatabaseDdlEvent,
};
use crate::rpc::ddl::{
AlterDatabaseKind, EventContext, SetDatabaseOption, SetDatabaseOptions, UnsetDatabaseOption,
AlterDatabaseKind, SetDatabaseOption, SetDatabaseOptions, UnsetDatabaseOption,
UnsetDatabaseOptions,
};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -50,13 +50,7 @@ fn test_create_database_submitted_event_contract() {
("password".to_string(), "do-not-record".to_string()),
("compaction.type".to_string(), "twcs".to_string()),
]);
let event = DatabaseDdlEvent::create_submitted(
"greptime",
"metrics",
true,
&options,
EventContext::default(),
);
let event = DatabaseDdlEvent::create_submitted("greptime", "metrics", true, &options);
assert_event_locator(
&event,
@@ -93,7 +87,6 @@ fn test_alter_database_set_and_unset_event_contracts() {
SetDatabaseOption::Other("secret_token".to_string(), "hidden".to_string()),
SetDatabaseOption::Ttl(std::time::Duration::from_secs(3600).into()),
])),
EventContext::default(),
);
assert_event_locator(
&set,
@@ -120,7 +113,6 @@ fn test_alter_database_set_and_unset_event_contracts() {
UnsetDatabaseOption::Ttl,
UnsetDatabaseOption::Other("compaction.type".to_string()),
])),
EventContext::default(),
);
assert_event_locator(
&unset,
@@ -140,8 +132,7 @@ fn test_alter_database_set_and_unset_event_contracts() {
#[test]
fn test_drop_database_submitted_event_contract() {
let event =
DatabaseDdlEvent::drop_submitted("greptime", "metrics", true, EventContext::default());
let event = DatabaseDdlEvent::drop_submitted("greptime", "metrics", true);
assert_event_locator(
&event,
@@ -177,14 +168,7 @@ fn test_database_lifecycle_events_preserve_locator_and_null_intent() {
assert_eq!(
DatabaseDdlEvent::create_lifecycle("greptime", "metrics").extra_schema(),
DatabaseDdlEvent::create_submitted(
"c",
"s",
false,
&HashMap::new(),
EventContext::default(),
)
.extra_schema()
DatabaseDdlEvent::create_submitted("c", "s", false, &HashMap::new()).extra_schema()
);
}
@@ -198,7 +182,6 @@ fn test_database_events_preserve_procedure_envelope_contract() {
"metrics",
false,
&HashMap::new(),
EventContext::default(),
)),
ProcedureState::Running,
EventTrigger::Submitted,
@@ -237,7 +220,6 @@ fn test_create_database_event_filter() {
false,
HashMap::new(),
None,
EventContext::default(),
new_ddl_context(Arc::new(MockDatanodeManager::new(()))),
);
@@ -258,7 +240,6 @@ fn test_alter_database_event_filter() {
)),
},
},
EventContext::default(),
new_ddl_context(Arc::new(MockDatanodeManager::new(()))),
)
.unwrap();
@@ -272,7 +253,6 @@ fn test_drop_database_event_filter() {
"greptime".to_string(),
"metrics".to_string(),
false,
EventContext::default(),
new_ddl_context(Arc::new(MockDatanodeManager::new(()))),
);
@@ -285,14 +265,12 @@ fn assert_event_locator(
catalog_name: Option<&str>,
schema_name: Option<&str>,
) {
let has_event_context = !event.json_payload().unwrap().is_null();
assert_event_contract(
event,
event_type,
&[
EVENT_TABLE_CATALOG_NAME_COLUMN.column_schema(),
EVENT_TABLE_SCHEMA_NAME_COLUMN.column_schema(),
EVENT_CONTEXT_COLUMN.column_schema(),
],
&[Row {
values: vec![
@@ -302,11 +280,6 @@ fn assert_event_locator(
Value {
value_data: schema_name.map(|value| ValueData::StringValue(value.to_string())),
},
if has_event_context {
default_event_context_value()
} else {
Value { value_data: None }
},
],
}],
);
@@ -320,7 +293,6 @@ fn assert_procedure_event_contract(
catalog_name: Option<&str>,
schema_name: Option<&str>,
) {
let has_event_context = procedure_trigger == "Submitted";
assert_event_contract(
event,
event_type,
@@ -346,18 +318,14 @@ fn assert_procedure_event_contract(
Value {
value_data: Some(ValueData::StringValue(String::new())),
},
procedure_trigger_value(procedure_trigger),
jsonb_value(&serde_json::json!({"type": procedure_trigger})),
Value {
value_data: catalog_name.map(|value| ValueData::StringValue(value.to_string())),
},
Value {
value_data: schema_name.map(|value| ValueData::StringValue(value.to_string())),
},
if has_event_context {
default_event_context_value()
} else {
Value { value_data: None }
},
Value { value_data: None },
],
}],
);
src/common/meta/src/ddl/tests/event/flow.rs
+6 -19
View File
@@ -21,12 +21,13 @@ use common_event_recorder::Event;
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN,
PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN,
jsonb_value,
};
use common_event_recorder::testing::assert_event_contract;
use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState};
use table::table_name::TableName;
use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value};
use super::test_util::assert_event_filter;
use crate::ddl::create_flow::CreateFlowProcedure;
use crate::ddl::drop_flow::DropFlowProcedure;
use crate::ddl::event::flow::{
@@ -35,7 +36,6 @@ use crate::ddl::event::flow::{
use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
use crate::ddl::tests::create_flow::{test_create_flow_task, test_query_context};
use crate::ddl::tests::drop_flow::test_drop_flow_task;
use crate::rpc::ddl::EventContext;
use crate::test_util::{MockFlownodeManager, new_ddl_context};
#[test]
@@ -49,7 +49,6 @@ fn test_flow_submitted_event_contracts() {
expire_after: Some(300),
eval_interval_secs: Some(60),
},
EventContext::default(),
);
assert_event_contract(
&create,
@@ -60,7 +59,6 @@ fn test_flow_submitted_event_contracts() {
ValueData::StringValue("greptime".to_string()).into(),
ValueData::StringValue("metrics".to_string()).into(),
Value { value_data: None },
default_event_context_value(),
],
}],
);
@@ -75,8 +73,7 @@ fn test_flow_submitted_event_contracts() {
})
);
let drop =
FlowDdlEvent::drop_submitted("greptime", "metrics", 42, true, EventContext::default());
let drop = FlowDdlEvent::drop_submitted("greptime", "metrics", 42, true);
assert_event_contract(
&drop,
DROP_FLOW_EVENT_TYPE,
@@ -86,7 +83,6 @@ fn test_flow_submitted_event_contracts() {
ValueData::StringValue("greptime".to_string()).into(),
ValueData::StringValue("metrics".to_string()).into(),
ValueData::U32Value(42).into(),
default_event_context_value(),
],
}],
);
@@ -125,7 +121,6 @@ fn test_flow_lifecycle_events_have_fixed_schema_and_null_intent() {
} else {
Value { value_data: None }
},
Value { value_data: None },
],
}],
);
@@ -142,7 +137,6 @@ fn test_flow_lifecycle_events_have_fixed_schema_and_null_intent() {
ValueData::StringValue("greptime".to_string()).into(),
ValueData::StringValue("metrics".to_string()).into(),
ValueData::U32Value(42).into(),
Value { value_data: None },
],
}],
);
@@ -162,7 +156,6 @@ fn test_flow_events_preserve_procedure_envelope_contract() {
expire_after: None,
eval_interval_secs: None,
},
EventContext::default(),
)),
ProcedureState::Running,
EventTrigger::Submitted,
@@ -212,7 +205,6 @@ fn test_create_flow_event_filter() {
false,
),
test_query_context(),
EventContext::default(),
new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))),
);
assert_event_filter(&procedure, CREATE_FLOW_EVENT_TYPE);
@@ -222,7 +214,6 @@ fn test_create_flow_event_filter() {
fn test_drop_flow_event_filter() {
let procedure = DropFlowProcedure::new(
test_drop_flow_task("flow", 42, false),
EventContext::default(),
new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))),
);
assert_event_filter(&procedure, DROP_FLOW_EVENT_TYPE);
@@ -233,7 +224,6 @@ fn flow_schema() -> Vec<ColumnSchema> {
CATALOG_NAME_COLUMN.column_schema(),
FLOW_NAME_COLUMN.column_schema(),
FLOW_ID_COLUMN.column_schema(),
EVENT_CONTEXT_COLUMN.column_schema(),
]
}
@@ -257,6 +247,7 @@ fn assert_procedure_event_contract(
PROCEDURE_TRIGGER_COLUMN.column_schema(),
];
schema.extend(flow_schema());
schema.push(EVENT_CONTEXT_COLUMN.column_schema());
assert_event_contract(
event,
event_type,
@@ -266,7 +257,7 @@ fn assert_procedure_event_contract(
ValueData::StringValue(event.procedure_id.to_string()).into(),
ValueData::StringValue(state.to_string()).into(),
ValueData::StringValue(String::new()).into(),
procedure_trigger_value(trigger),
jsonb_value(&serde_json::json!({"type": trigger})),
optional_string(locator.catalog_name),
optional_string(locator.flow_name),
locator
@@ -274,11 +265,7 @@ fn assert_procedure_event_contract(
.map(ValueData::U32Value)
.map(Into::into)
.unwrap_or(Value { value_data: None }),
if trigger == "Submitted" {
default_event_context_value()
} else {
Value { value_data: None }
},
Value { value_data: None },
],
}],
);
src/common/meta/src/ddl/tests/event/table.rs
+41 -125
View File
@@ -19,13 +19,13 @@ use api::v1::value::ValueData;
use api::v1::{ColumnDataType, Repartition, SemanticType, Value};
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN,
TABLE_ID_COLUMN, TABLE_NAME_COLUMN, jsonb_value,
CATALOG_NAME_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN,
TABLE_NAME_COLUMN,
};
use common_event_recorder::{Event, EventTypeFilter};
use common_procedure::{
ChildSubmissionOutcome, EventRuntimeContext, EventTrigger, Procedure, ProcedureId,
ProcedureState, RetryPhase,
ChildSubmissionOutcome, EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState,
RetryPhase,
};
use common_time::Timestamp;
use serde_json::{Value as JsonValue, json};
@@ -52,9 +52,9 @@ use crate::ddl::truncate_table::TruncateTableProcedure;
use crate::ddl::undrop_table::UndropTableProcedure;
use crate::key::DeserializedValueWithBytes;
use crate::key::table_info::TableInfoValue;
use crate::rpc::ddl::{DropTableTask, EventContext, QueryContext, TruncateTableTask};
use crate::rpc::ddl::{DropTableTask, TruncateTableTask};
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::{PurgeDroppedTableTask, TriggerReason, UndropTableTask};
use crate::rpc::ddl::{PurgeDroppedTableTask, UndropTableTask};
use crate::test_util::{MockDatanodeManager, new_ddl_context};
struct EventCase {
@@ -175,11 +175,7 @@ fn later_lifecycle_events_are_uniform() {
.extra_rows()
.unwrap()
.into_iter()
.map(|mut row| {
// Lifecycle events retain locators but not submitted event context.
*row.values.last_mut().unwrap() = Value::default();
row.values
})
.map(|row| row.values)
.collect::<Vec<_>>();
for trigger in &triggers {
@@ -205,13 +201,7 @@ fn later_lifecycle_events_are_uniform() {
fn create_success_events_keep_allocated_ids() {
let mut task = test_create_table_task("create_success");
task.table_info.ident.table_id = 7;
let create_table = CreateTableProcedure::new(
task,
QueryContext::default(),
EventContext::default(),
test_context(),
)
.unwrap();
let create_table = CreateTableProcedure::new(task, test_context()).unwrap();
let state = ProcedureState::Done {
output: Some(Arc::new(42_u32)),
};
@@ -230,7 +220,6 @@ fn create_success_events_keep_allocated_ids() {
test_create_logical_table_task("bar"),
],
1024,
EventContext::default(),
test_context(),
);
let state = ProcedureState::Done {
@@ -269,14 +258,13 @@ fn event_cases() -> Vec<EventCase> {
TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "create"),
true,
"mito2",
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"create_if_not_exists": true,
"engine": "mito2",
}),
rows: vec![submitted_table_locator_values(Some("create"), None)],
rows: vec![table_locator_values(Some("create"), None)],
},
EventCase {
event_type: TableDdlEventType::CreateLogicalTables,
@@ -288,15 +276,14 @@ fn event_cases() -> Vec<EventCase> {
.with_physical_table_id(10),
],
2,
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"table_count": 2,
}),
rows: vec![
submitted_logical_locator_values("logical1", None, 10),
submitted_logical_locator_values("logical2", None, 10),
logical_locator_values("logical1", None, 10),
logical_locator_values("logical2", None, 10),
],
},
EventCase {
@@ -305,13 +292,12 @@ fn event_cases() -> Vec<EventCase> {
TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "alter")
.with_table_id(11),
Some("drop_columns"),
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"kind": "drop_columns",
}),
rows: vec![submitted_table_locator_values(Some("alter"), Some(11))],
rows: vec![table_locator_values(Some("alter"), Some(11))],
},
EventCase {
event_type: TableDdlEventType::AlterLogicalTables,
@@ -324,7 +310,6 @@ fn event_cases() -> Vec<EventCase> {
],
2,
["rename_table", "add_columns", "add_columns"],
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
@@ -332,8 +317,8 @@ fn event_cases() -> Vec<EventCase> {
"kinds": ["add_columns", "rename_table"],
}),
rows: vec![
submitted_logical_locator_values("logical1", None, 10),
submitted_logical_locator_values("logical2", None, 10),
logical_locator_values("logical1", None, 10),
logical_locator_values("logical2", None, 10),
],
},
EventCase {
@@ -342,33 +327,26 @@ fn event_cases() -> Vec<EventCase> {
TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "drop")
.with_table_id(12),
true,
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"drop_if_exists": true,
}),
rows: vec![submitted_table_locator_values(Some("drop"), Some(12))],
rows: vec![table_locator_values(Some("drop"), Some(12))],
},
#[cfg(feature = "enterprise")]
EventCase {
event_type: TableDdlEventType::UndropTable,
event: TableDdlEvent::undrop_table_submitted(
TableDdlLocator::from_table_id(13),
EventContext::default(),
),
event: TableDdlEvent::undrop_table_submitted(TableDdlLocator::from_table_id(13)),
payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}),
rows: vec![submitted_table_locator_values(None, Some(13))],
rows: vec![table_locator_values(None, Some(13))],
},
#[cfg(feature = "enterprise")]
EventCase {
event_type: TableDdlEventType::PurgeDroppedTable,
event: TableDdlEvent::purge_dropped_table_submitted(
TableDdlLocator::from_table_id(14),
EventContext::default(),
),
event: TableDdlEvent::purge_dropped_table_submitted(TableDdlLocator::from_table_id(14)),
payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}),
rows: vec![submitted_table_locator_values(None, Some(14))],
rows: vec![table_locator_values(None, Some(14))],
},
EventCase {
event_type: TableDdlEventType::TruncateTable,
@@ -376,41 +354,29 @@ fn event_cases() -> Vec<EventCase> {
TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "truncate")
.with_table_id(15),
4,
EventContext::default(),
),
payload: json!({
"version": TABLE_DDL_PAYLOAD_VERSION,
"time_range_count": 4,
}),
rows: vec![submitted_table_locator_values(Some("truncate"), Some(15))],
rows: vec![table_locator_values(Some("truncate"), Some(15))],
},
]
}
fn procedure_cases() -> Vec<ProcedureCase> {
let create_table = CreateTableProcedure::new(
test_create_table_task("create"),
QueryContext::default(),
EventContext::default(),
test_context(),
)
.unwrap();
let create_table =
CreateTableProcedure::new(test_create_table_task("create"), test_context()).unwrap();
let create_logical_tables = CreateLogicalTablesProcedure::new(
vec![
test_create_logical_table_task("logical1"),
test_create_logical_table_task("logical2"),
],
41,
EventContext::default(),
test_context(),
);
let alter_table = AlterTableProcedure::new(
42,
test_alter_table_task("alter"),
EventContext::default(),
test_context(),
)
.unwrap();
let alter_table =
AlterTableProcedure::new(42, test_alter_table_task("alter"), test_context()).unwrap();
let alter_logical_tables = AlterLogicalTablesProcedure::new(
vec![
make_alter_logical_table_add_column_task(
@@ -425,7 +391,6 @@ fn procedure_cases() -> Vec<ProcedureCase> {
),
],
43,
EventContext::default(),
test_context(),
);
let drop_table = DropTableProcedure::new(
@@ -437,20 +402,12 @@ fn procedure_cases() -> Vec<ProcedureCase> {
drop_if_exists: true,
},
test_context(),
EventContext::default(),
);
#[cfg(feature = "enterprise")]
let undrop_table = UndropTableProcedure::new(
UndropTableTask { table_id: 45 },
test_context(),
EventContext::default(),
);
let undrop_table = UndropTableProcedure::new(UndropTableTask { table_id: 45 }, test_context());
#[cfg(feature = "enterprise")]
let purge_dropped_table = PurgeDroppedTableProcedure::new_if_expired(
PurgeDroppedTableTask { table_id: 46 },
test_context(),
EventContext::new(TriggerReason::ScheduledGc),
);
let purge_dropped_table =
PurgeDroppedTableProcedure::new(PurgeDroppedTableTask { table_id: 46 }, test_context());
let truncate_table = truncate_procedure(TruncateTableTask {
catalog: DEFAULT_CATALOG_NAME.to_string(),
schema: DEFAULT_SCHEMA_NAME.to_string(),
@@ -471,7 +428,7 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"create_if_not_exists": false,
"engine": "mito2",
}),
rows: vec![submitted_table_locator_values(Some("create"), None)],
rows: vec![table_locator_values(Some("create"), None)],
},
ProcedureCase {
procedure: Box::new(create_logical_tables),
@@ -481,8 +438,8 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"table_count": 2,
}),
rows: vec![
submitted_logical_locator_values("logical1", None, 41),
submitted_logical_locator_values("logical2", None, 41),
logical_locator_values("logical1", None, 41),
logical_locator_values("logical2", None, 41),
],
},
ProcedureCase {
@@ -492,7 +449,7 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"version": TABLE_DDL_PAYLOAD_VERSION,
"kind": "drop_columns",
}),
rows: vec![submitted_table_locator_values(Some("alter"), Some(42))],
rows: vec![table_locator_values(Some("alter"), Some(42))],
},
ProcedureCase {
procedure: Box::new(alter_logical_tables),
@@ -503,8 +460,8 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"kinds": ["add_columns"],
}),
rows: vec![
submitted_logical_locator_values("logical1", None, 43),
submitted_logical_locator_values("logical2", None, 43),
logical_locator_values("logical1", None, 43),
logical_locator_values("logical2", None, 43),
],
},
ProcedureCase {
@@ -514,25 +471,21 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"version": TABLE_DDL_PAYLOAD_VERSION,
"drop_if_exists": true,
}),
rows: vec![submitted_table_locator_values(Some("drop"), Some(44))],
rows: vec![table_locator_values(Some("drop"), Some(44))],
},
#[cfg(feature = "enterprise")]
ProcedureCase {
procedure: Box::new(undrop_table),
event_type: "undrop_table",
payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}),
rows: vec![submitted_table_locator_values(None, Some(45))],
rows: vec![table_locator_values(None, Some(45))],
},
#[cfg(feature = "enterprise")]
ProcedureCase {
procedure: Box::new(purge_dropped_table),
event_type: "purge_dropped_table",
payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}),
rows: vec![submitted_table_locator_values_with_event_context(
None,
Some(46),
EventContext::new(TriggerReason::ScheduledGc),
)],
rows: vec![table_locator_values(None, Some(46))],
},
ProcedureCase {
procedure: Box::new(truncate_table),
@@ -541,7 +494,7 @@ fn procedure_cases() -> Vec<ProcedureCase> {
"version": TABLE_DDL_PAYLOAD_VERSION,
"time_range_count": 1,
}),
rows: vec![submitted_table_locator_values(Some("truncate"), Some(47))],
rows: vec![table_locator_values(Some("truncate"), Some(47))],
},
]
}
@@ -562,7 +515,6 @@ fn expected_schema(event_type: TableDdlEventType) -> Vec<(&'static str, i32)> {
ColumnDataType::Uint32 as i32,
));
}
schema.push((EVENT_CONTEXT_COLUMN.name(), ColumnDataType::Binary as i32));
schema
}
@@ -575,14 +527,12 @@ fn table_locator_values(table_name: Option<&str>, table_id: Option<u32>) -> Vec<
} else {
(Value::default(), Value::default())
};
let mut values = vec![
vec![
catalog_name,
schema_name,
table_name.map(string_value).unwrap_or_default(),
table_id.map(table_id_value).unwrap_or_default(),
];
values.push(Value::default());
values
]
}
fn logical_locator_values(
@@ -591,44 +541,10 @@ fn logical_locator_values(
physical_table_id: u32,
) -> Vec<Value> {
let mut values = table_locator_values(Some(table_name), table_id);
let event_context = values.pop().unwrap();
values.push(table_id_value(physical_table_id));
values.push(event_context);
values
}
fn submitted_table_locator_values(table_name: Option<&str>, table_id: Option<u32>) -> Vec<Value> {
submitted_table_locator_values_with_event_context(table_name, table_id, EventContext::default())
}
fn submitted_table_locator_values_with_event_context(
table_name: Option<&str>,
table_id: Option<u32>,
event_context: EventContext,
) -> Vec<Value> {
let mut values = table_locator_values(table_name, table_id);
*values.last_mut().unwrap() = event_context_value_for(event_context);
values
}
fn submitted_logical_locator_values(
table_name: &str,
table_id: Option<u32>,
physical_table_id: u32,
) -> Vec<Value> {
let mut values = logical_locator_values(table_name, table_id, physical_table_id);
*values.last_mut().unwrap() = event_context_value();
values
}
fn event_context_value() -> Value {
event_context_value_for(EventContext::default())
}
fn event_context_value_for(event_context: EventContext) -> Value {
jsonb_value(&serde_json::to_value(event_context).unwrap())
}
fn string_value(value: &str) -> Value {
ValueData::StringValue(value.to_string()).into()
}
@@ -643,7 +559,6 @@ fn truncate_procedure(task: TruncateTableTask) -> TruncateTableProcedure {
task,
DeserializedValueWithBytes::from_inner(TableInfoValue::new(table_info)),
test_context(),
EventContext::default(),
)
}
@@ -661,11 +576,12 @@ fn event_for_state(
lifecycle_state: &ProcedureState,
) -> Box<dyn Event> {
procedure
.event(&EventRuntimeContext {
.event(&EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state,
trigger,
event_type_filter: Arc::new(EventTypeFilter::All),
event_context: None,
})
.unwrap()
}
src/common/meta/src/ddl/tests/event/test_util.rs
+5 -16
View File
@@ -15,32 +15,21 @@
use std::collections::HashSet;
use std::sync::Arc;
use api::v1::Value;
use common_event_recorder::EventTypeFilter;
use common_event_recorder::event_table::jsonb_value;
use common_procedure::{EventRuntimeContext, EventTrigger, Procedure, ProcedureId, ProcedureState};
use crate::rpc::ddl::EventContext;
pub(crate) fn default_event_context_value() -> Value {
jsonb_value(&serde_json::to_value(EventContext::default()).unwrap())
}
pub(crate) fn procedure_trigger_value(trigger: &str) -> Value {
jsonb_value(&serde_json::json!({"type": trigger}))
}
use common_procedure::{EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState};
pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) {
let state = ProcedureState::Running;
let runtime_context = |event_type_filter| EventRuntimeContext {
let event_context = |event_type_filter| EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
event_type_filter: Arc::new(event_type_filter),
event_context: None,
};
let allowed = procedure
.event(&runtime_context(EventTypeFilter::Only(HashSet::from([
.event(&event_context(EventTypeFilter::Only(HashSet::from([
event_type.to_string(),
]))))
.unwrap();
@@ -49,7 +38,7 @@ pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) {
for denied in [HashSet::from(["other_event".to_string()]), HashSet::new()] {
assert!(
procedure
.event(&runtime_context(EventTypeFilter::Only(denied)))
.event(&event_context(EventTypeFilter::Only(denied)))
.is_none()
);
}
src/common/meta/src/ddl/tests/event/view.rs
+17 -50
View File
@@ -19,16 +19,16 @@ use api::v1::{ColumnSchema, Row, Value};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN,
PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN,
VIEW_NAME_COLUMN,
VIEW_NAME_COLUMN, jsonb_value,
};
use common_event_recorder::testing::assert_event_contract;
use common_event_recorder::{Event, EventTypeFilter};
use common_procedure::{
ChildSubmissionOutcome, EventRuntimeContext, EventTrigger, Output, Procedure, ProcedureEvent,
ChildSubmissionOutcome, EventContext, EventTrigger, Output, Procedure, ProcedureEvent,
ProcedureId, ProcedureState, RetryPhase,
};
use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value};
use super::test_util::assert_event_filter;
use crate::ddl::create_view::CreateViewProcedure;
use crate::ddl::drop_view::DropViewProcedure;
use crate::ddl::event::view::{
@@ -36,7 +36,6 @@ use crate::ddl::event::view::{
};
use crate::ddl::tests::create_view::test_create_view_task;
use crate::ddl::tests::drop_view::new_drop_view_task;
use crate::rpc::ddl::EventContext;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
#[test]
@@ -44,7 +43,7 @@ fn test_view_submitted_event_contracts() {
let mut task = test_create_view_task("v_metrics");
task.create_view.or_replace = true;
task.create_view.create_if_not_exists = true;
let create = CreateViewProcedure::new(task, EventContext::default(), test_context());
let create = CreateViewProcedure::new(task, test_context());
let event = event_for(&create, EventTrigger::Submitted);
assert_view_event_contract(
@@ -72,11 +71,7 @@ fn test_view_submitted_event_contracts() {
assert!(!payload.contains(omitted));
}
let drop = DropViewProcedure::new(
new_drop_view_task("view_name", 42, true),
EventContext::default(),
test_context(),
);
let drop = DropViewProcedure::new(new_drop_view_task("view_name", 42, true), test_context());
let event = event_for(&drop, EventTrigger::Submitted);
assert_view_event_contract(
@@ -140,16 +135,8 @@ fn test_view_lifecycle_event_contracts() {
#[test]
fn test_view_procedures_preserve_lifecycle_locators() {
let create = CreateViewProcedure::new(
test_create_view_task("view_name"),
EventContext::default(),
test_context(),
);
let drop = DropViewProcedure::new(
new_drop_view_task("view_name", 42, false),
EventContext::default(),
test_context(),
);
let create = CreateViewProcedure::new(test_create_view_task("view_name"), test_context());
let drop = DropViewProcedure::new(new_drop_view_task("view_name", 42, false), test_context());
let triggers = [
EventTrigger::Recovered,
EventTrigger::ChildSubmitted {
@@ -199,11 +186,7 @@ fn test_view_procedures_preserve_lifecycle_locators() {
#[test]
fn test_create_view_succeeded_output_mapping() {
let procedure = CreateViewProcedure::new(
test_create_view_task("view_name"),
EventContext::default(),
test_context(),
);
let procedure = CreateViewProcedure::new(test_create_view_task("view_name"), test_context());
let state = ProcedureState::Done {
output: Some(Arc::new(84_u32)),
};
@@ -240,21 +223,14 @@ fn test_create_view_succeeded_output_mapping() {
#[test]
fn test_create_view_event_filter() {
let procedure = CreateViewProcedure::new(
test_create_view_task("view_name"),
EventContext::default(),
test_context(),
);
let procedure = CreateViewProcedure::new(test_create_view_task("view_name"), test_context());
assert_event_filter(&procedure, CREATE_VIEW_EVENT_TYPE);
}
#[test]
fn test_drop_view_event_filter() {
let procedure = DropViewProcedure::new(
new_drop_view_task("view_name", 42, false),
EventContext::default(),
test_context(),
);
let procedure =
DropViewProcedure::new(new_drop_view_task("view_name", 42, false), test_context());
assert_event_filter(&procedure, DROP_VIEW_EVENT_TYPE);
}
@@ -273,7 +249,6 @@ fn test_view_event_procedure_envelope_contract() {
referenced_table_count: 1,
column_count: 1,
},
EventContext::default(),
)),
ProcedureState::Running,
EventTrigger::Submitted,
@@ -344,17 +319,11 @@ fn view_schema() -> Vec<ColumnSchema> {
SCHEMA_NAME_COLUMN.column_schema(),
VIEW_NAME_COLUMN.column_schema(),
VIEW_ID_COLUMN.column_schema(),
EVENT_CONTEXT_COLUMN.column_schema(),
]
}
fn assert_view_event_contract(event: &dyn Event, event_type: &str, locator: ViewEventLocator<'_>) {
let mut values = locator.values();
values.push(if event.json_payload().unwrap().is_null() {
Value { value_data: None }
} else {
default_event_context_value()
});
let values = locator.values();
assert_event_contract(event, event_type, &view_schema(), &[Row { values }]);
}
@@ -372,19 +341,16 @@ fn assert_procedure_event_contract(
PROCEDURE_TRIGGER_COLUMN.column_schema(),
];
schema.extend(view_schema());
schema.push(EVENT_CONTEXT_COLUMN.column_schema());
let mut values = vec![
ValueData::StringValue(event.procedure_id.to_string()).into(),
ValueData::StringValue(state.to_string()).into(),
ValueData::StringValue(String::new()).into(),
procedure_trigger_value(trigger),
jsonb_value(&serde_json::json!({"type": trigger})),
];
values.extend(locator.values());
values.push(if trigger == "Submitted" {
default_event_context_value()
} else {
Value { value_data: None }
});
values.push(Value { value_data: None });
assert_event_contract(event, event_type, &schema, &[Row { values }]);
}
@@ -403,11 +369,12 @@ fn event_for_state(
lifecycle_state: &ProcedureState,
) -> Box<dyn Event> {
procedure
.event(&EventRuntimeContext {
.event(&EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state,
trigger,
event_type_filter: Arc::new(EventTypeFilter::All),
event_context: None,
})
.unwrap()
}
src/common/meta/src/ddl/truncate_table.rs
+7 -17
View File
@@ -20,7 +20,7 @@ use api::v1::region::{
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::debug;
@@ -43,7 +43,7 @@ use crate::key::table_info::TableInfoValue;
use crate::key::table_name::TableNameKey;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
use crate::metrics;
use crate::rpc::ddl::{EventContext, TruncateTableTask};
use crate::rpc::ddl::TruncateTableTask;
use crate::rpc::router::{find_leader_regions, find_leaders};
pub struct TruncateTableProcedure {
@@ -89,10 +89,7 @@ impl Procedure for TruncateTableProcedure {
LockKey::new(lock_key)
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::TruncateTable.as_str())
@@ -103,11 +100,9 @@ impl Procedure for TruncateTableProcedure {
let locator = TableDdlLocator::new(&task.catalog, &task.schema, &task.table)
.with_table_id(task.table_id);
let event = match &ctx.trigger {
EventTrigger::Submitted => TableDdlEvent::truncate_table_submitted(
locator,
task.time_ranges.len(),
self.data.event_context.clone(),
),
EventTrigger::Submitted => {
TableDdlEvent::truncate_table_submitted(locator, task.time_ranges.len())
}
_ => TableDdlEvent::lifecycle(TableDdlEventType::TruncateTable, [locator]),
};
@@ -122,11 +117,10 @@ impl TruncateTableProcedure {
task: TruncateTableTask,
table_info_value: DeserializedValueWithBytes<TableInfoValue>,
context: DdlContext,
event_context: EventContext,
) -> Self {
Self {
context,
data: TruncateTableData::new(task, table_info_value, event_context),
data: TruncateTableData::new(task, table_info_value),
}
}
@@ -233,21 +227,17 @@ pub struct TruncateTableData {
state: TruncateTableState,
task: TruncateTableTask,
table_info_value: DeserializedValueWithBytes<TableInfoValue>,
#[serde(default)]
event_context: EventContext,
}
impl TruncateTableData {
pub fn new(
task: TruncateTableTask,
table_info_value: DeserializedValueWithBytes<TableInfoValue>,
event_context: EventContext,
) -> Self {
Self {
state: TruncateTableState::Prepare,
task,
table_info_value,
event_context,
}
}
src/common/meta/src/ddl/undrop_table.rs
+13 -37
View File
@@ -17,7 +17,7 @@ use api::v1::region::{
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure,
Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure,
Result as ProcedureResult, Status,
};
use common_telemetry::tracing_context::TracingContext;
@@ -43,7 +43,7 @@ use crate::instruction::CacheIdent;
use crate::key::table_name::TableNameKey;
use crate::key::table_route::TableRouteValue;
use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
use crate::rpc::ddl::{EventContext, UndropTableTask};
use crate::rpc::ddl::UndropTableTask;
use crate::rpc::router::{
RegionRoute, find_follower_regions, find_followers, find_leader_regions, find_leaders,
};
@@ -56,17 +56,16 @@ pub struct UndropTableProcedure {
impl UndropTableProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::UndropTable";
pub fn new(task: UndropTableTask, context: DdlContext, event_context: EventContext) -> Self {
Self::new_with_original_table_name(task, context, None, event_context)
pub fn new(task: UndropTableTask, context: DdlContext) -> Self {
Self::new_with_original_table_name(task, context, None)
}
pub(crate) fn new_with_original_table_name(
task: UndropTableTask,
context: DdlContext,
table_name: Option<TableName>,
event_context: EventContext,
) -> Self {
let mut data = UndropTableData::new(task, event_context);
let mut data = UndropTableData::new(task);
data.table_name = table_name;
Self { context, data }
}
@@ -444,10 +443,7 @@ impl Procedure for UndropTableProcedure {
LockKey::new(lock_key)
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx
.event_type_filter
.allows(TableDdlEventType::UndropTable.as_str())
@@ -456,9 +452,7 @@ impl Procedure for UndropTableProcedure {
}
let locator = self.event_locator();
let event = match &ctx.trigger {
EventTrigger::Submitted => {
TableDdlEvent::undrop_table_submitted(locator, self.data.event_context.clone())
}
EventTrigger::Submitted => TableDdlEvent::undrop_table_submitted(locator),
_ => TableDdlEvent::lifecycle(TableDdlEventType::UndropTable, [locator]),
};
@@ -565,12 +559,10 @@ pub struct UndropTableData {
drop_generation: Option<String>,
#[serde(default)]
tombstone_identity_loaded: bool,
#[serde(default)]
event_context: EventContext,
}
impl UndropTableData {
fn new(task: UndropTableTask, event_context: EventContext) -> Self {
fn new(task: UndropTableTask) -> Self {
Self {
state: UndropTableState::Prepare,
task,
@@ -582,7 +574,6 @@ impl UndropTableData {
retention_expires_at: None,
drop_generation: None,
tombstone_identity_loaded: false,
event_context,
}
}
@@ -647,11 +638,7 @@ mod tests {
#[test]
fn test_map_restore_metadata_error_without_table_name() {
let context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
let procedure = UndropTableProcedure::new(
UndropTableTask { table_id: 42 },
context,
EventContext::default(),
);
let procedure = UndropTableProcedure::new(UndropTableTask { table_id: 42 }, context);
let err = procedure.map_restore_metadata_error(
error::TombstoneTargetAlreadyExistsSnafu {
@@ -666,11 +653,8 @@ mod tests {
#[test]
fn test_recovered_legacy_snapshot_reloads_tombstone_identity() {
let context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
let mut procedure = UndropTableProcedure::new(
UndropTableTask { table_id: 42 },
context.clone(),
EventContext::default(),
);
let mut procedure =
UndropTableProcedure::new(UndropTableTask { table_id: 42 }, context.clone());
procedure.data.state = UndropTableState::OpenRegions;
let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
let data = data.as_object_mut().unwrap();
@@ -695,11 +679,7 @@ mod tests {
let table_id = 1024;
let region_id = RegionId::new(table_id, 1);
let mut procedure = UndropTableProcedure::new(
UndropTableTask { table_id },
context,
EventContext::default(),
);
let mut procedure = UndropTableProcedure::new(UndropTableTask { table_id }, context);
procedure.data.table_route_value = Some(TableRouteValue::physical(vec![RegionRoute {
region: Region::new_test(region_id),
leader_peer: Some(Peer::empty(1)),
@@ -738,11 +718,7 @@ mod tests {
let table_id = 1024;
let region_id = RegionId::new(table_id, 1);
let mut procedure = UndropTableProcedure::new(
UndropTableTask { table_id },
context,
EventContext::default(),
);
let mut procedure = UndropTableProcedure::new(UndropTableTask { table_id }, context);
procedure.data.table_name = Some(TableName::new(
DEFAULT_CATALOG_NAME,
DEFAULT_SCHEMA_NAME,
src/common/meta/src/ddl_manager.rs
+110 -100
View File
@@ -78,8 +78,8 @@ use crate::rpc::ddl::trigger::DropTriggerTask;
use crate::rpc::ddl::{
AlterDatabaseTask, AlterTableTask, CommentOnTask, CreateDatabaseTask, CreateFlowTask,
CreateTableTask, CreateViewTask, DropDatabaseTask, DropFlowTask, DropTableTask, DropViewTask,
EventContext, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse,
TruncateTableTask, UndropTableTask,
PersistentEventContext, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest,
SubmitDdlTaskResponse, TruncateTableTask, UndropTableTask,
};
const MAX_REGION_ROUTE_CHANGE_RETRIES: usize = 3;
@@ -188,7 +188,6 @@ pub trait RepartitionProcedureFactory: Send + Sync {
source: RepartitionSource,
to_exprs: Vec<String>,
timeout: Option<Duration>,
event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError>;
fn register_loaders(
@@ -342,7 +341,7 @@ impl DdlManager {
repartition: Repartition,
wait: bool,
timeout: Duration,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
@@ -378,14 +377,14 @@ impl DdlManager {
source,
into_partition_exprs,
Some(timeout),
event_context,
)
.context(CreateRepartitionProcedureSnafu)?;
self.repartition_procedure_factory
.ensure_gc_requirement()
.await
.context(PersistRepartitionGcRequirementSnafu)?;
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
if wait {
self.execute_procedure_and_wait(procedure_with_id).await
} else {
@@ -401,7 +400,7 @@ impl DdlManager {
&self,
table_id: TableId,
alter_table_task: AlterTableTask,
event_context: EventContext,
event_context: PersistentEventContext,
ddl_options: DdlOptions,
) -> Result<(ProcedureId, Option<Output>)> {
// make alter_table_task mutable so we can call .take() on its field
@@ -456,12 +455,12 @@ impl DdlManager {
let procedure = AlterTableProcedure::new_with_region_locks(
table_id,
alter_table_task.clone(),
event_context.clone(),
region_ids_to_lock,
context,
)?;
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure))
.with_event_context(event_context.clone());
let result = self.execute_procedure_and_wait(procedure_with_id).await?;
if result
.1
@@ -490,14 +489,18 @@ impl DdlManager {
&self,
create_table_task: CreateTableTask,
query_context: QueryContext,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure =
CreateTableProcedure::new(create_table_task, query_context, event_context, context)?;
let procedure = CreateTableProcedure::new_with_query_context(
create_table_task,
query_context,
context,
)?;
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -507,13 +510,14 @@ impl DdlManager {
pub async fn submit_create_view_task(
&self,
create_view_task: CreateViewTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = CreateViewProcedure::new(create_view_task, event_context, context);
let procedure = CreateViewProcedure::new(create_view_task, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -524,18 +528,15 @@ impl DdlManager {
&self,
create_table_tasks: Vec<CreateTableTask>,
physical_table_id: TableId,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = CreateLogicalTablesProcedure::new(
create_table_tasks,
physical_table_id,
event_context,
context,
);
let procedure =
CreateLogicalTablesProcedure::new(create_table_tasks, physical_table_id, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -546,18 +547,15 @@ impl DdlManager {
&self,
alter_table_tasks: Vec<AlterTableTask>,
physical_table_id: TableId,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = AlterLogicalTablesProcedure::new(
alter_table_tasks,
physical_table_id,
event_context,
context,
);
let procedure =
AlterLogicalTablesProcedure::new(alter_table_tasks, physical_table_id, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -567,13 +565,14 @@ impl DdlManager {
pub async fn submit_drop_table_task(
&self,
drop_table_task: DropTableTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = DropTableProcedure::new(drop_table_task, context, event_context);
let procedure = DropTableProcedure::new(drop_table_task, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -584,7 +583,7 @@ impl DdlManager {
pub async fn submit_undrop_table_task(
&self,
undrop_table_task: UndropTableTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
#[cfg(not(feature = "enterprise"))]
{
@@ -610,9 +609,9 @@ impl DdlManager {
undrop_table_task,
context,
Some(original_table_name),
event_context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure))
.with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -624,7 +623,7 @@ impl DdlManager {
pub async fn submit_purge_dropped_table_task(
&self,
purge_dropped_table_task: PurgeDroppedTableTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
#[cfg(not(feature = "enterprise"))]
{
@@ -638,9 +637,9 @@ impl DdlManager {
#[cfg(feature = "enterprise")]
{
let context = self.create_context();
let procedure =
PurgeDroppedTableProcedure::new(purge_dropped_table_task, context, event_context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure = PurgeDroppedTableProcedure::new(purge_dropped_table_task, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure))
.with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -666,12 +665,11 @@ impl DdlManager {
#[cfg(feature = "enterprise")]
{
let context = self.create_context();
let procedure = PurgeDroppedTableProcedure::new_if_expired(
purge_dropped_table_task,
context,
EventContext::new(TriggerReason::ScheduledGc),
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let event_context = PersistentEventContext::new(TriggerReason::ScheduledGc);
let procedure =
PurgeDroppedTableProcedure::new_if_expired(purge_dropped_table_task, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure))
.with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -688,7 +686,7 @@ impl DdlManager {
options,
creator,
}: CreateDatabaseTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = CreateDatabaseProcedure::new(
@@ -697,10 +695,10 @@ impl DdlManager {
create_if_not_exists,
options,
creator,
event_context,
context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -714,12 +712,12 @@ impl DdlManager {
schema,
drop_if_exists,
}: DropDatabaseTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure =
DropDatabaseProcedure::new(catalog, schema, drop_if_exists, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure = DropDatabaseProcedure::new(catalog, schema, drop_if_exists, context);
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -727,11 +725,12 @@ impl DdlManager {
pub async fn submit_alter_database(
&self,
alter_database_task: AlterDatabaseTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = AlterDatabaseProcedure::new(alter_database_task, event_context, context)?;
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure = AlterDatabaseProcedure::new(alter_database_task, context)?;
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -742,12 +741,12 @@ impl DdlManager {
&self,
create_flow: CreateFlowTask,
query_context: QueryContext,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure =
CreateFlowProcedure::new(create_flow, query_context, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure = CreateFlowProcedure::new(create_flow, query_context, context);
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -757,11 +756,12 @@ impl DdlManager {
pub async fn submit_drop_flow_task(
&self,
drop_flow: DropFlowTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = DropFlowProcedure::new(drop_flow, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure = DropFlowProcedure::new(drop_flow, context);
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -771,11 +771,12 @@ impl DdlManager {
pub async fn submit_drop_view_task(
&self,
drop_view: DropViewTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = DropViewProcedure::new(drop_view, event_context, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure = DropViewProcedure::new(drop_view, context);
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -786,17 +787,13 @@ impl DdlManager {
&self,
truncate_table_task: TruncateTableTask,
table_info_value: DeserializedValueWithBytes<TableInfoValue>,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
let procedure = TruncateTableProcedure::new(
truncate_table_task,
table_info_value,
context,
event_context,
);
let procedure = TruncateTableProcedure::new(truncate_table_task, table_info_value, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -806,6 +803,7 @@ impl DdlManager {
pub async fn submit_comment_on_task(
&self,
mut comment_on_task: CommentOnTask,
event_context: PersistentEventContext,
) -> Result<(ProcedureId, Option<Output>)> {
let context = self.create_context();
comment_on_task
@@ -815,7 +813,8 @@ impl DdlManager {
)
.await?;
let procedure = CommentOnProcedure::new(comment_on_task, context);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
self.execute_procedure_and_wait(procedure_with_id).await
}
@@ -869,7 +868,10 @@ impl DdlManager {
timeout,
task,
} = request;
let event_context = EventContext::from_query_context(&query_context);
let event_context = ctx
.event_context
.clone()
.unwrap_or_else(|| crate::rpc::ddl::event_context_from_query_context(&query_context));
let ddl_options = DdlOptions { wait, timeout };
async move {
debug!("Submitting Ddl task: {:?}", task);
@@ -924,7 +926,9 @@ impl DdlManager {
DropView(drop_view_task) => {
handle_drop_view_task(self, drop_view_task, event_context).await
}
CommentOn(comment_on_task) => handle_comment_on_task(self, comment_on_task).await,
CommentOn(comment_on_task) => {
handle_comment_on_task(self, comment_on_task, event_context).await
}
#[cfg(feature = "enterprise")]
CreateTrigger(create_trigger_task) => {
handle_create_trigger_task(self, create_trigger_task, query_context).await
@@ -943,7 +947,7 @@ impl DdlManager {
async fn handle_truncate_table_task(
ddl_manager: &DdlManager,
truncate_table_task: TruncateTableTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let table_id = truncate_table_task.table_id;
let table_metadata_manager = &ddl_manager.table_metadata_manager();
@@ -983,7 +987,7 @@ async fn handle_alter_table_task(
ddl_manager: &DdlManager,
alter_table_task: AlterTableTask,
ddl_options: DdlOptions,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let table_ref = alter_table_task.table_ref();
@@ -1030,7 +1034,7 @@ async fn handle_alter_table_task(
async fn handle_drop_table_task(
ddl_manager: &DdlManager,
drop_table_task: DropTableTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let table_id = drop_table_task.table_id;
let (id, _) = ddl_manager
@@ -1048,7 +1052,7 @@ async fn handle_drop_table_task(
async fn handle_undrop_table_task(
ddl_manager: &DdlManager,
undrop_table_task: UndropTableTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let table_id = undrop_table_task.table_id;
let (id, _) = ddl_manager
@@ -1066,7 +1070,7 @@ async fn handle_undrop_table_task(
async fn handle_purge_dropped_table_task(
ddl_manager: &DdlManager,
purge_dropped_table_task: PurgeDroppedTableTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_purge_dropped_table_task(purge_dropped_table_task, event_context)
@@ -1084,7 +1088,7 @@ async fn handle_create_table_task(
ddl_manager: &DdlManager,
create_table_task: CreateTableTask,
query_context: QueryContext,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, output) = ddl_manager
.submit_create_table_task(create_table_task, query_context, event_context)
@@ -1110,7 +1114,7 @@ async fn handle_create_table_task(
async fn handle_create_logical_table_tasks(
ddl_manager: &DdlManager,
create_table_tasks: Vec<CreateTableTask>,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
ensure!(
!create_table_tasks.is_empty(),
@@ -1155,7 +1159,7 @@ async fn handle_create_logical_table_tasks(
async fn handle_create_database_task(
ddl_manager: &DdlManager,
create_database_task: CreateDatabaseTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let catalog = create_database_task.catalog.clone();
let schema = create_database_task.schema.clone();
@@ -1178,7 +1182,7 @@ async fn handle_create_database_task(
async fn handle_drop_database_task(
ddl_manager: &DdlManager,
drop_database_task: DropDatabaseTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_drop_database(drop_database_task.clone(), event_context)
@@ -1199,7 +1203,7 @@ async fn handle_drop_database_task(
async fn handle_alter_database_task(
ddl_manager: &DdlManager,
alter_database_task: AlterDatabaseTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_alter_database(alter_database_task.clone(), event_context)
@@ -1221,7 +1225,7 @@ async fn handle_alter_database_task(
async fn handle_drop_flow_task(
ddl_manager: &DdlManager,
drop_flow_task: DropFlowTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_drop_flow_task(drop_flow_task.clone(), event_context)
@@ -1266,7 +1270,7 @@ async fn handle_drop_trigger_task(
async fn handle_drop_view_task(
ddl_manager: &DdlManager,
drop_view_task: DropViewTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_drop_view_task(drop_view_task.clone(), event_context)
@@ -1289,7 +1293,7 @@ async fn handle_create_flow_task(
ddl_manager: &DdlManager,
create_flow_task: CreateFlowTask,
query_context: QueryContext,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, output) = ddl_manager
.submit_create_flow_task(create_flow_task.clone(), query_context, event_context)
@@ -1349,7 +1353,7 @@ async fn handle_create_trigger_task(
async fn handle_alter_logical_table_tasks(
ddl_manager: &DdlManager,
alter_table_tasks: Vec<AlterTableTask>,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
ensure!(
!alter_table_tasks.is_empty(),
@@ -1388,7 +1392,7 @@ async fn handle_alter_logical_table_tasks(
async fn handle_create_view_task(
ddl_manager: &DdlManager,
create_view_task: CreateViewTask,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, output) = ddl_manager
.submit_create_view_task(create_view_task, event_context)
@@ -1414,9 +1418,10 @@ async fn handle_create_view_task(
async fn handle_comment_on_task(
ddl_manager: &DdlManager,
comment_on_task: CommentOnTask,
event_context: PersistentEventContext,
) -> Result<SubmitDdlTaskResponse> {
let (id, _) = ddl_manager
.submit_comment_on_task(comment_on_task.clone())
.submit_comment_on_task(comment_on_task.clone(), event_context)
.await?;
let procedure_id = id.to_string();
@@ -1469,7 +1474,7 @@ mod tests {
use crate::procedure_executor::ExecutorContext;
use crate::region_keeper::MemoryRegionKeeper;
use crate::region_registry::LeaderRegionRegistry;
use crate::rpc::ddl::{CreatorGrantIntent, EventContext, UndropTableTask};
use crate::rpc::ddl::{CreatorGrantIntent, PersistentEventContext, UndropTableTask};
#[cfg(not(feature = "enterprise"))]
use crate::rpc::ddl::{DdlTask, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest};
use crate::sequence::SequenceBuilder;
@@ -1506,7 +1511,6 @@ mod tests {
_source: RepartitionSource,
_to_exprs: Vec<String>,
_timeout: Option<Duration>,
_event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError> {
unimplemented!()
}
@@ -1680,7 +1684,10 @@ mod tests {
let ddl_manager = build_soft_drop_test_ddl_manager();
let err = ddl_manager
.submit_undrop_table_task(UndropTableTask { table_id: 1024 }, EventContext::default())
.submit_undrop_table_task(
UndropTableTask { table_id: 1024 },
PersistentEventContext::default(),
)
.await
.unwrap_err();
@@ -1694,7 +1701,10 @@ mod tests {
let ddl_manager = build_soft_drop_test_ddl_manager();
let err = ddl_manager
.submit_undrop_table_task(UndropTableTask { table_id: 1024 }, EventContext::default())
.submit_undrop_table_task(
UndropTableTask { table_id: 1024 },
PersistentEventContext::default(),
)
.await
.unwrap_err();
assert!(matches!(err, crate::error::Error::Unsupported { .. }));
@@ -1702,7 +1712,7 @@ mod tests {
let err = ddl_manager
.submit_purge_dropped_table_task(
PurgeDroppedTableTask { table_id: 1024 },
EventContext::default(),
PersistentEventContext::default(),
)
.await
.unwrap_err();
src/common/meta/src/procedure_executor.rs
+4 -1
View File
@@ -15,6 +15,7 @@
use std::sync::Arc;
use api::v1::meta::{ProcedureDetailResponse, ReconcileRequest, ReconcileResponse};
use common_event_recorder::PersistentEventContext;
use common_procedure::{ProcedureId, ProcedureManagerRef};
use common_telemetry::tracing_context::W3cTrace;
use snafu::{OptionExt, ResultExt};
@@ -30,9 +31,11 @@ use crate::rpc::procedure::{
};
/// The context of procedure executor.
#[derive(Debug, Default)]
#[derive(Debug, Clone, Default)]
pub struct ExecutorContext {
pub tracing_context: Option<W3cTrace>,
pub actor: Option<String>,
pub event_context: Option<PersistentEventContext>,
}
/// The procedure executor that accepts ddl, region migration task etc.
src/common/meta/src/rpc/ddl.rs
+18 -80
View File
@@ -43,14 +43,14 @@ use api::v1::{
use base64::Engine as _;
use base64::engine::general_purpose;
use common_catalog::{format_full_flow_name, format_full_table_name};
use common_base::protocol::Channel;
use common_error::ext::BoxedError;
use common_session::channel_protocol;
pub use common_event_recorder::{PersistentEventContext, TriggerReason};
use common_time::{DatabaseTimeToLive, Timestamp};
use prost::Message;
use serde::{Deserialize, Serialize};
use serde_with::{DefaultOnNull, serde_as};
use snafu::{OptionExt, ResultExt};
use strum::{AsRefStr, EnumString};
use table::metadata::{TableId, TableInfo};
use table::requests::validate_database_option;
use table::table_name::TableName;
@@ -1668,83 +1668,21 @@ pub struct QueryContext {
pub sst_min_sequences: HashMap<u64, u64>,
}
impl QueryContext {
/// Returns the protocol name represented by the wire channel value.
pub fn channel_protocol(&self) -> Option<&'static str> {
channel_protocol(self.channel)
/// Builds event context from frontend query context metadata.
pub fn event_context_from_query_context(query_context: &QueryContext) -> PersistentEventContext {
let reason = query_context
.extensions
.get(TRIGGER_REASON_EXTENSION_KEY)
.map(|reason| TriggerReason::from_extension(reason))
.unwrap_or_default();
let channel = Channel::from(u32::from(query_context.channel));
let context = PersistentEventContext::new(reason);
if channel == Channel::Unknown {
context
} else {
context.with_protocol(channel.as_ref())
}
}
/// The stable context recorded for a procedure event.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct EventContext {
pub reason: TriggerReason,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub protocol: Option<String>,
#[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
pub extensions: serde_json::Map<String, serde_json::Value>,
}
impl EventContext {
/// Creates an event context with no additional extensions.
pub fn new(reason: TriggerReason) -> Self {
Self {
reason,
protocol: None,
extensions: Default::default(),
}
}
/// Adds the protocol that originated the operation.
pub fn with_protocol(mut self, protocol: impl Into<String>) -> Self {
self.protocol = Some(protocol.into());
self
}
/// Builds an event context from frontend query context metadata.
pub fn from_query_context(query_context: &QueryContext) -> Self {
let reason = query_context
.extensions
.get(TRIGGER_REASON_EXTENSION_KEY)
.map(|reason| TriggerReason::from_extension(reason))
.unwrap_or_default();
let context = Self::new(reason);
if let Some(protocol) = query_context.channel_protocol() {
context.with_protocol(protocol)
} else {
context
}
}
}
impl Default for EventContext {
fn default() -> Self {
Self::new(TriggerReason::default())
}
}
/// The stable classification of a procedure trigger.
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, AsRefStr, EnumString,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum TriggerReason {
Manual,
AutoCreate,
AutoAlter,
AutoRepartition,
AutoRebalance,
RegionFailover,
ScheduledGc,
#[default]
#[serde(other)]
Unknown,
}
impl TriggerReason {
pub fn from_extension(value: &str) -> Self {
value.parse().unwrap_or_default()
}
}
@@ -2329,7 +2267,7 @@ mod tests {
#[test]
fn test_event_context_serialization() {
let context = EventContext::new(TriggerReason::Manual).with_protocol("mysql");
let context = PersistentEventContext::new(TriggerReason::Manual).with_protocol("mysql");
assert_eq!(
serde_json::json!({
@@ -2341,7 +2279,7 @@ mod tests {
assert_eq!(
serde_json::json!({ "reason": "manual" }),
serde_json::to_value(EventContext::new(TriggerReason::Manual)).unwrap()
serde_json::to_value(PersistentEventContext::new(TriggerReason::Manual)).unwrap()
);
}
@@ -2354,7 +2292,7 @@ mod tests {
);
query_context.channel = 4;
let event_context = EventContext::from_query_context(&query_context);
let event_context = event_context_from_query_context(&query_context);
assert_eq!(event_context.reason, TriggerReason::AutoCreate);
assert_eq!(event_context.protocol.as_deref(), Some("prometheus"));
src/common/procedure-test/src/lib.rs
+4
View File
@@ -84,6 +84,7 @@ pub async fn execute_procedure_until_done(procedure: &mut dyn Procedure) -> Opti
let ctx = Context {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
loop {
@@ -110,6 +111,7 @@ pub async fn execute_procedure_once(
let ctx = Context {
procedure_id,
provider: Arc::new(provider),
event_context: None,
};
match procedure.execute(&ctx).await.unwrap() {
@@ -137,6 +139,7 @@ pub async fn execute_until_suspended_or_done(
let ctx = Context {
procedure_id,
provider: Arc::new(provider),
event_context: None,
};
loop {
@@ -155,6 +158,7 @@ pub fn new_test_procedure_context() -> Context {
Context {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
}
}
src/common/procedure/src/event.rs
+73 -9
View File
@@ -19,13 +19,13 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN,
jsonb_value, procedure_event_column_schemas,
EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN,
PROCEDURE_TRIGGER_COLUMN, jsonb_value, nullable_json, procedure_event_column_schemas,
};
use common_time::timestamp::{TimeUnit, Timestamp};
use snafu::ResultExt;
use crate::{EventTrigger, ProcedureId, ProcedureState};
use crate::{EventTrigger, ProcedureContext, ProcedureId, ProcedureState};
pub const EVENTS_TABLE_PROCEDURE_ID_COLUMN_NAME: &str = PROCEDURE_ID_COLUMN.name();
pub const EVENTS_TABLE_PROCEDURE_STATE_COLUMN_NAME: &str = PROCEDURE_STATE_COLUMN.name();
@@ -43,6 +43,8 @@ pub struct ProcedureEvent {
pub state: ProcedureState,
/// The lifecycle trigger that caused the event to be emitted.
pub trigger: EventTrigger,
/// Context associated with the root submission.
pub context: ProcedureContext,
/// The event emitted by the procedure. It's generated by [Procedure::event].
pub internal_event: Box<dyn Event>,
}
@@ -53,6 +55,22 @@ impl ProcedureEvent {
internal_event: Box<dyn Event>,
state: ProcedureState,
trigger: EventTrigger,
) -> Self {
Self::new_with_context(
procedure_id,
internal_event,
state,
trigger,
ProcedureContext::default(),
)
}
pub fn new_with_context(
procedure_id: ProcedureId,
internal_event: Box<dyn Event>,
state: ProcedureState,
trigger: EventTrigger,
context: ProcedureContext,
) -> Self {
Self {
procedure_id,
@@ -60,6 +78,7 @@ impl ProcedureEvent {
timestamp: Timestamp::current_time(TimeUnit::Nanosecond),
state,
trigger,
context,
}
}
}
@@ -79,7 +98,9 @@ impl Event for ProcedureEvent {
fn extra_schema(&self) -> Vec<ColumnSchema> {
let mut schema = procedure_event_column_schemas();
schema.append(&mut self.internal_event.extra_schema());
let mut internal_schema = self.internal_event.extra_schema();
schema.append(&mut internal_schema);
schema.push(EVENT_CONTEXT_COLUMN.column_schema());
schema
}
@@ -97,9 +118,15 @@ impl Event for ProcedureEvent {
_ => String::new(),
};
let trigger = serde_json::to_value(&self.trigger).context(SerializeEventSnafu)?;
let event_context = matches!(self.trigger, EventTrigger::Submitted)
.then_some(self.context.event_context.as_ref())
.flatten()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
for internal_event_extra_row in internal_event_extra_rows.iter_mut() {
let mut values = Vec::with_capacity(4 + internal_event_extra_row.values.len());
let mut values = Vec::with_capacity(5 + internal_event_extra_row.values.len());
values.extend([
ValueData::StringValue(procedure_id.clone()).into(),
ValueData::StringValue(state.clone()).into(),
@@ -107,6 +134,7 @@ impl Event for ProcedureEvent {
jsonb_value(&trigger),
]);
values.append(&mut internal_event_extra_row.values);
values.push(nullable_json(event_context.as_ref()));
rows.push(Row { values });
}
@@ -126,13 +154,15 @@ mod tests {
use api::v1::{ColumnDataType, ColumnSchema, Row, SemanticType, Value};
use common_error::mock::MockError;
use common_error::status_code::StatusCode;
use common_event_recorder::Event;
use common_event_recorder::event_table::{PROCEDURE_TRIGGER_COLUMN, jsonb_value};
use common_event_recorder::event_table::{
EVENT_CONTEXT_COLUMN, PROCEDURE_TRIGGER_COLUMN, jsonb_value,
};
use common_event_recorder::{Event, PersistentEventContext, TriggerReason};
use serde_json::json;
use crate::{
ChildSubmissionOutcome, Error, EventTrigger, ProcedureEvent, ProcedureId, ProcedureState,
RetryPhase,
ChildSubmissionOutcome, Error, EventTrigger, ProcedureContext, ProcedureEvent, ProcedureId,
ProcedureState, RetryPhase,
};
#[derive(Debug)]
@@ -192,6 +222,7 @@ mod tests {
ValueData::StringValue(String::new()).into(),
jsonb_value(&json!({"type": "Submitted"})),
ValueData::StringValue("test_event1".to_string()).into(),
Value { value_data: None },
],
},
Row {
@@ -201,6 +232,7 @@ mod tests {
ValueData::StringValue(String::new()).into(),
jsonb_value(&json!({"type": "Submitted"})),
ValueData::StringValue("test_event2".to_string()).into(),
Value { value_data: None },
],
},
Row {
@@ -210,6 +242,7 @@ mod tests {
ValueData::StringValue(String::new()).into(),
jsonb_value(&json!({"type": "Submitted"})),
Value { value_data: None },
Value { value_data: None },
],
},
]
@@ -280,10 +313,41 @@ mod tests {
semantic_type: SemanticType::Field.into(),
..Default::default()
},
EVENT_CONTEXT_COLUMN.column_schema(),
]
);
}
#[test]
fn procedure_event_only_records_context_on_submission() {
let context = ProcedureContext::from_event_context(PersistentEventContext::new(
TriggerReason::AutoRepartition,
));
let submitted = ProcedureEvent::new_with_context(
ProcedureId::random(),
Box::new(TestEvent {}),
ProcedureState::Running,
EventTrigger::Submitted,
context.clone(),
);
let succeeded = ProcedureEvent::new_with_context(
ProcedureId::random(),
Box::new(TestEvent {}),
ProcedureState::Done { output: None },
EventTrigger::Succeeded,
context,
);
assert_eq!(
submitted.extra_rows().unwrap()[0].values[5],
jsonb_value(&json!({"reason": "auto_repartition"}))
);
assert_eq!(
succeeded.extra_rows().unwrap()[0].values[5],
Value { value_data: None }
);
}
#[test]
fn test_event_trigger_serialization() {
let procedure_id = ProcedureId::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
src/common/procedure/src/lib.rs
+2 -2
View File
@@ -30,8 +30,8 @@ pub use crate::error::{Error, Result};
pub use crate::event::ProcedureEvent;
pub use crate::procedure::{
BoxedProcedure, BoxedProcedureLoader, ChildSubmissionOutcome, Context, ContextProvider,
ContextProviderRef, EventRuntimeContext, EventTrigger, LockKey, Output, ParseIdError,
PoisonKey, PoisonKeys, Procedure, ProcedureId, ProcedureInfo, ProcedureManager,
ContextProviderRef, EventContext, EventTrigger, LockKey, Output, ParseIdError, PoisonKey,
PoisonKeys, Procedure, ProcedureContext, ProcedureId, ProcedureInfo, ProcedureManager,
ProcedureManagerRef, ProcedureState, ProcedureWithId, RetryPhase, Status, StringKey,
};
pub use crate::watcher::Watcher;
src/common/procedure/src/local.rs
+70 -10
View File
@@ -43,8 +43,8 @@ use crate::rwlock::{KeyRwLock, OwnedKeyRwLockGuard};
use crate::store::poison_store::PoisonStoreRef;
use crate::store::{ProcedureMessage, ProcedureMessages, ProcedureStore, StateStoreRef};
use crate::{
BoxedProcedure, ContextProvider, EventTrigger, LockKey, PoisonKey, ProcedureId,
ProcedureManager, ProcedureState, ProcedureWithId, StringKey, Watcher,
BoxedProcedure, ContextProvider, EventTrigger, LockKey, PoisonKey, ProcedureContext,
ProcedureId, ProcedureManager, ProcedureState, ProcedureWithId, StringKey, Watcher,
};
/// The expired time of a procedure's metadata.
@@ -81,6 +81,8 @@ pub(crate) struct ProcedureMeta {
type_name: String,
/// Parent procedure id.
parent_id: Option<ProcedureId>,
/// Context associated with the root submission.
context: ProcedureContext,
/// Notify to wait for subprocedures.
child_notify: Notify,
/// Lock required by this procedure.
@@ -105,6 +107,7 @@ impl ProcedureMeta {
id: ProcedureId,
procedure_state: ProcedureState,
parent_id: Option<ProcedureId>,
context: ProcedureContext,
lock_key: LockKey,
poison_keys: PoisonKeys,
type_name: &str,
@@ -113,6 +116,7 @@ impl ProcedureMeta {
ProcedureMeta {
id,
parent_id,
context,
child_notify: Notify::new(),
lock_key,
poison_keys,
@@ -716,6 +720,7 @@ impl LocalManager {
procedure_state: ProcedureState,
step: u32,
procedure: BoxedProcedure,
context: ProcedureContext,
origin: RootSubmissionOrigin,
) -> Result<Watcher> {
ensure!(self.manager_ctx.running(), ManagerNotStartSnafu);
@@ -724,6 +729,7 @@ impl LocalManager {
procedure_id,
procedure_state,
None,
context,
procedure.lock_key(),
procedure.poison_keys(),
procedure.type_name(),
@@ -837,6 +843,7 @@ impl LocalManager {
procedure_state,
loaded_procedure.step,
loaded_procedure.procedure,
message.context.clone(),
RootSubmissionOrigin::Recovery,
) {
error!(e; "Failed to recover procedure {}", procedure_id);
@@ -964,6 +971,7 @@ impl ProcedureManager for LocalManager {
ProcedureState::Running,
0,
procedure.procedure,
procedure.context,
RootSubmissionOrigin::Fresh,
)
}
@@ -1021,6 +1029,7 @@ pub(crate) mod test_util {
ProcedureId::random(),
ProcedureState::Running,
None,
ProcedureContext::default(),
LockKey::default(),
PoisonKeys::default(),
"ProcedureAdapter",
@@ -1043,7 +1052,10 @@ mod tests {
use common_error::mock::MockError;
use common_error::status_code::StatusCode;
use common_event_recorder::{Event, EventRecorder, EventTypeFilter, EventTypeFilterRef};
use common_event_recorder::{
Event, EventRecorder, EventTypeFilter, EventTypeFilterRef, PersistentEventContext,
TriggerReason,
};
use common_test_util::temp_dir::create_temp_dir;
use tokio::sync::oneshot;
use tokio::time::{sleep, timeout};
@@ -1053,8 +1065,8 @@ mod tests {
use crate::store::state_store::ObjectStateStore;
use crate::test_util::InMemoryPoisonStore;
use crate::{
ChildSubmissionOutcome, Context, EventRuntimeContext, EventTrigger, Procedure,
ProcedureEvent, Status,
ChildSubmissionOutcome, Context, EventContext, EventTrigger, Procedure, ProcedureEvent,
Status,
};
fn new_test_manager_context() -> ManagerContext {
@@ -1103,6 +1115,13 @@ mod tests {
})
.collect()
}
fn procedure_context(&self, procedure_id: ProcedureId) -> Option<ProcedureContext> {
self.events.lock().unwrap().iter().find_map(|event| {
let event = event.as_any().downcast_ref::<ProcedureEvent>().unwrap();
(event.procedure_id == procedure_id).then(|| event.context.clone())
})
}
}
impl EventRecorder for CapturingEventRecorder {
@@ -1296,7 +1315,7 @@ mod tests {
self.poison_keys.clone()
}
fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, _ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
Some(Box::new(TestProcedureEvent))
}
}
@@ -1341,6 +1360,7 @@ mod tests {
vec![ProcedureWithId {
id: self.child_id,
procedure: Box::new(ProcedureToLoad::new("child")),
context: ProcedureContext::default(),
}],
false,
))
@@ -1354,7 +1374,7 @@ mod tests {
LockKey::default()
}
fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, _ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
Some(Box::new(TestProcedureEvent))
}
}
@@ -1384,6 +1404,7 @@ mod tests {
vec![ProcedureWithId {
id: self.child_id,
procedure: Box::new(ProcedureToLoad::new("child")),
context: ProcedureContext::default(),
}],
false,
))
@@ -1397,7 +1418,7 @@ mod tests {
LockKey::default()
}
fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, _ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
Some(Box::new(TestProcedureEvent))
}
}
@@ -1424,7 +1445,7 @@ mod tests {
LockKey::default()
}
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
*self.captured_filter.lock().unwrap() = Some(ctx.event_type_filter.clone());
Some(Box::new(TestProcedureEvent))
}
@@ -1458,6 +1479,7 @@ mod tests {
procedure: Box::new(FilterCapturingProcedure {
captured_filter: captured_filter.clone(),
}),
context: ProcedureContext::default(),
})
.await
.unwrap();
@@ -1499,6 +1521,7 @@ mod tests {
.submit(ProcedureWithId {
id: ProcedureId::random(),
procedure: Box::new(ProcedureToLoad::new("fresh submission")),
context: ProcedureContext::default(),
})
.await
.unwrap();
@@ -1533,6 +1556,7 @@ mod tests {
ProcedureState::Running,
0,
Box::new(ProcedureToLoad::new("submit root without runner")),
ProcedureContext::default(),
RootSubmissionOrigin::Fresh,
)
});
@@ -1584,6 +1608,9 @@ mod tests {
child_id,
child_submitted: false,
}),
context: ProcedureContext::from_event_context(PersistentEventContext::new(
TriggerReason::AutoRepartition,
)),
})
.await
.unwrap();
@@ -1604,6 +1631,15 @@ mod tests {
},
)));
assert!(procedure_events.contains(&(child_id, EventTrigger::Submitted)));
assert_eq!(
event_recorder
.procedure_context(child_id)
.unwrap()
.event_context
.unwrap()
.reason,
TriggerReason::AutoRepartition
);
}
#[tokio::test]
@@ -1637,6 +1673,7 @@ mod tests {
proceed: Some(proceed_rx),
child_submitted: false,
}),
context: ProcedureContext::default(),
})
.await
.unwrap();
@@ -1711,12 +1748,15 @@ mod tests {
let procedure = ProcedureToLoad::new("recovered submission");
let procedure_id = ProcedureId::random();
ProcedureStore::from_object_store(object_store)
.store_procedure(
.store_procedure_with_context(
procedure_id,
0,
procedure.type_name().to_string(),
procedure.dump().unwrap(),
None,
ProcedureContext::from_event_context(PersistentEventContext::new(
TriggerReason::AutoRebalance,
)),
)
.await
.unwrap();
@@ -1732,6 +1772,15 @@ mod tests {
);
wait_for_trigger(&event_recorder, EventTrigger::Recovered).await;
assert!(!event_recorder.triggers().contains(&EventTrigger::Submitted));
assert_eq!(
event_recorder
.procedure_context(procedure_id)
.unwrap()
.event_context
.unwrap()
.reason,
TriggerReason::AutoRebalance
);
}
#[tokio::test]
@@ -1813,6 +1862,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.unwrap();
@@ -1965,6 +2015,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.is_ok()
@@ -1986,6 +2037,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(ProcedureToLoad::new("submit")),
context: ProcedureContext::default(),
})
.await
.unwrap_err();
@@ -2054,6 +2106,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.unwrap()
@@ -2094,6 +2147,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.unwrap_err(),
@@ -2126,6 +2180,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.is_ok()
@@ -2164,6 +2219,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.is_ok()
@@ -2193,6 +2249,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.is_ok()
@@ -2217,6 +2274,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.is_ok()
@@ -2265,6 +2323,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.unwrap_err();
@@ -2285,6 +2344,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: ProcedureContext::default(),
})
.await
.is_ok()
src/common/procedure/src/local/runner.rs
+25 -7
View File
@@ -34,8 +34,8 @@ use crate::procedure::{Output, StringKey};
use crate::rwlock::OwnedKeyRwLockGuard;
use crate::store::{ProcedureMessage, ProcedureStore};
use crate::{
BoxedProcedure, ChildSubmissionOutcome, Context, Error, EventRuntimeContext, EventTrigger,
Procedure, ProcedureId, ProcedureState, ProcedureWithId, RetryPhase, Status,
BoxedProcedure, ChildSubmissionOutcome, Context, Error, EventContext, EventTrigger, Procedure,
ProcedureId, ProcedureState, ProcedureWithId, RetryPhase, Status,
};
/// A guard to cleanup procedure state.
@@ -227,6 +227,7 @@ impl Runner {
let ctx = Context {
procedure_id: self.meta.id,
provider: self.manager_ctx.clone(),
event_context: self.meta.context.event_context.clone(),
};
self.rolling_back = false;
@@ -508,6 +509,7 @@ impl Runner {
procedure_id,
procedure_state,
Some(self.meta.id),
self.meta.context.clone(),
procedure.lock_key(),
procedure.poison_keys(),
procedure.type_name(),
@@ -662,12 +664,13 @@ impl Runner {
let data = self.procedure.dump()?;
self.store
.store_procedure(
.store_procedure_with_context(
self.meta.id,
self.step,
type_name,
data,
self.meta.parent_id,
self.meta.context.clone(),
)
.await
.map_err(|e| {
@@ -708,6 +711,7 @@ impl Runner {
parent_id: self.meta.parent_id,
step: self.step,
error: Some(error),
context: self.meta.context.clone(),
};
self.store
.rollback_procedure(self.meta.id, message)
@@ -776,15 +780,22 @@ impl Runner {
pub(crate) fn build_event(&self, trigger: EventTrigger) -> Option<ProcedureEvent> {
let recorder = self.event_recorder.as_ref()?;
let state = self.meta.state();
let context = EventRuntimeContext {
let context = EventContext {
procedure_id: self.meta.id,
lifecycle_state: &state,
trigger: trigger.clone(),
event_type_filter: recorder.event_type_filter(),
event_context: self.meta.context.event_context.as_ref(),
};
self.procedure
.event(&context)
.map(|event| ProcedureEvent::new(self.meta.id, event, state, trigger))
self.procedure.event(&context).map(|event| {
ProcedureEvent::new_with_context(
self.meta.id,
event,
state,
trigger,
self.meta.context.clone(),
)
})
}
/// Builds and dispatches an event from the live procedure. Delivery is best effort and is
@@ -876,6 +887,7 @@ mod tests {
Context {
procedure_id,
provider,
event_context: None,
}
}
@@ -914,6 +926,7 @@ mod tests {
Context {
procedure_id,
provider: Arc::new(MockProvider),
event_context: None,
}
}
@@ -1100,6 +1113,7 @@ mod tests {
ProcedureWithId {
id: procedure_id,
procedure: Box::new(child),
context: Default::default(),
}
}
@@ -1480,6 +1494,7 @@ mod tests {
subprocedures: vec![ProcedureWithId {
id: child_id,
procedure: Box::new(fail),
context: Default::default(),
}],
persist: true,
})
@@ -1672,6 +1687,7 @@ mod tests {
subprocedures: vec![ProcedureWithId {
id: child_id,
procedure: Box::new(fail),
context: Default::default(),
}],
persist: true,
})
@@ -2192,6 +2208,7 @@ mod tests {
subprocedures: vec![ProcedureWithId {
id: child_id,
procedure: Box::new(child),
context: Default::default(),
}],
persist: false,
})
@@ -2248,6 +2265,7 @@ mod tests {
subprocedures: vec![ProcedureWithId {
id: child_id,
procedure: Box::new(child),
context: Default::default(),
}],
persist: false,
})
src/common/procedure/src/procedure.rs
+39 -5
View File
@@ -19,7 +19,7 @@ use std::str::FromStr;
use std::sync::Arc;
use async_trait::async_trait;
use common_event_recorder::{Event, EventTypeFilterRef};
use common_event_recorder::{Event, EventTypeFilterRef, PersistentEventContext};
use serde::{Deserialize, Serialize};
use smallvec::{SmallVec, smallvec};
use snafu::{ResultExt, Snafu};
@@ -32,6 +32,26 @@ use crate::watcher::Watcher;
pub type Output = Arc<dyn Any + Send + Sync>;
/// Context attached to a procedure submission and inherited by its children.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProcedureContext {
/// Context describing why and how the procedure was submitted.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub event_context: Option<PersistentEventContext>,
}
impl ProcedureContext {
pub fn from_event_context(event_context: PersistentEventContext) -> Self {
Self {
event_context: Some(event_context),
}
}
pub fn is_empty(&self) -> bool {
self.event_context.is_none()
}
}
/// Procedure execution status.
#[derive(Debug)]
pub enum Status {
@@ -175,6 +195,8 @@ pub struct Context {
pub procedure_id: ProcedureId,
/// [ProcedureManager] context provider.
pub provider: ContextProviderRef,
/// Event context inherited from the root submission.
pub event_context: Option<PersistentEventContext>,
}
impl Context {
@@ -237,13 +259,13 @@ pub trait Procedure: Send {
/// [`Event::extra_schema`] values. The event recorder batches events by type
/// and rejects incompatible schemas; use a distinct event type for a
/// different schema.
fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, _ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
None
}
}
/// Framework-owned context supplied when a procedure builds a lifecycle event.
pub struct EventRuntimeContext<'a> {
pub struct EventContext<'a> {
/// Id of the procedure associated with the event.
pub procedure_id: ProcedureId,
/// Current framework state of the procedure.
@@ -252,6 +274,8 @@ pub struct EventRuntimeContext<'a> {
pub trigger: EventTrigger,
/// Event types retained by the configured recorder.
pub event_type_filter: EventTypeFilterRef,
/// Event context inherited from the root submission.
pub event_context: Option<&'a PersistentEventContext>,
}
/// Lifecycle action that causes the framework to invoke [`Procedure::event`].
@@ -336,7 +360,7 @@ impl<T: Procedure + ?Sized> Procedure for Box<T> {
(**self).poison_keys()
}
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
(**self).event(ctx)
}
}
@@ -459,6 +483,8 @@ pub struct ProcedureWithId {
/// Id of the procedure.
pub id: ProcedureId,
pub procedure: BoxedProcedure,
/// Context associated with this procedure submission.
pub context: ProcedureContext,
}
impl ProcedureWithId {
@@ -468,8 +494,15 @@ impl ProcedureWithId {
ProcedureWithId {
id: ProcedureId::random(),
procedure,
context: ProcedureContext::default(),
}
}
/// Attaches event context to this procedure submission.
pub fn with_event_context(mut self, event_context: PersistentEventContext) -> Self {
self.context.event_context = Some(event_context);
self
}
}
impl fmt::Debug for ProcedureWithId {
@@ -721,11 +754,12 @@ mod tests {
#[test]
fn test_default_procedure_event_hook() {
let state = ProcedureState::Running;
let context = EventRuntimeContext {
let context = EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Succeeded,
event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
event_context: None,
};
assert!(DefaultEventProcedure.event(&context).is_none());
src/common/procedure/src/store.rs
+45 -1
View File
@@ -20,9 +20,9 @@ use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
use snafu::ResultExt;
use crate::ProcedureId;
use crate::error::{Result, ToJsonSnafu};
pub(crate) use crate::store::state_store::StateStoreRef;
use crate::{ProcedureContext, ProcedureId};
pub mod poison_store;
pub mod state_store;
@@ -55,6 +55,9 @@ pub struct ProcedureMessage {
/// Errors raised during the procedure.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
/// Context associated with the root submission.
#[serde(default, skip_serializing_if = "ProcedureContext::is_empty")]
pub context: ProcedureContext,
}
/// A collection of all procedures' messages.
@@ -88,6 +91,7 @@ impl ProcedureStore {
}
/// Dump the `procedure` to the storage.
#[cfg(test)]
pub(crate) async fn store_procedure(
&self,
procedure_id: ProcedureId,
@@ -95,6 +99,26 @@ impl ProcedureStore {
type_name: String,
data: String,
parent_id: Option<ProcedureId>,
) -> Result<()> {
self.store_procedure_with_context(
procedure_id,
step,
type_name,
data,
parent_id,
ProcedureContext::default(),
)
.await
}
pub(crate) async fn store_procedure_with_context(
&self,
procedure_id: ProcedureId,
step: u32,
type_name: String,
data: String,
parent_id: Option<ProcedureId>,
context: ProcedureContext,
) -> Result<()> {
let message = ProcedureMessage {
type_name,
@@ -102,6 +126,7 @@ impl ProcedureStore {
parent_id,
step,
error: None,
context,
};
let key = ParsedKey {
prefix: &self.proc_path,
@@ -356,6 +381,7 @@ mod tests {
}
use async_trait::async_trait;
use common_event_recorder::{PersistentEventContext, TriggerReason};
use common_test_util::temp_dir::{TempDir, create_temp_dir};
use object_store::services::Fs as Builder;
@@ -462,6 +488,7 @@ mod tests {
parent_id: None,
step: 4,
error: None,
context: ProcedureContext::default(),
};
let json = serde_json::to_string(&message).unwrap();
@@ -477,6 +504,21 @@ mod tests {
json,
r#"{"type_name":"TestMessage","data":"no parent id","parent_id":"9f805a1f-05f7-490c-9f91-bd56e3cc54c1","step":4}"#
);
message.context = ProcedureContext::from_event_context(PersistentEventContext::new(
TriggerReason::AutoRebalance,
));
let json = serde_json::to_string(&message).unwrap();
assert_eq!(
json,
r#"{"type_name":"TestMessage","data":"no parent id","parent_id":"9f805a1f-05f7-490c-9f91-bd56e3cc54c1","step":4,"context":{"event_context":{"reason":"auto_rebalance"}}}"#
);
let legacy: ProcedureMessage = serde_json::from_str(
r#"{"type_name":"TestMessage","data":"legacy","parent_id":null,"step":1}"#,
)
.unwrap();
assert_eq!(legacy.context, ProcedureContext::default());
}
struct MockProcedure {
@@ -541,6 +583,7 @@ mod tests {
parent_id: None,
step: 0,
error: None,
context: ProcedureContext::default(),
};
assert_eq!(expect, *msg);
}
@@ -589,6 +632,7 @@ mod tests {
parent_id: None,
step: 1,
error: None,
context: ProcedureContext::default(),
};
store
.rollback_procedure(procedure_id, message)
src/common/procedure/src/watcher.rs
+1
View File
@@ -124,6 +124,7 @@ mod tests {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(MockProcedure { error: true }),
context: Default::default(),
})
.await
.unwrap();
src/meta-srv/src/event/gc.rs
+37 -49
View File
@@ -21,10 +21,9 @@ use api::v1::{ColumnSchema, Row, Value};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
EVENT_CONTEXT_COLUMN, GC_REPORT_COLUMN, REGION_ID_COLUMN, REGION_NUMBER_COLUMN,
TABLE_ID_COLUMN, column_schemas, nullable_json,
GC_REPORT_COLUMN, REGION_ID_COLUMN, REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas,
nullable_json,
};
use common_meta::rpc::ddl::EventContext;
use serde::Serialize;
use snafu::ResultExt;
use store_api::storage::{GcReport, IndexVersion, RegionId};
@@ -67,7 +66,6 @@ struct BatchGcRegionRow {
#[derive(Debug)]
pub(crate) struct BatchGcEvent {
payload: Option<BatchGcPayload>,
event_context: Option<EventContext>,
// None emits a procedure-level lifecycle row with null Region dimensions.
regions: Option<Vec<BatchGcRegionRow>>,
}
@@ -77,7 +75,6 @@ impl BatchGcEvent {
regions: &[RegionId],
full_file_listing: bool,
timeout: Duration,
event_context: Option<EventContext>,
) -> Self {
Self {
payload: Some(BatchGcPayload {
@@ -86,7 +83,6 @@ impl BatchGcEvent {
full_file_listing,
timeout,
}),
event_context,
regions: None,
}
}
@@ -111,7 +107,6 @@ impl BatchGcEvent {
(!regions.is_empty()).then_some(Self {
payload: None,
event_context: None,
regions: Some(regions),
})
}
@@ -136,22 +131,8 @@ impl Event for BatchGcEvent {
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
let Some(regions) = &self.regions else {
return Ok(vec![Row {
values: vec![
Value { value_data: None },
Value { value_data: None },
Value { value_data: None },
Value { value_data: None },
nullable_json(event_context.as_ref()),
],
}]);
return Ok(vec![null_row()]);
};
regions
@@ -164,7 +145,6 @@ impl Event for BatchGcEvent {
ValueData::U32Value(region.region_id.table_id()).into(),
ValueData::U32Value(region.region_id.region_number()).into(),
nullable_json(Some(&report)),
nullable_json(None),
],
})
})
@@ -182,11 +162,9 @@ fn schema() -> Vec<ColumnSchema> {
&TABLE_ID_COLUMN,
&REGION_NUMBER_COLUMN,
&GC_REPORT_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
#[cfg(test)]
fn null_row() -> Row {
Row {
values: (0..schema().len())
@@ -233,17 +211,17 @@ mod tests {
use std::sync::Arc;
use api::v1::ColumnSchema;
use common_event_recorder::EventTypeFilter;
use common_event_recorder::event_table::{
PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN,
EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN,
PROCEDURE_TRIGGER_COLUMN, jsonb_value,
};
use common_event_recorder::testing::assert_event_contract;
use common_event_recorder::{EventTypeFilter, PersistentEventContext, TriggerReason};
use common_meta::key::TableMetadataManager;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::sequence::SequenceBuilder;
use common_procedure::{
EventRuntimeContext, EventTrigger, Procedure, ProcedureEvent, ProcedureId, ProcedureState,
EventContext, EventTrigger, Procedure, ProcedureEvent, ProcedureId, ProcedureState,
RetryPhase,
};
use store_api::storage::FileId;
@@ -256,12 +234,8 @@ mod tests {
fn test_batch_gc_lifecycle_event_contract() {
let first = RegionId::new(1024, 1);
let second = RegionId::new(1024, 2);
let event = BatchGcEvent::with_config(
&[second, first, second],
true,
Duration::from_secs(10),
None,
);
let event =
BatchGcEvent::with_config(&[second, first, second], true, Duration::from_secs(10));
assert_event_contract(&event, BATCH_GC_EVENT_TYPE, &schema(), &[null_row()]);
assert_eq!(
@@ -356,22 +330,35 @@ mod tests {
fn test_batch_gc_event_filter() {
let procedure = batch_gc_procedure();
let running = ProcedureState::Running;
let runtime_context = |trigger, lifecycle_state, event_type_filter| EventRuntimeContext {
let manual_context = PersistentEventContext::new(TriggerReason::Manual);
let event_context = |trigger, lifecycle_state, event_type_filter| EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state,
trigger,
event_type_filter: Arc::new(event_type_filter),
event_context: None,
};
assert!(
procedure
.event(&runtime_context(
.event(&event_context(
EventTrigger::Submitted,
&running,
EventTypeFilter::All,
))
.is_none()
);
assert!(
procedure
.event(&EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &running,
trigger: EventTrigger::Submitted,
event_type_filter: Arc::new(EventTypeFilter::All),
event_context: Some(&manual_context),
})
.is_some()
);
let report = GcReport {
processed_regions: HashSet::from([RegionId::new(1024, 1)]),
@@ -382,7 +369,7 @@ mod tests {
};
assert!(
procedure
.event(&runtime_context(
.event(&event_context(
EventTrigger::Succeeded,
&done,
EventTypeFilter::All,
@@ -392,7 +379,7 @@ mod tests {
assert!(
procedure
.event(&runtime_context(
.event(&event_context(
EventTrigger::Recovered,
&running,
EventTypeFilter::All,
@@ -401,7 +388,7 @@ mod tests {
);
let retrying = procedure
.event(&runtime_context(
.event(&event_context(
EventTrigger::Retrying {
phase: RetryPhase::Execute,
attempt: 1,
@@ -423,7 +410,7 @@ mod tests {
assert!(
procedure
.event(&runtime_context(
.event(&event_context(
EventTrigger::Recovered,
&running,
EventTypeFilter::Only(HashSet::from(["another_event".to_string()])),
@@ -432,7 +419,7 @@ mod tests {
);
assert!(
procedure
.event(&runtime_context(
.event(&event_context(
EventTrigger::Recovered,
&running,
EventTypeFilter::Only(HashSet::new()),
@@ -443,7 +430,7 @@ mod tests {
let missing = ProcedureState::Done { output: None };
assert!(
procedure
.event(&runtime_context(
.event(&event_context(
EventTrigger::Succeeded,
&missing,
EventTypeFilter::All,
@@ -455,7 +442,7 @@ mod tests {
};
assert!(
procedure
.event(&runtime_context(
.event(&event_context(
EventTrigger::Succeeded,
&wrong,
EventTypeFilter::All,
@@ -468,11 +455,12 @@ mod tests {
fn test_batch_gc_terminal_event_with_report() {
let mut procedure = batch_gc_procedure();
let running = ProcedureState::Running;
let runtime_context = |trigger| EventRuntimeContext {
let event_context = |trigger| EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &running,
trigger,
event_type_filter: Arc::new(EventTypeFilter::All),
event_context: None,
};
let report = GcReport {
need_retry_regions: HashSet::from([RegionId::new(1024, 2)]),
@@ -481,7 +469,7 @@ mod tests {
procedure.set_gc_report_for_test(report);
let retrying = procedure
.event(&runtime_context(EventTrigger::Retrying {
.event(&event_context(EventTrigger::Retrying {
phase: RetryPhase::Execute,
attempt: 2,
}))
@@ -498,7 +486,7 @@ mod tests {
assert_eq!(retrying.extra_rows().unwrap(), vec![null_row()]);
for trigger in [EventTrigger::Failed, EventTrigger::Poisoned] {
let event = procedure.event(&runtime_context(trigger)).unwrap();
let event = procedure.event(&event_context(trigger)).unwrap();
assert_eq!(event.json_payload().unwrap(), serde_json::Value::Null);
assert_eq!(
event.extra_rows().unwrap(),
@@ -513,7 +501,7 @@ mod tests {
let procedure = batch_gc_procedure();
let failed_without_report = procedure
.event(&runtime_context(EventTrigger::Failed))
.event(&event_context(EventTrigger::Failed))
.unwrap();
assert_eq!(
failed_without_report.json_payload().unwrap(),
@@ -548,6 +536,7 @@ mod tests {
);
let mut event_schema = procedure_schema();
event_schema.extend(schema());
event_schema.push(EVENT_CONTEXT_COLUMN.column_schema());
let mut values = vec![
ValueData::StringValue(procedure_id.to_string()).into(),
ValueData::StringValue("Done".to_string()).into(),
@@ -564,6 +553,7 @@ mod tests {
)
.values,
);
values.push(Value { value_data: None });
assert_event_contract(
&event,
@@ -589,7 +579,6 @@ mod tests {
ValueData::U32Value(region_id.table_id()).into(),
ValueData::U32Value(region_id.region_number()).into(),
nullable_json(report.as_ref()),
nullable_json(None),
],
}
}
@@ -607,7 +596,6 @@ mod tests {
true,
Duration::from_secs(10),
HashMap::new(),
EventContext::default(),
)
}
}
src/meta-srv/src/event/region_migration.rs
+6 -16
View File
@@ -17,14 +17,13 @@ use std::time::Duration;
use api::v1::value::ValueData;
use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
EVENT_CONTEXT_COLUMN, REGION_ID_COLUMN, REGION_MIGRATION_DST_NODE_ID_COLUMN,
REGION_MIGRATION_DST_PEER_ADDR_COLUMN, REGION_MIGRATION_SRC_NODE_ID_COLUMN,
REGION_MIGRATION_SRC_PEER_ADDR_COLUMN, REGION_MIGRATION_TRIGGER_REASON_COLUMN,
REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas, nullable_json,
REGION_ID_COLUMN, REGION_MIGRATION_DST_NODE_ID_COLUMN, REGION_MIGRATION_DST_PEER_ADDR_COLUMN,
REGION_MIGRATION_SRC_NODE_ID_COLUMN, REGION_MIGRATION_SRC_PEER_ADDR_COLUMN,
REGION_MIGRATION_TRIGGER_REASON_COLUMN, REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas,
};
use common_event_recorder::{Event, TriggerReason};
use serde::Serialize;
use snafu::ResultExt;
use store_api::storage::RegionId;
@@ -50,7 +49,6 @@ pub(crate) struct RegionMigrationEvent {
dst_peer_addr: String,
// The timeout of the region migration.
timeout: Duration,
event_context: Option<common_meta::rpc::ddl::EventContext>,
}
#[derive(Debug, Serialize)]
@@ -60,16 +58,15 @@ struct Payload {
}
impl RegionMigrationEvent {
pub fn from_persistent_ctx(ctx: &PersistentContext, include_event_context: bool) -> Self {
pub fn from_persistent_ctx(ctx: &PersistentContext, trigger_reason: TriggerReason) -> Self {
Self {
region_ids: ctx.region_ids.clone(),
trigger_reason: ctx.trigger_reason,
trigger_reason: RegionMigrationTriggerReason::from_trigger_reason(trigger_reason),
src_node_id: ctx.from_peer.id,
src_peer_addr: ctx.from_peer.addr.clone(),
dst_node_id: ctx.to_peer.id,
dst_peer_addr: ctx.to_peer.addr.clone(),
timeout: ctx.timeout,
event_context: include_event_context.then(|| ctx.event_context.clone()),
}
}
}
@@ -89,7 +86,6 @@ impl Event for RegionMigrationEvent {
&REGION_MIGRATION_SRC_PEER_ADDR_COLUMN,
&REGION_MIGRATION_DST_NODE_ID_COLUMN,
&REGION_MIGRATION_DST_PEER_ADDR_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
@@ -106,12 +102,6 @@ impl Event for RegionMigrationEvent {
ValueData::StringValue(self.src_peer_addr.clone()).into(),
ValueData::U64Value(self.dst_node_id).into(),
ValueData::StringValue(self.dst_peer_addr.clone()).into(),
self.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)
.map(|value| nullable_json(value.as_ref()))?,
],
});
}
src/meta-srv/src/event/repartition.rs
+11 -35
View File
@@ -21,11 +21,11 @@ use api::v1::{ColumnSchema, Row};
use common_event_recorder::Event;
use common_event_recorder::error::{Result, SerializeEventSnafu};
use common_event_recorder::event_table::{
CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PARENT_PROCEDURE_ID_COLUMN,
REPARTITION_GROUP_ID_COLUMN, SCHEMA_NAME_COLUMN, SOURCE_PARTITION_EXPR_COLUMN,
SOURCE_REGION_ID_COLUMN, SOURCE_REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN,
TARGET_PARTITION_EXPR_COLUMN, TARGET_REGION_ID_COLUMN, TARGET_REGION_NUMBER_COLUMN,
column_schemas, nullable_json, nullable_string, nullable_value,
CATALOG_NAME_COLUMN, PARENT_PROCEDURE_ID_COLUMN, REPARTITION_GROUP_ID_COLUMN,
SCHEMA_NAME_COLUMN, SOURCE_PARTITION_EXPR_COLUMN, SOURCE_REGION_ID_COLUMN,
SOURCE_REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN, TARGET_PARTITION_EXPR_COLUMN,
TARGET_REGION_ID_COLUMN, TARGET_REGION_NUMBER_COLUMN, column_schemas, nullable_string,
nullable_value,
};
use serde::Serialize;
use snafu::ResultExt;
@@ -61,7 +61,6 @@ pub(crate) struct RepartitionEvent {
table_name: Option<String>,
table_id: Option<TableId>,
payload: Option<RepartitionSubmittedPayload>,
event_context: Option<common_meta::rpc::ddl::EventContext>,
}
impl RepartitionEvent {
@@ -91,7 +90,6 @@ impl RepartitionEvent {
target_partition_columns: intent.target_partition_columns().map(ToOwned::to_owned),
timeout: persistent_ctx.timeout,
}),
event_context: Some(persistent_ctx.event_context.clone()),
}
}
@@ -102,17 +100,10 @@ impl RepartitionEvent {
table_name: Some(persistent_ctx.table_name.clone()),
table_id: Some(persistent_ctx.table_id),
payload: None,
event_context: None,
}
}
fn schema() -> Vec<ColumnSchema> {
let mut schema = Self::base_schema();
schema.push(EVENT_CONTEXT_COLUMN.column_schema());
schema
}
fn base_schema() -> Vec<ColumnSchema> {
column_schemas([
&CATALOG_NAME_COLUMN,
&SCHEMA_NAME_COLUMN,
@@ -141,19 +132,12 @@ impl Event for RepartitionEvent {
}
fn extra_rows(&self) -> Result<Vec<Row>> {
let event_context = self
.event_context
.as_ref()
.map(serde_json::to_value)
.transpose()
.context(SerializeEventSnafu)?;
Ok(vec![Row {
values: vec![
nullable_string(self.catalog_name.as_deref()),
nullable_string(self.schema_name.as_deref()),
nullable_string(self.table_name.as_deref()),
nullable_value(self.table_id.map(ValueData::U32Value)),
nullable_json(event_context.as_ref()),
],
}])
}
@@ -323,7 +307,7 @@ impl RepartitionGroupEvent {
}
fn schema() -> Vec<ColumnSchema> {
let mut schema = RepartitionEvent::base_schema();
let mut schema = RepartitionEvent::schema();
schema.extend(column_schemas([
&PARENT_PROCEDURE_ID_COLUMN,
&REPARTITION_GROUP_ID_COLUMN,
@@ -377,14 +361,13 @@ mod tests {
use std::time::Duration;
use api::v1::value::ValueData;
use api::v1::{ColumnSchema, Row};
use api::v1::{ColumnSchema, Row, Value};
use common_event_recorder::Event;
use common_event_recorder::event_table::{
PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN,
EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN,
PROCEDURE_TRIGGER_COLUMN, jsonb_value,
};
use common_event_recorder::testing::assert_event_contract;
use common_meta::rpc::ddl::EventContext;
use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState};
use table::table_name::TableName;
use uuid::Uuid;
@@ -402,7 +385,6 @@ mod tests {
TableName::new("greptime", "public", "repartition_events"),
1024,
Some(Duration::from_secs(30)),
EventContext::default(),
)
}
@@ -438,9 +420,6 @@ mod tests {
ValueData::StringValue("public".to_string()).into(),
ValueData::StringValue("repartition_events".to_string()).into(),
ValueData::U32Value(1024).into(),
jsonb_value(&serde_json::json!({
"reason": "unknown",
})),
],
}],
);
@@ -654,7 +633,6 @@ mod tests {
ValueData::StringValue("public".to_string()).into(),
ValueData::StringValue("repartition_events".to_string()).into(),
ValueData::U32Value(1024).into(),
Default::default(),
],
}],
);
@@ -690,6 +668,7 @@ mod tests {
);
let mut schema = procedure_schema();
schema.extend(parent_schema());
schema.push(EVENT_CONTEXT_COLUMN.column_schema());
assert_event_contract(
&event,
@@ -705,9 +684,7 @@ mod tests {
ValueData::StringValue("public".to_string()).into(),
ValueData::StringValue("repartition_events".to_string()).into(),
ValueData::U32Value(1024).into(),
jsonb_value(&serde_json::json!({
"reason": "unknown",
})),
Value { value_data: None },
],
}],
);
@@ -719,12 +696,11 @@ mod tests {
&SCHEMA_NAME_COLUMN,
&TABLE_NAME_COLUMN,
&TABLE_ID_COLUMN,
&EVENT_CONTEXT_COLUMN,
])
}
fn group_schema() -> Vec<ColumnSchema> {
let mut schema = RepartitionEvent::base_schema();
let mut schema = parent_schema();
schema.extend(column_schemas([
&PARENT_PROCEDURE_ID_COLUMN,
&REPARTITION_GROUP_ID_COLUMN,
src/meta-srv/src/gc/ctx.rs
+6 -6
View File
@@ -29,7 +29,7 @@ use common_meta::key::DroppedTableName;
use common_meta::key::TableMetadataManagerRef;
use common_meta::key::table_repart::TableRepartValue;
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::ddl::PersistentEventContext;
#[cfg(feature = "enterprise")]
use common_meta::rpc::ddl::PurgeDroppedTableTask;
use common_procedure::{ProcedureManagerRef, ProcedureWithId, watcher};
@@ -68,7 +68,7 @@ pub(crate) trait SchedulerCtx: Send + Sync {
full_file_listing: bool,
timeout: Duration,
region_routes_override: Region2Peers,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<GcReport>;
#[cfg(feature = "enterprise")]
@@ -256,7 +256,7 @@ impl SchedulerCtx for DefaultGcSchedulerCtx {
full_file_listing: bool,
timeout: Duration,
region_routes_override: Region2Peers,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<GcReport> {
self.gc_regions_inner(
region_ids,
@@ -310,7 +310,7 @@ impl DefaultGcSchedulerCtx {
full_file_listing: bool,
timeout: Duration,
region_routes_override: Region2Peers,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<GcReport> {
debug!(
"Sending GC instruction for {} regions (full_file_listing: {})",
@@ -326,9 +326,9 @@ impl DefaultGcSchedulerCtx {
full_file_listing,
timeout,
region_routes_override,
event_context,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
let id = procedure_with_id.id;
src/meta-srv/src/gc/handler.rs
+7 -4
View File
@@ -18,7 +18,7 @@ use std::time::Instant;
use common_catalog::consts::MITO_ENGINE;
use common_meta::datanode::{RegionManifestInfo, RegionStat};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::ddl::PersistentEventContext;
use common_telemetry::tracing::Instrument as _;
use common_telemetry::{debug, error, info, warn};
use futures::StreamExt;
@@ -37,7 +37,10 @@ use crate::gc::tracker::RegionGcInfo;
use crate::metrics::METRIC_META_GC_CANDIDATE_REGIONS;
impl GcScheduler {
pub(crate) async fn trigger_gc(&self, event_context: EventContext) -> Result<GcJobReport> {
pub(crate) async fn trigger_gc(
&self,
event_context: PersistentEventContext,
) -> Result<GcJobReport> {
let start_time = Instant::now();
info!("Starting GC cycle");
@@ -212,7 +215,7 @@ impl GcScheduler {
datanode_to_candidates: HashMap<Peer, Vec<(TableId, GcCandidate)>>,
force_full_listing_by_peer: HashMap<Peer, HashSet<RegionId>>,
region_routes_override_by_peer: HashMap<Peer, Region2Peers>,
event_context: EventContext,
event_context: PersistentEventContext,
) -> GcJobReport {
let mut per_datanode_reports = HashMap::new();
let mut failed_datanodes: HashMap<_, Vec<_>> = HashMap::new();
@@ -283,7 +286,7 @@ impl GcScheduler {
candidates: Vec<(TableId, GcCandidate)>,
force_full_listing: HashSet<RegionId>,
region_routes_override: Region2Peers,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<GcReport> {
info!(
"Starting GC for datanode {} with {} candidate regions",
src/meta-srv/src/gc/mock.rs
+3 -3
View File
@@ -30,7 +30,7 @@ use common_meta::datanode::{RegionManifestInfo, RegionStat};
use common_meta::key::table_repart::TableRepartValue;
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::ddl::PersistentEventContext;
use common_meta::rpc::router::{Region, RegionRoute};
use common_telemetry::debug;
use ordered_float::OrderedFloat;
@@ -75,7 +75,7 @@ pub struct MockSchedulerCtx {
pub candidates: Arc<Mutex<Option<HashMap<TableId, Vec<GcCandidate>>>>>,
pub get_table_to_region_stats_calls: Arc<Mutex<usize>>,
pub gc_regions_calls: Arc<Mutex<usize>>,
pub gc_event_contexts: Arc<Mutex<Vec<EventContext>>>,
pub gc_event_contexts: Arc<Mutex<Vec<PersistentEventContext>>>,
// Error injection fields for testing
pub get_table_to_region_stats_error: Arc<Mutex<Option<crate::error::Error>>>,
pub get_table_route_error: Arc<Mutex<Option<crate::error::Error>>>,
@@ -214,7 +214,7 @@ impl SchedulerCtx for MockSchedulerCtx {
_full_file_listing: bool,
_timeout: Duration,
_region_routes_override: Region2Peers,
event_context: EventContext,
event_context: PersistentEventContext,
) -> Result<GcReport> {
*self.gc_regions_calls.lock().unwrap() += 1;
self.gc_event_contexts.lock().unwrap().push(event_context);
src/meta-srv/src/gc/mock/basic.rs
+5 -5
View File
@@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex};
use std::time::Instant;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::ddl::{PersistentEventContext, TriggerReason};
use common_telemetry::init_default_ut_logging;
use store_api::region_engine::RegionRole;
use store_api::storage::{FileId, FileRefsManifest, GcReport, RegionId};
@@ -36,7 +36,7 @@ async fn test_parallel_process_datanodes_empty() {
HashMap::new(),
HashMap::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await;
@@ -105,7 +105,7 @@ async fn test_parallel_process_datanodes_with_candidates() {
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await;
@@ -189,7 +189,7 @@ async fn test_handle_tick() {
assert_eq!(*ctx.gc_regions_calls.lock().unwrap(), 1);
assert_eq!(
ctx.gc_event_contexts.lock().unwrap().as_slice(),
&[EventContext::new(TriggerReason::ScheduledGc)]
&[PersistentEventContext::new(TriggerReason::ScheduledGc)]
);
let tracker = scheduler.region_gc_tracker.lock().await;
@@ -249,6 +249,6 @@ async fn test_handle_manual_gc_without_regions_records_manual_event_context() {
assert_eq!(
ctx.gc_event_contexts.lock().unwrap().as_slice(),
&[EventContext::new(TriggerReason::Manual)]
&[PersistentEventContext::new(TriggerReason::Manual)]
);
}
src/meta-srv/src/gc/mock/concurrent.rs
+6 -6
View File
@@ -18,7 +18,7 @@ use std::time::{Duration, Instant};
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::ddl::PersistentEventContext;
use common_meta::rpc::router::{Region, RegionRoute};
use common_telemetry::{info, init_default_ut_logging};
use store_api::region_engine::RegionRole;
@@ -102,7 +102,7 @@ async fn test_concurrent_table_processing_limits() {
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await;
@@ -190,7 +190,7 @@ async fn test_datanode_processes_tables_with_partial_gc_failures() {
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await;
@@ -296,7 +296,7 @@ async fn test_region_gc_concurrency_limit() {
candidates.into_iter().map(|c| (table_id, c)).collect(),
HashSet::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await
.unwrap();
@@ -413,7 +413,7 @@ async fn test_region_gc_concurrency_with_partial_failures() {
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await;
@@ -557,7 +557,7 @@ async fn test_region_gc_concurrency_with_retryable_errors() {
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await;
src/meta-srv/src/gc/mock/full_list.rs
+6 -6
View File
@@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::ddl::PersistentEventContext;
use common_telemetry::init_default_ut_logging;
use store_api::region_engine::RegionRole;
use store_api::storage::{FileId, FileRefsManifest, GcReport, RegionId};
@@ -83,7 +83,7 @@ async fn test_full_file_listing_first_time_gc() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await
.unwrap();
@@ -157,7 +157,7 @@ async fn test_full_file_listing_interval_enforcement() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await
.unwrap();
@@ -184,7 +184,7 @@ async fn test_full_file_listing_interval_enforcement() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await
.unwrap();
@@ -260,7 +260,7 @@ async fn test_full_file_listing_no_interval_passed() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await
.unwrap();
@@ -284,7 +284,7 @@ async fn test_full_file_listing_no_interval_passed() {
vec![(table_id, mock_candidate(region_id))],
HashSet::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await
.unwrap();
src/meta-srv/src/gc/mock/misc.rs
+3 -3
View File
@@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex};
use std::time::Instant;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::ddl::PersistentEventContext;
use common_telemetry::init_default_ut_logging;
use store_api::storage::{FileRefsManifest, GcReport, RegionId};
@@ -76,7 +76,7 @@ async fn test_empty_file_refs_manifest() {
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await;
@@ -169,7 +169,7 @@ async fn test_multiple_regions_per_table() {
datanode_to_candidates,
HashMap::new(),
HashMap::new(),
EventContext::default(),
PersistentEventContext::default(),
)
.await;
src/meta-srv/src/gc/procedure.rs
+7 -29
View File
@@ -24,11 +24,11 @@ use common_meta::key::table_repart::TableRepartValue;
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::lock_key::{RegionLock, TableLock};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::ddl::TriggerReason;
use common_procedure::error::ToJsonSnafu;
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger,
LockKey, Procedure, ProcedureState, Result as ProcedureResult, Status,
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, ProcedureState, Result as ProcedureResult, Status,
};
use common_telemetry::tracing::Instrument as _;
use common_telemetry::tracing_context::TracingContext;
@@ -200,8 +200,6 @@ pub struct BatchGcData {
/// mailbox timeout duration
timeout: Duration,
gc_report: Option<GcReport>,
#[serde(default)]
event_context: EventContext,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
@@ -219,7 +217,6 @@ pub enum State {
impl BatchGcProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::BatchGcProcedure";
#[allow(clippy::too_many_arguments)]
pub fn new(
mailbox: MailboxRef,
table_metadata_manager: TableMetadataManagerRef,
@@ -228,7 +225,6 @@ impl BatchGcProcedure {
full_file_listing: bool,
timeout: Duration,
region_routes_override: Region2Peers,
event_context: EventContext,
) -> Self {
Self {
mailbox,
@@ -244,7 +240,6 @@ impl BatchGcProcedure {
related_regions: HashMap::new(),
file_refs: FileRefsManifest::default(),
gc_report: None,
event_context,
},
}
}
@@ -275,7 +270,6 @@ impl BatchGcProcedure {
related_regions: HashMap::new(),
file_refs,
gc_report: Some(GcReport::default()),
event_context: EventContext::default(),
},
}
}
@@ -1072,23 +1066,21 @@ impl Procedure for BatchGcProcedure {
LockKey::new(lock_key)
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(BATCH_GC_EVENT_TYPE) {
return None;
}
let event = match &ctx.trigger {
// Keep scheduled GC low-noise; record submitted manual requests for auditability.
EventTrigger::Submitted => (self.data.event_context.reason == TriggerReason::Manual)
EventTrigger::Submitted => ctx
.event_context
.is_some_and(|context| context.reason == TriggerReason::Manual)
.then(|| {
BatchGcEvent::with_config(
&self.data.regions,
self.data.full_file_listing,
self.data.timeout,
Some(self.data.event_context.clone()),
)
})?,
EventTrigger::Recovered | EventTrigger::ChildSubmitted { .. } => return None,
@@ -1106,7 +1098,6 @@ impl Procedure for BatchGcProcedure {
&self.data.regions,
self.data.full_file_listing,
self.data.timeout,
None,
),
EventTrigger::Failed | EventTrigger::Poisoned => self
.data
@@ -1118,7 +1109,6 @@ impl Procedure for BatchGcProcedure {
&self.data.regions,
self.data.full_file_listing,
self.data.timeout,
None,
)
}),
};
@@ -1211,16 +1201,6 @@ mod tests {
assert!(report.need_retry_regions.contains(&region_id));
}
#[test]
fn test_batch_gc_data_defaults_missing_event_context() {
let mut data = serde_json::to_value(&batch_gc_procedure().data).unwrap();
data.as_object_mut().unwrap().remove("event_context");
let recovered: BatchGcData = serde_json::from_value(data).unwrap();
assert_eq!(recovered.event_context, EventContext::default());
}
#[tokio::test]
async fn test_send_gc_instructions_preserves_partial_report() {
let first_region = RegionId::new(1024, 1);
@@ -1255,7 +1235,6 @@ mod tests {
true,
Duration::from_secs(10),
HashMap::new(),
EventContext::default(),
);
procedure.data.region_routes = HashMap::from([
(first_region, (first_peer, vec![])),
@@ -1296,7 +1275,6 @@ mod tests {
true,
Duration::from_secs(10),
HashMap::new(),
EventContext::default(),
)
}
}
src/meta-srv/src/gc/scheduler.rs
+8 -8
View File
@@ -18,7 +18,7 @@ use std::time::{Duration, Instant};
use common_meta::DatanodeId;
use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::ddl::{PersistentEventContext, TriggerReason};
use common_telemetry::tracing::Instrument as _;
use common_telemetry::{error, info};
use snafu::ResultExt;
@@ -218,7 +218,7 @@ impl GcScheduler {
}
let span = common_telemetry::tracing::info_span!("meta_gc_handle_tick");
let report = self
.trigger_gc(EventContext::new(TriggerReason::ScheduledGc))
.trigger_gc(PersistentEventContext::new(TriggerReason::ScheduledGc))
.instrument(span)
.await?;
@@ -305,7 +305,7 @@ impl GcScheduler {
// No specific regions, use default tick behavior
let Some(regions) = region_ids else {
let report = self
.trigger_gc(EventContext::new(TriggerReason::Manual))
.trigger_gc(PersistentEventContext::new(TriggerReason::Manual))
.await?;
info!("Finished manual gc request");
return Ok(report);
@@ -353,7 +353,7 @@ impl GcScheduler {
full_listing,
gc_timeout,
Region2Peers::new(),
EventContext::new(TriggerReason::Manual),
PersistentEventContext::new(TriggerReason::Manual),
)
.await?;
combined_report.merge(report);
@@ -367,7 +367,7 @@ impl GcScheduler {
true,
gc_timeout,
dropped_routes_override,
EventContext::new(TriggerReason::Manual),
PersistentEventContext::new(TriggerReason::Manual),
)
.await?;
combined_report.merge(report);
@@ -490,7 +490,7 @@ mod tests {
_full_file_listing: bool,
_timeout: Duration,
_region_routes_override: Region2Peers,
_event_context: EventContext,
_event_context: PersistentEventContext,
) -> Result<GcReport> {
self.gc_regions_calls.fetch_add(1, Ordering::Relaxed);
panic!("gc_regions should not be called in maintenance mode")
@@ -564,7 +564,7 @@ mod tests {
_full_file_listing: bool,
_timeout: Duration,
_region_routes_override: Region2Peers,
_event_context: EventContext,
_event_context: PersistentEventContext,
) -> Result<GcReport> {
Ok(GcReport::default())
}
@@ -898,7 +898,7 @@ mod tests {
_full_file_listing: bool,
_timeout: Duration,
_region_routes_override: Region2Peers,
_event_context: EventContext,
_event_context: PersistentEventContext,
) -> Result<GcReport> {
crate::error::UnexpectedSnafu {
violated: "mock gc failure".to_string(),
src/meta-srv/src/procedure/region_migration.rs
+93 -59
View File
@@ -45,13 +45,12 @@ use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
use common_meta::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock};
use common_meta::peer::Peer;
use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard};
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::ddl::{PersistentEventContext, TriggerReason};
use common_procedure::error::{
Error as ProcedureError, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu,
};
use common_procedure::{
Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status,
StringKey,
Context as ProcedureContext, EventContext, LockKey, Procedure, Status, StringKey,
};
use common_telemetry::{debug, error, info};
use manager::RegionMigrationProcedureGuard;
@@ -123,11 +122,9 @@ pub struct PersistentContext {
/// The timeout for downgrading leader region and upgrading candidate region operations.
#[serde(with = "humantime_serde", default = "default_timeout")]
pub(crate) timeout: Duration,
/// The trigger reason of region migration.
/// The trigger reason persisted for compatibility with versions without procedure context.
#[serde(default)]
pub(crate) trigger_reason: RegionMigrationTriggerReason,
#[serde(default)]
pub(crate) event_context: EventContext,
trigger_reason: RegionMigrationTriggerReason,
}
impl PersistentContext {
@@ -137,7 +134,7 @@ impl PersistentContext {
to_peer: Peer,
region_ids: Vec<RegionId>,
timeout: Duration,
event_context: EventContext,
trigger_reason: RegionMigrationTriggerReason,
) -> Self {
#[allow(deprecated)]
Self {
@@ -148,17 +145,7 @@ impl PersistentContext {
to_peer,
region_ids,
timeout,
trigger_reason: RegionMigrationTriggerReason::from_trigger_reason(event_context.reason),
event_context,
}
}
fn normalize_trigger_reasons(&mut self) {
if self.trigger_reason != RegionMigrationTriggerReason::Unknown {
self.event_context.reason = self.trigger_reason.to_trigger_reason();
} else if self.event_context.reason != TriggerReason::Unknown {
self.trigger_reason =
RegionMigrationTriggerReason::from_trigger_reason(self.event_context.reason);
trigger_reason,
}
}
}
@@ -173,7 +160,7 @@ impl RegionMigrationTriggerReason {
}
}
fn from_trigger_reason(reason: TriggerReason) -> Self {
pub(crate) fn from_trigger_reason(reason: TriggerReason) -> Self {
match reason {
TriggerReason::Manual => Self::Manual,
TriggerReason::AutoRebalance => Self::AutoRebalance,
@@ -457,6 +444,14 @@ pub struct Context {
}
impl Context {
pub(crate) fn trigger_reason(
&self,
event_context: Option<&PersistentEventContext>,
) -> RegionMigrationTriggerReason {
event_context
.map(|ctx| RegionMigrationTriggerReason::from_trigger_reason(ctx.reason))
.unwrap_or(self.persistent_ctx.trigger_reason)
}
/// Returns the next operation's timeout.
pub fn next_operation_timeout(&self) -> Option<Duration> {
self.persistent_ctx
@@ -852,10 +847,9 @@ impl RegionMigrationProcedure {
tracker: RegionMigrationProcedureTracker,
) -> ProcedureResult<Self> {
let RegionMigrationDataOwned {
mut persistent_ctx,
persistent_ctx,
state,
} = serde_json::from_str(json).context(FromJsonSnafu)?;
persistent_ctx.normalize_trigger_reasons();
let guards = persistent_ctx
.region_ids
.iter()
@@ -999,14 +993,16 @@ impl Procedure for RegionMigrationProcedure {
LockKey::new(self.context.persistent_ctx.lock_key())
}
fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option<Box<dyn Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
if !ctx.event_type_filter.allows(REGION_MIGRATION_EVENT_TYPE) {
return None;
}
Some(Box::new(RegionMigrationEvent::from_persistent_ctx(
&self.context.persistent_ctx,
matches!(ctx.trigger, EventTrigger::Submitted),
self.context
.trigger_reason(ctx.event_context)
.to_trigger_reason(),
)))
}
}
@@ -1063,7 +1059,7 @@ mod tests {
let procedure = RegionMigrationProcedure::new(persistent_context, context, vec![]);
let serialized = procedure.dump().unwrap();
let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown","event_context":{"reason":"unknown"}},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
assert_eq!(expected, serialized);
}
@@ -1087,11 +1083,12 @@ mod tests {
for trigger in triggers {
let event = procedure
.event(&EventRuntimeContext {
.event(&EventContext {
procedure_id: common_procedure::ProcedureId::random(),
lifecycle_state: &state,
trigger,
event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
event_context: None,
})
.unwrap();
assert_eq!(event.event_type(), "region_migration");
@@ -1111,61 +1108,98 @@ mod tests {
to_peer: Peer::empty(2),
region_ids: vec![RegionId::new(1024, 1)],
timeout: Duration::from_secs(10),
trigger_reason: RegionMigrationTriggerReason::default(),
event_context: EventContext::default(),
trigger_reason: RegionMigrationTriggerReason::Unknown,
};
// NOTES: Changes it will break backward compatibility.
let serialized = r#"{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105}"#;
let deserialized: PersistentContext = serde_json::from_str(serialized).unwrap();
assert_eq!(persistent_ctx, deserialized);
assert_eq!(deserialized.event_context, EventContext::default());
}
#[test]
fn test_new_persistent_context_keeps_event_context() {
let mut event_context =
EventContext::new(TriggerReason::AutoRebalance).with_protocol("internal");
event_context
.extensions
.insert("scheduler".into(), serde_json::json!("load_based"));
let context = PersistentContext::new(
vec![("greptime".into(), "public".into())],
Peer::empty(1),
Peer::empty(2),
vec![RegionId::new(1024, 1)],
Duration::from_secs(10),
event_context.clone(),
fn test_legacy_trigger_reason_survives_recovery_and_repersistence() {
let serialized = r#"{"persistent_ctx":{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105,"trigger_reason":"Failover"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
let env = TestingEnv::new();
let procedure = RegionMigrationProcedure::from_json(
serialized,
env.context_factory(),
RegionMigrationProcedureTracker::default(),
)
.unwrap();
assert_eq!(
procedure.context.persistent_ctx.trigger_reason,
RegionMigrationTriggerReason::Failover
);
let repersisted = procedure.dump().unwrap();
assert_eq!(
serde_json::from_str::<serde_json::Value>(&repersisted).unwrap()["persistent_ctx"]["trigger_reason"],
"Failover"
);
let recovered = RegionMigrationProcedure::from_json(
&repersisted,
env.context_factory(),
RegionMigrationProcedureTracker::default(),
)
.unwrap();
assert_eq!(
context.trigger_reason,
recovered.context.trigger_reason(None),
RegionMigrationTriggerReason::Failover
);
let event_context = PersistentEventContext::new(TriggerReason::AutoRebalance);
assert_eq!(
recovered.context.trigger_reason(Some(&event_context)),
RegionMigrationTriggerReason::AutoRebalance
);
assert_eq!(context.event_context, event_context);
let state = common_procedure::ProcedureState::Running;
let event = recovered
.event(&EventContext {
procedure_id: common_procedure::ProcedureId::random(),
lifecycle_state: &state,
trigger: common_procedure::EventTrigger::Recovered,
event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
event_context: None,
})
.unwrap();
assert_eq!(
event.extra_rows().unwrap()[0].values[3].value_data,
Some(api::v1::value::ValueData::StringValue(
"Failover".to_string()
))
);
}
#[test]
fn test_normalize_trigger_reasons() {
let mut context = new_persistent_context();
context.trigger_reason = RegionMigrationTriggerReason::Manual;
context.event_context = EventContext::default();
context.normalize_trigger_reasons();
assert_eq!(context.event_context.reason, TriggerReason::Manual);
fn test_migration_reason_uses_event_context() {
let env = TestingEnv::new();
let context = env.context_factory().new_context(new_persistent_context());
let event_context = PersistentEventContext::new(TriggerReason::RegionFailover);
context.trigger_reason = RegionMigrationTriggerReason::Unknown;
context.event_context = EventContext::new(TriggerReason::RegionFailover);
context.normalize_trigger_reasons();
assert_eq!(
context.trigger_reason,
context.trigger_reason(Some(&event_context)),
RegionMigrationTriggerReason::Failover
);
context.trigger_reason = RegionMigrationTriggerReason::Manual;
context.event_context = EventContext::new(TriggerReason::RegionFailover);
context.normalize_trigger_reasons();
assert_eq!(context.event_context.reason, TriggerReason::Manual);
let procedure =
RegionMigrationProcedure::new(new_persistent_context(), env.context_factory(), vec![]);
let state = common_procedure::ProcedureState::Running;
let event = procedure
.event(&EventContext {
procedure_id: common_procedure::ProcedureId::random(),
lifecycle_state: &state,
trigger: common_procedure::EventTrigger::Submitted,
event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
event_context: Some(&event_context),
})
.unwrap();
assert_eq!(
event.extra_rows().unwrap()[0].values[3].value_data,
Some(api::v1::value::ValueData::StringValue(
"Failover".to_string()
))
);
}
#[derive(Debug, Serialize, Deserialize, Default)]
src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs
+2 -2
View File
@@ -40,7 +40,7 @@ impl State for CloseDowngradedRegion {
async fn next(
&mut self,
ctx: &mut Context,
_procedure_ctx: &ProcedureContext,
procedure_ctx: &ProcedureContext,
) -> Result<(Box<dyn State>, Status)> {
if let Err(err) = self.close_downgraded_leader_region(ctx).await {
let downgrade_leader_datanode = &ctx.persistent_ctx.from_peer;
@@ -52,7 +52,7 @@ impl State for CloseDowngradedRegion {
ctx.persistent_ctx.region_ids,
ctx.persistent_ctx.from_peer,
ctx.persistent_ctx.to_peer,
ctx.persistent_ctx.trigger_reason,
ctx.trigger_reason(procedure_ctx.event_context.as_ref()),
ctx.volatile_ctx.metrics,
);
Ok((Box::new(RegionMigrationEnd), Status::done()))
src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs
+4 -3
View File
@@ -382,7 +382,6 @@ mod tests {
use common_meta::key::table_route::TableRouteValue;
use common_meta::key::test_utils::new_test_table_info;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::router::{Region, RegionRoute};
use common_meta::wal_provider::RegionWalOptions;
use store_api::storage::RegionId;
@@ -391,7 +390,9 @@ mod tests {
use super::*;
use crate::error::Error;
use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context};
use crate::procedure::region_migration::{ContextFactory, PersistentContext};
use crate::procedure::region_migration::{
ContextFactory, PersistentContext, RegionMigrationTriggerReason,
};
use crate::procedure::test_util::{
new_close_region_reply, new_downgrade_region_reply, send_mock_reply,
};
@@ -403,7 +404,7 @@ mod tests {
Peer::empty(2),
vec![RegionId::new(1024, 1)],
Duration::from_millis(1000),
EventContext::new(TriggerReason::Manual),
RegionMigrationTriggerReason::Unknown,
)
}
src/meta-srv/src/procedure/region_migration/manager.rs
+9 -5
View File
@@ -21,7 +21,7 @@ use std::time::Duration;
use common_meta::key::table_info::TableInfoValue;
use common_meta::key::table_route::TableRouteValue;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::ddl::PersistentEventContext;
use common_meta::rpc::router::RegionRoute;
use common_procedure::{ProcedureId, ProcedureManagerRef, ProcedureWithId, watcher};
use common_telemetry::{error, info, warn};
@@ -479,6 +479,7 @@ impl RegionMigrationManager {
procedure_guards: Vec<RegionMigrationProcedureGuard>,
catalog_and_schema: Vec<(String, String)>,
) -> Result<ProcedureId> {
let event_context = PersistentEventContext::new(task.trigger_reason.to_trigger_reason());
let procedure = RegionMigrationProcedure::new(
PersistentContext::new(
catalog_and_schema,
@@ -486,12 +487,13 @@ impl RegionMigrationManager {
task.to_peer.clone(),
task.region_ids.clone(),
task.timeout,
EventContext::new(task.trigger_reason.to_trigger_reason()),
task.trigger_reason,
),
self.context_factory.clone(),
procedure_guards,
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
let procedure_id = procedure_with_id.id;
info!("Starting region migration procedure {procedure_id} for {task}");
let procedure_manager = self.procedure_manager.clone();
@@ -575,6 +577,7 @@ impl RegionMigrationManager {
timeout,
trigger_reason,
} = task.clone();
let event_context = PersistentEventContext::new(trigger_reason.to_trigger_reason());
let procedure = RegionMigrationProcedure::new(
PersistentContext::new(
vec![(catalog_name, schema_name)],
@@ -582,12 +585,13 @@ impl RegionMigrationManager {
to_peer,
vec![region_id],
timeout,
EventContext::new(trigger_reason.to_trigger_reason()),
trigger_reason,
),
self.context_factory.clone(),
vec![guard],
);
let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
let procedure_with_id =
ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context);
let procedure_id = procedure_with_id.id;
info!("Starting region migration procedure {procedure_id} for {task}");
let procedure_manager = self.procedure_manager.clone();
src/meta-srv/src/procedure/region_migration/migration_abort.rs
+2 -2
View File
@@ -41,7 +41,7 @@ impl State for RegionMigrationAbort {
async fn next(
&mut self,
ctx: &mut Context,
_procedure_ctx: &ProcedureContext,
procedure_ctx: &ProcedureContext,
) -> Result<(Box<dyn State>, Status)> {
warn!(
"Region migration is aborted: {}, regions: {:?}, from_peer: {}, to_peer: {}, trigger_reason: {}, {}",
@@ -49,7 +49,7 @@ impl State for RegionMigrationAbort {
ctx.persistent_ctx.region_ids,
ctx.persistent_ctx.from_peer,
ctx.persistent_ctx.to_peer,
ctx.persistent_ctx.trigger_reason,
ctx.trigger_reason(procedure_ctx.event_context.as_ref()),
ctx.volatile_ctx.metrics,
);
error::MigrationAbortSnafu {
src/meta-srv/src/procedure/region_migration/open_candidate_region.rs
+21 -9
View File
@@ -47,9 +47,12 @@ impl State for OpenCandidateRegion {
async fn next(
&mut self,
ctx: &mut Context,
_procedure_ctx: &ProcedureContext,
procedure_ctx: &ProcedureContext,
) -> Result<(Box<dyn State>, Status)> {
let instruction = self.build_open_region_instruction(ctx).await?;
let trigger_reason = ctx.trigger_reason(procedure_ctx.event_context.as_ref());
let instruction = self
.build_open_region_instruction(ctx, trigger_reason)
.await?;
let now = Instant::now();
self.open_candidate_region(ctx, instruction).await?;
ctx.update_open_candidate_region_elapsed(now);
@@ -67,11 +70,15 @@ impl OpenCandidateRegion {
///
/// Abort(non-retry):
/// - Datanode Table is not found.
async fn build_open_region_instruction(&self, ctx: &mut Context) -> Result<Instruction> {
async fn build_open_region_instruction(
&self,
ctx: &mut Context,
trigger_reason: RegionMigrationTriggerReason,
) -> Result<Instruction> {
let region_ids = ctx.persistent_ctx.region_ids.clone();
let from_peer_id = ctx.persistent_ctx.from_peer.id;
let to_peer_id = ctx.persistent_ctx.to_peer.id;
let reason = match ctx.persistent_ctx.trigger_reason {
let reason = match trigger_reason {
RegionMigrationTriggerReason::Failover => OpenRegionReason::RegionFailover,
_ => OpenRegionReason::RegionMigration,
};
@@ -280,7 +287,7 @@ mod tests {
let mut ctx = env.context_factory().new_context(persistent_context);
let err = state
.build_open_region_instruction(&mut ctx)
.build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Unknown)
.await
.unwrap_err();
@@ -291,7 +298,7 @@ mod tests {
#[tokio::test]
async fn test_build_open_region_instruction_reason() {
let state = OpenCandidateRegion;
let mut persistent_context = new_persistent_context();
let persistent_context = new_persistent_context();
let from_peer_id = persistent_context.from_peer.id;
let region_id = persistent_context.region_ids[0];
let env = TestingEnv::new();
@@ -314,7 +321,10 @@ mod tests {
let mut ctx = env
.context_factory()
.new_context(persistent_context.clone());
let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap();
let instruction = state
.build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Unknown)
.await
.unwrap();
let open_regions = instruction.into_open_regions().unwrap();
assert_eq!(
Some(OpenRegionReason::RegionMigration),
@@ -325,9 +335,11 @@ mod tests {
open_regions[0].requirements
);
persistent_context.trigger_reason = RegionMigrationTriggerReason::Failover;
let mut ctx = env.context_factory().new_context(persistent_context);
let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap();
let instruction = state
.build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Failover)
.await
.unwrap();
let open_regions = instruction.into_open_regions().unwrap();
assert_eq!(
Some(OpenRegionReason::RegionFailover),
src/meta-srv/src/procedure/region_migration/test_util.rs
+5 -3
View File
@@ -27,7 +27,6 @@ use common_meta::kv_backend::KvBackendRef;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::peer::Peer;
use common_meta::region_keeper::{MemoryRegionKeeper, MemoryRegionKeeperRef};
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::RegionRoute;
use common_meta::sequence::SequenceBuilder;
use common_meta::state_store::KvStateStore;
@@ -53,7 +52,8 @@ use crate::procedure::region_migration::open_candidate_region::OpenCandidateRegi
use crate::procedure::region_migration::update_metadata::UpdateMetadata;
use crate::procedure::region_migration::upgrade_candidate_region::UpgradeCandidateRegion;
use crate::procedure::region_migration::{
Context, ContextFactory, DefaultContextFactory, PersistentContext, State, VolatileContext,
Context, ContextFactory, DefaultContextFactory, PersistentContext,
RegionMigrationTriggerReason, State, VolatileContext,
};
use crate::procedure::test_util::{MailboxContext, send_mock_reply};
use crate::service::mailbox::Channel;
@@ -157,6 +157,7 @@ impl TestingEnv {
ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
}
}
@@ -190,7 +191,7 @@ pub fn new_persistent_context(from: u64, to: u64, region_id: RegionId) -> Persis
Peer::empty(to),
vec![region_id],
Duration::from_secs(10),
EventContext::default(),
RegionMigrationTriggerReason::Unknown,
)
}
@@ -577,5 +578,6 @@ pub fn new_procedure_context() -> ProcedureContext {
ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
}
}
src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
+4 -3
View File
@@ -356,7 +356,6 @@ mod tests {
use common_meta::key::topic_name::TopicNameKey;
use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey, TopicRegionValue};
use common_meta::peer::Peer;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::router::{Region, RegionRoute};
use common_meta::wal_provider::RegionWalOptions;
use common_wal::options::KafkaWalOptions;
@@ -365,7 +364,9 @@ mod tests {
use super::*;
use crate::error::Error;
use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context};
use crate::procedure::region_migration::{ContextFactory, PersistentContext};
use crate::procedure::region_migration::{
ContextFactory, PersistentContext, RegionMigrationTriggerReason,
};
use crate::procedure::test_util::{
new_close_region_reply, new_upgrade_region_reply, send_mock_reply,
};
@@ -377,7 +378,7 @@ mod tests {
Peer::empty(2),
vec![RegionId::new(1024, 1)],
Duration::from_millis(1000),
EventContext::new(TriggerReason::Manual),
RegionMigrationTriggerReason::Unknown,
)
}
src/meta-srv/src/procedure/repartition.rs
+15 -32
View File
@@ -44,12 +44,11 @@ use common_meta::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
use common_meta::node_manager::NodeManagerRef;
use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard};
use common_meta::region_registry::LeaderRegionRegistryRef;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::{RegionRoute, operating_leader_region_roles};
use common_meta::wal_provider::RegionWalOptions;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext,
BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventContext,
EventTrigger, LockKey, Procedure, ProcedureManagerRef, Result as ProcedureResult, Status,
StringKey,
};
@@ -104,8 +103,6 @@ pub struct PersistentContext {
#[serde(default)]
/// Records table-level partition metadata updated by this repartition.
pub partition_metadata_update: Option<PartitionMetadataUpdate>,
#[serde(default)]
pub event_context: EventContext,
}
fn default_timeout() -> Duration {
@@ -113,7 +110,7 @@ fn default_timeout() -> Duration {
}
impl PersistentContext {
/// Creates a new [PersistentContext] with the table, timeout and event context.
/// Creates a new [PersistentContext] with the given table name, table id and timeout.
///
/// If the timeout is not provided, the default timeout will be used.
pub fn new(
@@ -124,7 +121,6 @@ impl PersistentContext {
}: TableName,
table_id: TableId,
timeout: Option<Duration>,
event_context: EventContext,
) -> Self {
Self {
catalog_name,
@@ -136,7 +132,6 @@ impl PersistentContext {
unknown_procedures: vec![],
timeout: timeout.unwrap_or_else(default_timeout),
partition_metadata_update: None,
event_context,
}
}
@@ -797,10 +792,7 @@ impl Procedure for RepartitionProcedure {
LockKey::new(self.context.persistent_ctx.lock_key())
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(REPARTITION_EVENT_TYPE) {
return None;
}
@@ -869,7 +861,6 @@ impl RepartitionProcedureFactory for GcDisabledRepartitionProcedureFactory {
_source: RepartitionSource,
_to_exprs: Vec<String>,
_timeout: Option<Duration>,
_event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError> {
Err(BoxedError::new(
error::InvalidArgumentsSnafu {
@@ -908,9 +899,8 @@ impl RepartitionProcedureFactory for DefaultRepartitionProcedureFactory {
source: RepartitionSource,
to_exprs: Vec<String>,
timeout: Option<Duration>,
event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError> {
let persistent_ctx = PersistentContext::new(table_name, table_id, timeout, event_context);
let persistent_ctx = PersistentContext::new(table_name, table_id, timeout);
let from = match source {
RepartitionSource::Partitioned {
exprs,
@@ -1143,7 +1133,6 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
Context::new(
@@ -1175,7 +1164,6 @@ mod tests {
},
vec![],
None,
EventContext::default(),
)
.err()
.expect("GC-disabled factory must reject repartition");
@@ -1277,53 +1265,58 @@ mod tests {
let all = Arc::new(EventTypeFilter::All);
let submitted = procedure
.event(&EventRuntimeContext {
.event(&EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
event_type_filter: all.clone(),
event_context: None,
})
.unwrap();
assert_eq!(submitted.event_type(), REPARTITION_EVENT_TYPE);
assert_ne!(submitted.json_payload().unwrap(), serde_json::Value::Null);
let allowed = procedure
.event(&EventRuntimeContext {
.event(&EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::from([
REPARTITION_EVENT_TYPE.to_string(),
]))),
event_context: None,
})
.unwrap();
assert_eq!(allowed.event_type(), REPARTITION_EVENT_TYPE);
let succeeded = procedure
.event(&EventRuntimeContext {
.event(&EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Succeeded,
event_type_filter: all,
event_context: None,
})
.unwrap();
assert_eq!(succeeded.json_payload().unwrap(), serde_json::Value::Null);
let filtered = procedure.event(&EventRuntimeContext {
let filtered = procedure.event(&EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::from([
"another_event".to_string()
]))),
event_context: None,
});
assert!(filtered.is_none());
let empty = procedure.event(&EventRuntimeContext {
let empty = procedure.event(&EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::new())),
event_context: None,
});
assert!(empty.is_none());
}
@@ -1382,7 +1375,7 @@ mod tests {
}
#[test]
fn test_persistent_context_serde_defaults_for_new_fields() {
fn test_persistent_context_partition_metadata_update_serde_default() {
let json = r#"{
"catalog_name":"test_catalog",
"schema_name":"test_schema",
@@ -1394,10 +1387,7 @@ mod tests {
let persistent_ctx: PersistentContext = serde_json::from_str(json).unwrap();
assert!(persistent_ctx.failed_procedures.is_empty());
assert!(persistent_ctx.unknown_procedures.is_empty());
assert!(persistent_ctx.partition_metadata_update.is_none());
assert_eq!(persistent_ctx.event_context, EventContext::default());
}
#[tokio::test]
@@ -1474,7 +1464,6 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![with_rollback_metadata(
test_plan(table_id),
@@ -1541,7 +1530,6 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![test_plan(table_id)];
let context = Context::new(
@@ -1595,7 +1583,6 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
let failed_plan = test_plan(table_id);
let failed_plan = with_rollback_metadata(
@@ -1729,7 +1716,6 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![failed_plan, succeeded_plan.clone()];
persistent_ctx.failed_procedures = vec![ProcedureMeta {
@@ -1839,7 +1825,6 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![plan.clone()];
persistent_ctx.unknown_procedures = vec![ProcedureMeta {
@@ -1901,7 +1886,6 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![with_rollback_metadata(
test_plan(table_id),
@@ -2030,7 +2014,6 @@ mod tests {
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
persistent_ctx.plans = vec![failed_merge_plan, succeeded_split_plan.clone()];
persistent_ctx.failed_procedures = vec![ProcedureMeta {
src/meta-srv/src/procedure/repartition/allocate_region.rs
+1
View File
@@ -777,6 +777,7 @@ mod tests {
concurrent_region_route,
region_wal_options: test_region_wal_options(&[1, 2]),
}),
event_context: None,
};
let mut state = ExecutePlan;
src/meta-srv/src/procedure/repartition/collect.rs
+1 -3
View File
@@ -136,7 +136,6 @@ mod tests {
use common_error::mock::MockError;
use common_error::status_code::StatusCode;
use common_meta::rpc::ddl::EventContext;
use common_meta::test_util::MockDatanodeManager;
use common_procedure::{
Context as ProcedureContext, ContextProvider, Error as ProcedureError, ProcedureId,
@@ -194,7 +193,6 @@ mod tests {
table::table_name::TableName::new("test_catalog", "test_schema", "test_table"),
1024,
None,
EventContext::default(),
);
let mut ctx = crate::procedure::repartition::Context::new(
&ddl_ctx,
@@ -229,7 +227,6 @@ mod tests {
table::table_name::TableName::new("test_catalog", "test_schema", "test_table"),
1024,
None,
EventContext::default(),
);
let mut ctx = crate::procedure::repartition::Context::new(
&ddl_ctx,
@@ -249,6 +246,7 @@ mod tests {
receiver: rx,
inner: MockContextProvider::default(),
}),
event_context: None,
};
let mut state = Collect {
inflight_procedures: vec![ProcedureMeta {
src/meta-srv/src/procedure/repartition/gc_requirement.rs
+1
View File
@@ -244,6 +244,7 @@ mod tests {
block_after_persist: true,
persisted_tx: Some(persisted_tx),
}),
context: Default::default(),
})
.await
.unwrap();
src/meta-srv/src/procedure/repartition/group.rs
+9 -13
View File
@@ -38,8 +38,8 @@ use common_meta::peer::Peer;
use common_meta::rpc::router::RegionRoute;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger,
LockKey, Procedure, ProcedureId, Result as ProcedureResult, Status, StringKey,
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, ProcedureId, Result as ProcedureResult, Status, StringKey,
};
use common_telemetry::{error, info};
use serde::{Deserialize, Serialize};
@@ -262,10 +262,7 @@ impl Procedure for RepartitionGroupProcedure {
LockKey::new(self.context.persistent_ctx.lock_key())
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(REPARTITION_GROUP_EVENT_TYPE) {
return None;
}
@@ -634,9 +631,7 @@ mod tests {
use common_event_recorder::EventTypeFilter;
use common_meta::key::TableMetadataManager;
use common_meta::kv_backend::test_util::MockKvBackendBuilder;
use common_procedure::{
EventRuntimeContext, EventTrigger, Procedure, ProcedureId, ProcedureState,
};
use common_procedure::{EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState};
use crate::error::Error;
use crate::event::repartition::REPARTITION_GROUP_EVENT_TYPE;
@@ -713,15 +708,16 @@ mod tests {
context: env.create_context(new_persistent_context(1024, vec![], vec![])),
};
let state = ProcedureState::Running;
let runtime_context = |event_type_filter| EventRuntimeContext {
let event_context = |event_type_filter| EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state: &state,
trigger: EventTrigger::Submitted,
event_type_filter: Arc::new(event_type_filter),
event_context: None,
};
let allowed = procedure
.event(&runtime_context(EventTypeFilter::Only(HashSet::from([
.event(&event_context(EventTypeFilter::Only(HashSet::from([
REPARTITION_GROUP_EVENT_TYPE.to_string(),
]))))
.unwrap();
@@ -729,14 +725,14 @@ mod tests {
assert!(
procedure
.event(&runtime_context(EventTypeFilter::Only(HashSet::from([
.event(&event_context(EventTypeFilter::Only(HashSet::from([
"another_event".to_string(),
]))))
.is_none()
);
assert!(
procedure
.event(&runtime_context(EventTypeFilter::Only(HashSet::new())))
.event(&event_context(EventTypeFilter::Only(HashSet::new())))
.is_none()
);
}
src/meta-srv/src/procedure/repartition/test_util.rs
+2 -2
View File
@@ -24,7 +24,6 @@ use common_meta::kv_backend::KvBackendRef;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::node_manager::NodeManagerRef;
use common_meta::peer::Peer;
use common_meta::rpc::ddl::EventContext;
use common_meta::rpc::router::{Region, RegionRoute};
use common_meta::sequence::SequenceBuilder;
use common_meta::test_util::new_ddl_context_with_kv_backend;
@@ -103,6 +102,7 @@ impl TestingEnv {
ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
}
}
@@ -225,7 +225,6 @@ pub fn new_parent_context(
TableName::new("test_catalog", "test_schema", "test_table"),
table_id,
None,
EventContext::default(),
);
ParentContext::new(
@@ -266,6 +265,7 @@ pub fn procedure_context_with_receivers(
receivers,
inner: MockContextProvider::default(),
}),
event_context: None,
}
}
src/meta-srv/src/procedure/tests.rs
+3 -11
View File
@@ -29,7 +29,7 @@ use common_meta::ddl::test_util::create_table::{
};
use common_meta::key::table_route::{PhysicalTableRouteValue, TableRouteValue};
use common_meta::node_manager::NodeManagerRef;
use common_meta::rpc::ddl::{CreateTableTask, EventContext, QueryContext};
use common_meta::rpc::ddl::CreateTableTask;
use common_meta::rpc::router::{RegionRoute, find_leaders};
use common_procedure::Status;
use store_api::storage::RegionId;
@@ -91,8 +91,6 @@ fn create_table_task(table_name: Option<&str>) -> CreateTableTask {
fn test_region_request_builder() {
let mut procedure = CreateTableProcedure::new(
create_table_task(None),
QueryContext::default(),
EventContext::default(),
test_data::new_ddl_context(Arc::new(NodeClients::default())),
)
.unwrap();
@@ -190,8 +188,6 @@ async fn test_on_datanode_create_regions() {
let mut procedure = CreateTableProcedure::new(
create_table_task(None),
QueryContext::default(),
EventContext::default(),
test_data::new_ddl_context(node_manager),
)
.unwrap();
@@ -264,12 +260,8 @@ async fn test_on_datanode_create_logical_regions() {
.unwrap()
.0;
let _ = kv_backend.txn(physical_route_txn).await.unwrap();
let mut procedure = CreateLogicalTablesProcedure::new(
vec![task1, task2, task3],
physical_table_id,
EventContext::default(),
ctx,
);
let mut procedure =
CreateLogicalTablesProcedure::new(vec![task1, task2, task3], physical_table_id, ctx);
let expected_created_regions = Arc::new(Mutex::new(HashMap::from([(1, 3), (2, 3), (3, 3)])));
src/meta-srv/src/procedure/wal_prune.rs
+5 -7
View File
@@ -25,8 +25,8 @@ use common_meta::lock_key::RemoteWalLock;
use common_meta::region_registry::LeaderRegionRegistryRef;
use common_procedure::error::ToJsonSnafu;
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger,
LockKey, Procedure, ProcedureState, Result as ProcedureResult, Status, StringKey,
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, ProcedureState, Result as ProcedureResult, Status, StringKey,
};
use common_telemetry::{info, warn};
use manager::{WalPruneProcedureGuard, WalPruneProcedureTracker};
@@ -213,10 +213,7 @@ impl Procedure for WalPruneProcedure {
LockKey::new(vec![lock_key])
}
fn event(
&self,
ctx: &EventRuntimeContext<'_>,
) -> Option<Box<dyn common_event_recorder::Event>> {
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(WAL_PRUNE_EVENT_TYPE) {
return None;
}
@@ -506,11 +503,12 @@ mod tests {
let mut procedure =
WalPruneProcedure::new(context, None, "test_topic".to_string(), 42, false);
let running = ProcedureState::Running;
let runtime_context = |trigger, lifecycle_state, event_type_filter| EventRuntimeContext {
let runtime_context = |trigger, lifecycle_state, event_type_filter| EventContext {
procedure_id: ProcedureId::random(),
lifecycle_state,
trigger,
event_type_filter: Arc::new(event_type_filter),
event_context: None,
};
for trigger in [EventTrigger::Submitted, EventTrigger::Recovered] {
src/meta-srv/src/service/procedure.rs
+1
View File
@@ -107,6 +107,7 @@ impl procedure_service_server::ProcedureService for Metasrv {
.submit_ddl_task(
&ExecutorContext {
tracing_context: Some(header.tracing_context),
..Default::default()
},
SubmitDdlTaskRequest {
query_context,
src/standalone/src/procedure.rs
-2
View File
@@ -20,7 +20,6 @@ use common_meta::ddl::DdlContext;
use common_meta::ddl_manager::{RepartitionProcedureFactory, RepartitionSource};
use common_meta::key::runtime_switch::RuntimeSwitchManager;
use common_meta::kv_backend::KvBackendRef;
use common_meta::rpc::ddl::EventContext;
use common_meta::state_store::KvStateStore;
use common_procedure::local::{EventRecorderHandle, LocalManager, ManagerConfig};
use common_procedure::options::ProcedureConfig;
@@ -74,7 +73,6 @@ impl RepartitionProcedureFactory for StandaloneRepartitionProcedureFactory {
_source: RepartitionSource,
_to_exprs: Vec<String>,
_timeout: Option<Duration>,
_event_context: EventContext,
) -> std::result::Result<BoxedProcedure, BoxedError> {
Err(BoxedError::new(NoSupportRepartitionProcedureSnafu.build()))
}
tests-integration/src/tests/gc.rs
-2
View File
@@ -17,7 +17,6 @@ use std::sync::Arc;
use std::time::Duration;
use common_meta::key::TableMetadataManagerRef;
use common_meta::rpc::ddl::EventContext;
use common_procedure::ProcedureWithId;
use common_telemetry::info;
use common_test_util::recordbatch::check_output_stream;
@@ -214,7 +213,6 @@ async fn test_gc_basic(store_type: &StorageType) {
false, // full_file_listing
Duration::from_secs(10), // timeout
Default::default(),
EventContext::default(),
);
// Submit the procedure to the procedure manager
tests-integration/tests/gc_event.rs
+4 -2
View File
@@ -15,7 +15,7 @@
use std::sync::Arc;
use std::time::Duration;
use common_meta::rpc::ddl::{EventContext, TriggerReason};
use common_meta::rpc::ddl::{PersistentEventContext, TriggerReason};
use common_procedure::{ProcedureId, ProcedureWithId, watcher};
use common_test_util::temp_dir::create_temp_dir;
use meta_srv::gc::{BatchGcProcedure, GcSchedulerOptions};
@@ -129,7 +129,6 @@ async fn test_batch_gc_event() {
false,
Duration::from_secs(10),
Default::default(),
EventContext::new(TriggerReason::Manual),
);
let procedure_id = ProcedureId::parse_str("00000000-0000-0000-0000-00000000bac0").unwrap();
let mut watcher = cluster
@@ -138,6 +137,9 @@ async fn test_batch_gc_event() {
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
context: common_procedure::ProcedureContext::from_event_context(
PersistentEventContext::new(TriggerReason::Manual),
),
})
.await
.unwrap();
tests-integration/tests/repartition.rs
-2
View File
@@ -19,7 +19,6 @@ use std::time::Duration;
use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_error::root_source;
use common_meta::key::table_name::TableNameKey;
use common_meta::rpc::ddl::EventContext;
use common_procedure::{ProcedureWithId, watcher};
use common_query::Output;
use common_telemetry::info;
@@ -817,7 +816,6 @@ async fn trigger_table_gc(metasrv: &Arc<Metasrv>, table_name: &str) {
false, // full_file_listing
Duration::from_secs(10), // timeout
Default::default(),
EventContext::default(),
);
// Submit the procedure to the procedure manager
tests-integration/tests/table_ddl_event.rs
+2 -2
View File
@@ -377,7 +377,7 @@ async fn test_table_ddl_procedure_events() {
.ddl_manager()
.submit_undrop_table_task(
common_meta::rpc::ddl::UndropTableTask { table_id },
common_meta::rpc::ddl::EventContext::new(
common_meta::rpc::ddl::PersistentEventContext::new(
common_meta::rpc::ddl::TriggerReason::Manual,
),
)
@@ -419,7 +419,7 @@ async fn test_table_ddl_procedure_events() {
.ddl_manager()
.submit_purge_dropped_table_task(
common_meta::rpc::ddl::PurgeDroppedTableTask { table_id },
common_meta::rpc::ddl::EventContext::new(
common_meta::rpc::ddl::PersistentEventContext::new(
common_meta::rpc::ddl::TriggerReason::Manual,
),
)
tests/conf/frontend-test.toml.template
+3
View File
@@ -1,3 +1,6 @@
[grpc]
bind_addr = "{addrs.grpc_addr}"
server_addr = "{addrs.grpc_addr}"
[event_recorder]
event_types = []