rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-16 13:00:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: move election trait and implementations to the common-meta crate (#7820)

Browse Source 
* refactor: move election impl to common-meta

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix: adding back comment

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

---------

Signed-off-by: shuiyisong <xixing.sys@gmail.com>
This commit is contained in:
shuiyisong
2026-03-24 18:21:31 +08:00
committed by GitHub
parent 0e22d6a72b
commit c8c2e09eed
14 changed files with 218 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/meta-srv/src/election.rs → src/common/meta/src/election.rs
View File

@@ -21,15 +21,85 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use common_telemetry::{error, info, warn};
use serde::{Deserialize, Serialize};
use tokio::sync::broadcast::error::RecvError;
use tokio::sync::broadcast::{self, Receiver, Sender};
use crate::error::Result;
use crate::metasrv::MetasrvNodeInfo;
pub(crate) const CANDIDATE_LEASE_SECS: u64 = 600;
pub const CANDIDATE_LEASE_SECS: u64 = 600;
const KEEP_ALIVE_INTERVAL_SECS: u64 = CANDIDATE_LEASE_SECS / 2;
/// The value of the leader. It is used to store the leader's address.
pub struct LeaderValue(pub String);
impl<T: AsRef<[u8]>> From<T> for LeaderValue {
fn from(value: T) -> Self {
let string = String::from_utf8_lossy(value.as_ref());
Self(string.to_string())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MetasrvNodeInfo {
// The metasrv's address
pub addr: String,
// The node build version
pub version: String,
// The node build git commit hash
pub git_commit: String,
// The node start timestamp in milliseconds
pub start_time_ms: u64,
// The node total cpu millicores
#[serde(default)]
pub total_cpu_millicores: i64,
// The node total memory bytes
#[serde(default)]
pub total_memory_bytes: i64,
/// The node build cpu usage millicores
#[serde(default)]
pub cpu_usage_millicores: i64,
/// The node build memory usage bytes
#[serde(default)]
pub memory_usage_bytes: i64,
// The node hostname
#[serde(default)]
pub hostname: String,
}
// TODO(zyy17): Allow deprecated fields for backward compatibility. Remove this when the deprecated top-level fields are removed from the proto.
#[allow(deprecated)]
impl From<MetasrvNodeInfo> for api::v1::meta::MetasrvNodeInfo {
fn from(node_info: MetasrvNodeInfo) -> Self {
Self {
peer: Some(api::v1::meta::Peer {
addr: node_info.addr,
..Default::default()
}),
// TODO(zyy17): The following top-level fields are deprecated. They are kept for backward compatibility and will be removed in a future version.
// New code should use the fields in `info.NodeInfo` instead.
version: node_info.version.clone(),
git_commit: node_info.git_commit.clone(),
start_time_ms: node_info.start_time_ms,
cpus: node_info.total_cpu_millicores as u32,
memory_bytes: node_info.total_memory_bytes as u64,
// The canonical location for node information.
info: Some(api::v1::meta::NodeInfo {
version: node_info.version,
git_commit: node_info.git_commit,
start_time_ms: node_info.start_time_ms,
total_cpu_millicores: node_info.total_cpu_millicores,
total_memory_bytes: node_info.total_memory_bytes,
cpu_usage_millicores: node_info.cpu_usage_millicores,
memory_usage_bytes: node_info.memory_usage_bytes,
cpus: node_info.total_cpu_millicores as u32,
memory_bytes: node_info.total_memory_bytes as u64,
hostname: node_info.hostname,
}),
}
}
}
/// Messages sent when the leader changes.
#[derive(Debug, Clone)]
pub enum LeaderChangeMessage {
@@ -168,3 +238,5 @@ pub trait Election: Send + Sync {
fn subscribe_leader_change(&self) -> Receiver<LeaderChangeMessage>;
}
pub type ElectionRef = Arc<dyn Election<Leader = LeaderValue>>;

14
src/meta-srv/src/election/etcd.rs → src/common/meta/src/election/etcd.rs
View File

@@ -16,8 +16,6 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use common_meta::distributed_time_constants::{META_KEEP_ALIVE_INTERVAL_SECS, META_LEASE_SECS};
use common_meta::key::{CANDIDATES_ROOT, ELECTION_KEY};
use common_telemetry::{error, info, warn};
use etcd_client::{
Client, GetOptions, LeaderKey as EtcdLeaderKey, LeaseKeepAliveStream, LeaseKeeper, PutOptions,
@@ -27,13 +25,15 @@ use tokio::sync::broadcast;
use tokio::sync::broadcast::Receiver;
use tokio::time::{MissedTickBehavior, timeout};
use crate::distributed_time_constants::{META_KEEP_ALIVE_INTERVAL_SECS, META_LEASE_SECS};
use crate::election::{
CANDIDATE_LEASE_SECS, Election, KEEP_ALIVE_INTERVAL_SECS, LeaderChangeMessage, LeaderKey,
listen_leader_change, send_leader_change_and_set_flags,
CANDIDATE_LEASE_SECS, Election, ElectionRef, KEEP_ALIVE_INTERVAL_SECS, LeaderChangeMessage,
LeaderKey, LeaderValue, MetasrvNodeInfo, listen_leader_change,
send_leader_change_and_set_flags,
};
use crate::error;
use crate::error::Result;
use crate::metasrv::{ElectionRef, LeaderValue, MetasrvNodeInfo};
use crate::key::{CANDIDATES_ROOT, ELECTION_KEY};
impl LeaderKey for EtcdLeaderKey {
fn name(&self) -> &[u8] {
@@ -253,7 +253,7 @@ impl Election for EtcdElection {
.leader(self.election_key())
.await
.context(error::EtcdFailedSnafu)?;
let leader_value = res.kv().context(error::NoLeaderSnafu)?.value();
let leader_value = res.kv().context(error::ElectionNoLeaderSnafu)?.value();
Ok(leader_value.into())
}
}
@@ -279,7 +279,7 @@ impl EtcdElection {
ensure!(
res.ttl() > 0,
error::UnexpectedSnafu {
violated: "Failed to refresh the lease",
err_msg: "Failed to refresh the lease".to_string(),
}
);

4
src/meta-srv/src/election/rds.rs → src/common/meta/src/election/rds.rs
View File

@@ -36,7 +36,7 @@ fn parse_value_and_expire_time(value: &str) -> Result<(String, Timestamp)> {
.split(LEASE_SEP)
.collect_tuple()
.with_context(|| UnexpectedSnafu {
violated: format!(
err_msg: format!(
"Invalid value {}, expect node info || {} || expire time",
value, LEASE_SEP
),
@@ -45,7 +45,7 @@ fn parse_value_and_expire_time(value: &str) -> Result<(String, Timestamp)> {
let expire_time = match Timestamp::from_str(expire_time, None) {
Ok(ts) => ts,
Err(_) => UnexpectedSnafu {
violated: format!("Invalid timestamp: {}", expire_time),
err_msg: format!("Invalid timestamp: {}", expire_time),
}
.fail()?,
};

42
src/meta-srv/src/election/rds/mysql.rs → src/common/meta/src/election/rds/mysql.rs
View File

@@ -16,7 +16,6 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use common_meta::key::{CANDIDATES_ROOT, ELECTION_KEY};
use common_telemetry::{error, info, warn};
use common_time::Timestamp;
use snafu::{OptionExt, ResultExt, ensure};
@@ -29,14 +28,15 @@ use tokio::time::MissedTickBehavior;
use crate::election::rds::{LEASE_SEP, Lease, RdsLeaderKey, parse_value_and_expire_time};
use crate::election::{
Election, LeaderChangeMessage, listen_leader_change, send_leader_change_and_set_flags,
Election, ElectionRef, LeaderChangeMessage, LeaderValue, MetasrvNodeInfo, listen_leader_change,
send_leader_change_and_set_flags,
};
use crate::error::{
AcquireMySqlClientSnafu, DecodeSqlValueSnafu, DeserializeFromJsonSnafu,
LeaderLeaseChangedSnafu, LeaderLeaseExpiredSnafu, MySqlExecutionSnafu, NoLeaderSnafu, Result,
SerializeToJsonSnafu, SqlExecutionTimeoutSnafu, UnexpectedSnafu,
ElectionLeaderLeaseChangedSnafu, ElectionLeaderLeaseExpiredSnafu, ElectionNoLeaderSnafu,
MySqlExecutionSnafu, Result, SerializeToJsonSnafu, SqlExecutionTimeoutSnafu, UnexpectedSnafu,
};
use crate::metasrv::{ElectionRef, LeaderValue, MetasrvNodeInfo};
use crate::key::{CANDIDATES_ROOT, ELECTION_KEY};
struct ElectionSqlFactory<'a> {
table_name: &'a str,
@@ -592,7 +592,7 @@ impl Election for MySqlElection {
ensure!(
lease.expire_time > lease.current,
UnexpectedSnafu {
violated: format!(
err_msg: format!(
"Candidate lease expired at {:?} (current time: {:?}), key: {:?}",
lease.expire_time,
lease.current,
@@ -667,10 +667,10 @@ impl Election for MySqlElection {
let client = self.client.lock().await;
let mut executor = Executor::Default(client);
if let Some(lease) = self.get_value_with_lease(&key, &mut executor).await? {
ensure!(lease.expire_time > lease.current, NoLeaderSnafu);
ensure!(lease.expire_time > lease.current, ElectionNoLeaderSnafu);
Ok(lease.leader_value.as_bytes().into())
} else {
NoLeaderSnafu.fail()
ElectionNoLeaderSnafu.fail()
}
}
}
@@ -705,7 +705,7 @@ impl MySqlElection {
let current_time = match Timestamp::from_str(&current_time_str, None) {
Ok(ts) => ts,
Err(_) => UnexpectedSnafu {
violated: format!("Invalid timestamp: {}", current_time_str),
err_msg: format!("Invalid timestamp: {}", current_time_str),
}
.fail()?,
};
@@ -740,7 +740,7 @@ impl MySqlElection {
current = match Timestamp::from_str(current_time_str, None) {
Ok(ts) => ts,
Err(_) => UnexpectedSnafu {
violated: format!("Invalid timestamp: {}", current_time_str),
err_msg: format!("Invalid timestamp: {}", current_time_str),
}
.fail()?,
};
@@ -777,7 +777,7 @@ impl MySqlElection {
ensure!(
res == 1,
UnexpectedSnafu {
violated: format!("Failed to update key: {}", String::from_utf8_lossy(key)),
err_msg: format!("Failed to update key: {}", String::from_utf8_lossy(key)),
}
);
@@ -920,9 +920,12 @@ impl MySqlElection {
/// will be released.
/// - **Case 2**: If all checks pass, the function returns without performing any actions.
fn lease_check(&self, lease: &Option<Lease>) -> Result<Lease> {
let lease = lease.as_ref().context(NoLeaderSnafu)?;
let lease = lease.as_ref().context(ElectionNoLeaderSnafu)?;
// Case 1: Lease expired
ensure!(lease.expire_time > lease.current, LeaderLeaseExpiredSnafu);
ensure!(
lease.expire_time > lease.current,
ElectionLeaderLeaseExpiredSnafu
);
// Case 2: Everything is fine
Ok(lease.clone())
}
@@ -960,7 +963,7 @@ impl MySqlElection {
let remote_lease = self.get_value_with_lease(&key, &mut executor).await?;
ensure!(
expected_lease.map(|lease| lease.origin) == remote_lease.map(|lease| lease.origin),
LeaderLeaseChangedSnafu
ElectionLeaderLeaseChangedSnafu
);
self.delete_value(&key, &mut executor).await?;
self.put_value_with_lease(
@@ -987,12 +990,11 @@ mod tests {
use std::assert_matches::assert_matches;
use std::env;
use common_meta::maybe_skip_mysql_integration_test;
use common_telemetry::init_default_ut_logging;
use sqlx::MySqlPool;
use super::*;
use crate::error;
use crate::utils::mysql::create_mysql_pool;
use crate::{error, maybe_skip_mysql_integration_test};
async fn create_mysql_client(
table_name: Option<&str>,
@@ -1003,11 +1005,11 @@ mod tests {
let endpoint = env::var("GT_MYSQL_ENDPOINTS").unwrap_or_default();
if endpoint.is_empty() {
return UnexpectedSnafu {
violated: "MySQL endpoint is empty".to_string(),
err_msg: "MySQL endpoint is empty".to_string(),
}
.fail();
}
let pool = create_mysql_pool(&[endpoint], None).await.unwrap();
let pool = MySqlPool::connect(&endpoint).await.unwrap();
let mut client = ElectionMysqlClient::new(
pool,
execution_timeout,
@@ -1302,7 +1304,7 @@ mod tests {
let err = elected(&leader_mysql_election, table_name, Some(incorrect_lease))
.await
.unwrap_err();
assert_matches!(err, error::Error::LeaderLeaseChanged { .. });
assert_matches!(err, error::Error::ElectionLeaderLeaseChanged { .. });
let lease = get_lease(&leader_mysql_election).await;
assert!(lease.is_none());
drop_table(&leader_mysql_election.client, table_name).await;

45
src/meta-srv/src/election/rds/postgres.rs → src/common/meta/src/election/rds/postgres.rs
View File

@@ -16,7 +16,6 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use common_meta::key::{CANDIDATES_ROOT, ELECTION_KEY};
use common_telemetry::{error, info, warn};
use common_time::Timestamp;
use deadpool_postgres::{Manager, Pool};
@@ -28,13 +27,15 @@ use tokio_postgres::types::ToSql;
use crate::election::rds::{LEASE_SEP, Lease, RdsLeaderKey, parse_value_and_expire_time};
use crate::election::{
Election, LeaderChangeMessage, listen_leader_change, send_leader_change_and_set_flags,
Election, ElectionRef, LeaderChangeMessage, LeaderValue, MetasrvNodeInfo, listen_leader_change,
send_leader_change_and_set_flags,
};
use crate::error::{
DeserializeFromJsonSnafu, GetPostgresClientSnafu, NoLeaderSnafu, PostgresExecutionSnafu,
Result, SerializeToJsonSnafu, SqlExecutionTimeoutSnafu, UnexpectedSnafu,
DeserializeFromJsonSnafu, ElectionNoLeaderSnafu, GetPostgresClientSnafu,
PostgresExecutionSnafu, Result, SerializeToJsonSnafu, SqlExecutionTimeoutSnafu,
UnexpectedSnafu,
};
use crate::metasrv::{ElectionRef, LeaderValue, MetasrvNodeInfo};
use crate::key::{CANDIDATES_ROOT, ELECTION_KEY};
struct ElectionSqlFactory<'a> {
lock_id: u64,
@@ -404,13 +405,13 @@ impl Election for PgElection {
.get_value_with_lease(&key)
.await?
.context(UnexpectedSnafu {
violated: format!("Failed to get lease for key: {:?}", key),
err_msg: format!("Failed to get lease for key: {:?}", key),
})?;
ensure!(
lease.expire_time > lease.current,
UnexpectedSnafu {
violated: format!(
err_msg: format!(
"Candidate lease expired at {:?} (current time {:?}), key: {:?}",
lease.expire_time, lease.current, key
),
@@ -464,11 +465,11 @@ impl Election for PgElection {
.query(&self.sql_set.campaign, &[])
.await?;
let row = res.first().context(UnexpectedSnafu {
violated: "Failed to get the result of acquiring advisory lock",
err_msg: "Failed to get the result of acquiring advisory lock".to_string(),
})?;
let is_leader = row.try_get(0).map_err(|_| {
UnexpectedSnafu {
violated: "Failed to get the result of get lock",
err_msg: "Failed to get the result of get lock".to_string(),
}
.build()
})?;
@@ -500,10 +501,10 @@ impl Election for PgElection {
} else {
let key = self.election_key();
if let Some(lease) = self.get_value_with_lease(&key).await? {
ensure!(lease.expire_time > lease.current, NoLeaderSnafu);
ensure!(lease.expire_time > lease.current, ElectionNoLeaderSnafu);
Ok(lease.leader_value.as_bytes().into())
} else {
NoLeaderSnafu.fail()
ElectionNoLeaderSnafu.fail()
}
}
}
@@ -537,7 +538,7 @@ impl PgElection {
let current_time = match Timestamp::from_str(current_time_str, None) {
Ok(ts) => ts,
Err(_) => UnexpectedSnafu {
violated: format!("Invalid timestamp: {}", current_time_str),
err_msg: format!("Invalid timestamp: {}", current_time_str),
}
.fail()?,
};
@@ -576,7 +577,7 @@ impl PgElection {
current = match Timestamp::from_str(current_time_str, None) {
Ok(ts) => ts,
Err(_) => UnexpectedSnafu {
violated: format!("Invalid timestamp: {}", current_time_str),
err_msg: format!("Invalid timestamp: {}", current_time_str),
}
.fail()?,
};
@@ -613,7 +614,7 @@ impl PgElection {
ensure!(
res == 1,
UnexpectedSnafu {
violated: format!("Failed to update key: {}", String::from_utf8_lossy(key)),
err_msg: format!("Failed to update key: {}", String::from_utf8_lossy(key)),
}
);
@@ -742,9 +743,9 @@ impl PgElection {
let lease = self
.get_value_with_lease(&key)
.await?
.context(NoLeaderSnafu)?;
.context(ElectionNoLeaderSnafu)?;
// Case 2
ensure!(lease.expire_time > lease.current, NoLeaderSnafu);
ensure!(lease.expire_time > lease.current, ElectionNoLeaderSnafu);
// Case 3
Ok(())
}
@@ -831,11 +832,11 @@ mod tests {
use std::assert_matches::assert_matches;
use std::env;
use common_meta::maybe_skip_postgres_integration_test;
use deadpool_postgres::{Config, Runtime};
use tokio_postgres::NoTls;
use super::*;
use crate::error;
use crate::utils::postgres::create_postgres_pool;
use crate::{error, maybe_skip_postgres_integration_test};
async fn create_postgres_client(
table_name: Option<&str>,
@@ -846,11 +847,13 @@ mod tests {
let endpoint = env::var("GT_POSTGRES_ENDPOINTS").unwrap_or_default();
if endpoint.is_empty() {
return UnexpectedSnafu {
violated: "Postgres endpoint is empty".to_string(),
err_msg: "Postgres endpoint is empty".to_string(),
}
.fail();
}
let pool = create_postgres_pool(&[endpoint], None, None).await.unwrap();
let mut cfg = Config::new();
cfg.url = Some(endpoint);
let pool = cfg.create_pool(Some(Runtime::Tokio1), NoTls).unwrap();
let mut pg_client = ElectionPgClient::new(
pool,
execution_timeout,

70
src/common/meta/src/error.rs
View File

@@ -338,6 +338,24 @@ pub enum Error {
location: Location,
},
#[snafu(display("Metasrv election has no leader at this moment"))]
ElectionNoLeader {
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Metasrv election leader lease expired"))]
ElectionLeaderLeaseExpired {
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Metasrv election leader lease changed during election"))]
ElectionLeaderLeaseChanged {
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Table already exists, table: {}", table_name))]
TableAlreadyExists {
table_name: String,
@@ -751,6 +769,15 @@ pub enum Error {
location: Location,
},
#[cfg(feature = "pg_kvbackend")]
#[snafu(display("Failed to get Postgres client"))]
GetPostgresClient {
#[snafu(source)]
error: deadpool::managed::PoolError<tokio_postgres::Error>,
#[snafu(implicit)]
location: Location,
},
#[cfg(feature = "pg_kvbackend")]
#[snafu(display("Failed to {} Postgres transaction", operation))]
PostgresTransaction {
@@ -805,6 +832,24 @@ pub enum Error {
location: Location,
},
#[cfg(feature = "mysql_kvbackend")]
#[snafu(display("Failed to decode sql value"))]
DecodeSqlValue {
#[snafu(source)]
error: sqlx::error::Error,
#[snafu(implicit)]
location: Location,
},
#[cfg(feature = "mysql_kvbackend")]
#[snafu(display("Failed to acquire mysql client from pool"))]
AcquireMySqlClient {
#[snafu(source)]
error: sqlx::Error,
#[snafu(implicit)]
location: Location,
},
#[cfg(feature = "mysql_kvbackend")]
#[snafu(display("Failed to {} MySql transaction", operation))]
MySqlTransaction {
@@ -822,6 +867,15 @@ pub enum Error {
location: Location,
},
#[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
#[snafu(display("Sql execution timeout, sql: {}, duration: {:?}", sql, duration))]
SqlExecutionTimeout {
sql: String,
duration: std::time::Duration,
#[snafu(implicit)]
location: Location,
},
#[snafu(display(
"Datanode table info not found, table id: {}, datanode id: {}",
table_id,
@@ -1075,7 +1129,10 @@ impl ErrorExt for Error {
| GetCache { .. }
| GetLatestCacheRetryExceeded { .. }
| SerializeToJson { .. }
| DeserializeFromJson { .. } => StatusCode::Internal,
| DeserializeFromJson { .. }
| ElectionNoLeader { .. }
| ElectionLeaderLeaseExpired { .. }
| ElectionLeaderLeaseChanged { .. } => StatusCode::Internal,
NoLeader { .. } => StatusCode::TableUnavailable,
ValueNotExist { .. }
@@ -1198,15 +1255,18 @@ impl ErrorExt for Error {
PostgresExecution { .. }
| CreatePostgresPool { .. }
| GetPostgresConnection { .. }
| GetPostgresClient { .. }
| PostgresTransaction { .. }
| PostgresTlsConfig { .. }
| InvalidTlsConfig { .. } => StatusCode::Internal,
#[cfg(feature = "mysql_kvbackend")]
MySqlExecution { .. } | CreateMySqlPool { .. } | MySqlTransaction { .. } => {
StatusCode::Internal
}
MySqlExecution { .. }
| CreateMySqlPool { .. }
| DecodeSqlValue { .. }
| AcquireMySqlClient { .. }
| MySqlTransaction { .. } => StatusCode::Internal,
#[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
RdsTransactionRetryFailed { .. } => StatusCode::Internal,
RdsTransactionRetryFailed { .. } | SqlExecutionTimeout { .. } => StatusCode::Internal,
DatanodeTableInfoNotFound { .. } => StatusCode::Internal,
}
}

1
src/common/meta/src/lib.rs
View File

@@ -22,6 +22,7 @@ pub mod datanode;
pub mod ddl;
pub mod ddl_manager;
pub mod distributed_time_constants;
pub mod election;
pub mod error;
pub mod flow_name;
pub mod heartbeat;

21
src/meta-srv/src/bootstrap.rs
View File

@@ -24,6 +24,8 @@ use common_base::Plugins;
use common_config::Configurable;
#[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
use common_meta::distributed_time_constants::META_LEASE_SECS;
use common_meta::election::CANDIDATE_LEASE_SECS;
use common_meta::election::etcd::EtcdElection;
use common_meta::kv_backend::chroot::ChrootKvBackend;
use common_meta::kv_backend::etcd::EtcdStore;
use common_meta::kv_backend::memory::MemoryKvBackend;
@@ -42,9 +44,6 @@ use tonic::codec::CompressionEncoding;
use tonic::transport::server::{Router, TcpIncoming};
use crate::cluster::{MetaPeerClientBuilder, MetaPeerClientRef};
#[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
use crate::election::CANDIDATE_LEASE_SECS;
use crate::election::etcd::EtcdElection;
use crate::error::OtherSnafu;
use crate::metasrv::builder::MetasrvBuilder;
use crate::metasrv::{
@@ -281,7 +280,8 @@ pub async fn metasrv_builder(
etcd_client,
opts.store_key_prefix.clone(),
)
.await?;
.await
.context(error::KvBackendSnafu)?;
(kv_backend, Some(election))
}
@@ -290,10 +290,10 @@ pub async fn metasrv_builder(
use std::time::Duration;
use common_meta::distributed_time_constants::POSTGRES_KEEP_ALIVE_SECS;
use common_meta::election::rds::postgres::{ElectionPgClient, PgElection};
use common_meta::kv_backend::rds::PgStore;
use deadpool_postgres::{Config, ManagerConfig, RecyclingMethod};
use crate::election::rds::postgres::{ElectionPgClient, PgElection};
use crate::utils::postgres::create_postgres_pool;
let candidate_lease_ttl = Duration::from_secs(CANDIDATE_LEASE_SECS);
@@ -321,7 +321,8 @@ pub async fn metasrv_builder(
execution_timeout,
idle_session_timeout,
statement_timeout,
)?;
)
.context(error::KvBackendSnafu)?;
let election = PgElection::with_pg_client(
opts.grpc.server_addr.clone(),
election_client,
@@ -332,7 +333,8 @@ pub async fn metasrv_builder(
&opts.meta_table_name,
opts.meta_election_lock_id,
)
.await?;
.await
.context(error::KvBackendSnafu)?;
let pool = create_postgres_pool(&opts.store_addrs, Some(cfg), opts.backend_tls.clone())
.await?;
@@ -352,9 +354,9 @@ pub async fn metasrv_builder(
(None, BackendImpl::MysqlStore) => {
use std::time::Duration;
use common_meta::election::rds::mysql::{ElectionMysqlClient, MySqlElection};
use common_meta::kv_backend::rds::MySqlStore;
use crate::election::rds::mysql::{ElectionMysqlClient, MySqlElection};
use crate::utils::mysql::create_mysql_pool;
let pool = create_mysql_pool(&opts.store_addrs, opts.backend_tls.as_ref()).await?;
@@ -389,7 +391,8 @@ pub async fn metasrv_builder(
meta_lease_ttl,
&election_table_name,
)
.await?;
.await
.context(error::KvBackendSnafu)?;
(kv_backend, Some(election))
}
};

4
src/meta-srv/src/cluster.rs
View File

@@ -247,7 +247,7 @@ impl MetaPeerClient {
// Safety: when self.is_leader() == false, election must not empty.
let election = self.election.as_ref().unwrap();
let leader_addr = election.leader().await?.0;
let leader_addr = election.leader().await.context(error::KvBackendSnafu)?.0;
let channel = self
.channel_manager
@@ -279,7 +279,7 @@ impl MetaPeerClient {
// Safety: when self.is_leader() == false, election must not empty.
let election = self.election.as_ref().unwrap();
let leader_addr = election.leader().await?.0;
let leader_addr = election.leader().await.context(error::KvBackendSnafu)?.0;
let channel = self
.channel_manager

1
src/meta-srv/src/lib.rs
View File

@@ -21,7 +21,6 @@ pub mod bootstrap;
pub mod cache_invalidator;
pub mod cluster;
pub mod discovery;
pub mod election;
pub mod error;
pub mod events;
mod failure_detector;

74
src/meta-srv/src/metasrv.rs
View File

@@ -32,6 +32,8 @@ use common_meta::ddl_manager::DdlManagerRef;
use common_meta::distributed_time_constants::{
self, BASE_HEARTBEAT_INTERVAL, default_distributed_time_constants, frontend_heartbeat_interval,
};
use common_meta::election::LeaderChangeMessage;
pub use common_meta::election::{ElectionRef, MetasrvNodeInfo};
use common_meta::key::TableMetadataManagerRef;
use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
use common_meta::kv_backend::{KvBackendRef, ResettableKvBackend, ResettableKvBackendRef};
@@ -64,7 +66,6 @@ use tokio::sync::broadcast::error::RecvError;
use crate::cluster::MetaPeerClientRef;
use crate::discovery;
use crate::election::{Election, LeaderChangeMessage};
use crate::error::{
self, InitMetadataSnafu, KvBackendSnafu, Result, StartProcedureManagerSnafu,
StartTelemetryTaskSnafu, StopProcedureManagerSnafu,
@@ -459,76 +460,6 @@ impl Context {
}
}
/// The value of the leader. It is used to store the leader's address.
pub struct LeaderValue(pub String);
impl<T: AsRef<[u8]>> From<T> for LeaderValue {
fn from(value: T) -> Self {
let string = String::from_utf8_lossy(value.as_ref());
Self(string.to_string())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MetasrvNodeInfo {
// The metasrv's address
pub addr: String,
// The node build version
pub version: String,
// The node build git commit hash
pub git_commit: String,
// The node start timestamp in milliseconds
pub start_time_ms: u64,
// The node total cpu millicores
#[serde(default)]
pub total_cpu_millicores: i64,
// The node total memory bytes
#[serde(default)]
pub total_memory_bytes: i64,
/// The node build cpu usage millicores
#[serde(default)]
pub cpu_usage_millicores: i64,
/// The node build memory usage bytes
#[serde(default)]
pub memory_usage_bytes: i64,
// The node hostname
#[serde(default)]
pub hostname: String,
}
// TODO(zyy17): Allow deprecated fields for backward compatibility. Remove this when the deprecated top-level fields are removed from the proto.
#[allow(deprecated)]
impl From<MetasrvNodeInfo> for api::v1::meta::MetasrvNodeInfo {
fn from(node_info: MetasrvNodeInfo) -> Self {
Self {
peer: Some(api::v1::meta::Peer {
addr: node_info.addr,
..Default::default()
}),
// TODO(zyy17): The following top-level fields are deprecated. They are kept for backward compatibility and will be removed in a future version.
// New code should use the fields in `info.NodeInfo` instead.
version: node_info.version.clone(),
git_commit: node_info.git_commit.clone(),
start_time_ms: node_info.start_time_ms,
cpus: node_info.total_cpu_millicores as u32,
memory_bytes: node_info.total_memory_bytes as u64,
// The canonical location for node information.
info: Some(api::v1::meta::NodeInfo {
version: node_info.version,
git_commit: node_info.git_commit,
start_time_ms: node_info.start_time_ms,
total_cpu_millicores: node_info.total_cpu_millicores,
total_memory_bytes: node_info.total_memory_bytes,
cpu_usage_millicores: node_info.cpu_usage_millicores,
memory_usage_bytes: node_info.memory_usage_bytes,
cpus: node_info.total_cpu_millicores as u32,
memory_bytes: node_info.total_memory_bytes as u64,
hostname: node_info.hostname,
}),
}
}
}
#[derive(Clone, Copy)]
pub enum SelectTarget {
Datanode,
@@ -552,7 +483,6 @@ pub struct SelectorContext {
pub type SelectorRef = Arc<dyn Selector<Context = SelectorContext, Output = Vec<Peer>>>;
pub type RegionStatAwareSelectorRef =
Arc<dyn RegionStatAwareSelector<Context = SelectorContext, Output = Vec<(RegionId, Peer)>>>;
pub type ElectionRef = Arc<dyn Election<Leader = LeaderValue>>;
pub struct MetaStateHandler {
subscribe_manager: Option<SubscriptionManagerRef>,

2
src/meta-srv/src/service/admin/leader.rs
View File

@@ -32,7 +32,7 @@ pub struct LeaderHandler {
impl LeaderHandler {
async fn get_leader(&self) -> Result<Option<String>> {
if let Some(election) = &self.election {
let leader_addr = election.leader().await?.0;
let leader_addr = election.leader().await.context(error::KvBackendSnafu)?.0;
return Ok(Some(leader_addr));
}
Ok(None)

5
src/meta-srv/src/service/cluster.rs
View File

@@ -63,7 +63,10 @@ impl cluster_server::Cluster for Metasrv {
let leader_addr = &self.options().grpc.server_addr;
let (leader, followers) = match self.election() {
Some(election) => {
let nodes = election.all_candidates().await?;
let nodes = election
.all_candidates()
.await
.context(error::KvBackendSnafu)?;
let followers = nodes
.into_iter()
.filter(|node_info| &node_info.addr != leader_addr)

4
src/meta-srv/src/service/heartbeat.rs
View File

@@ -23,7 +23,7 @@ use api::v1::meta::{
use common_telemetry::{debug, error, info, warn};
use futures::StreamExt;
use once_cell::sync::OnceCell;
use snafu::OptionExt;
use snafu::{OptionExt, ResultExt};
use tokio::sync::mpsc;
use tokio::sync::mpsc::Sender;
use tokio_stream::wrappers::ReceiverStream;
@@ -148,7 +148,7 @@ async fn handle_ask_leader(_req: AskLeaderRequest, ctx: Context) -> Result<AskLe
if election.is_leader() {
ctx.server_addr
} else {
election.leader().await?.0
election.leader().await.context(error::KvBackendSnafu)?.0
}
}
None => ctx.server_addr,