refactor: unify KvBackend and KvStore (#1890)

* refactor: unify KvBackend and KvStore

Cargo.lock

@@ -1812,6 +1812,7 @@ dependencies = [
  "common-telemetry",
  "common-time",
  "datatypes",
+ "etcd-client",
  "futures",
  "lazy_static",
  "prost",

Cargo.toml

@@ -71,6 +71,7 @@ datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git
 datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
 datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
 datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
+etcd-client = "0.11"
 futures = "0.3"
 futures-util = "0.3"
 greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "917ead6274b4dccbaf33c59a7360646ba2f285a9" }

src/catalog/src/remote.rs

@@ -30,76 +30,3 @@ pub trait KvCacheInvalidator: Send + Sync {
 }
 
 pub type KvCacheInvalidatorRef = Arc<dyn KvCacheInvalidator>;
-
-#[cfg(test)]
-mod tests {
-    use std::any::Any;
-
-    use async_stream::stream;
-    use common_meta::kv_backend::{Kv, KvBackend, ValueIter};
-
-    use crate::error::Error;
-
-    struct MockKvBackend {}
-
-    #[async_trait::async_trait]
-    impl KvBackend for MockKvBackend {
-        type Error = Error;
-
-        fn range<'a, 'b>(&'a self, _key: &[u8]) -> ValueIter<'b, Error>
-        where
-            'a: 'b,
-        {
-            Box::pin(stream!({
-                for i in 0..3 {
-                    yield Ok(Kv(
-                        i.to_string().as_bytes().to_vec(),
-                        i.to_string().as_bytes().to_vec(),
-                    ))
-                }
-            }))
-        }
-
-        async fn set(&self, _key: &[u8], _val: &[u8]) -> Result<(), Error> {
-            unimplemented!()
-        }
-
-        async fn compare_and_set(
-            &self,
-            _key: &[u8],
-            _expect: &[u8],
-            _val: &[u8],
-        ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
-            unimplemented!()
-        }
-
-        async fn delete_range(&self, _key: &[u8], _end: &[u8]) -> Result<(), Error> {
-            unimplemented!()
-        }
-
-        async fn move_value(&self, _from_key: &[u8], _to_key: &[u8]) -> Result<(), Error> {
-            unimplemented!()
-        }
-
-        fn as_any(&self) -> &dyn Any {
-            self
-        }
-    }
-
-    #[tokio::test]
-    async fn test_get() {
-        let backend = MockKvBackend {};
-
-        let result = backend.get(0.to_string().as_bytes()).await;
-        assert_eq!(0.to_string().as_bytes(), result.unwrap().unwrap().0);
-
-        let result = backend.get(1.to_string().as_bytes()).await;
-        assert_eq!(1.to_string().as_bytes(), result.unwrap().unwrap().0);
-
-        let result = backend.get(2.to_string().as_bytes()).await;
-        assert_eq!(2.to_string().as_bytes(), result.unwrap().unwrap().0);
-
-        let result = backend.get(3.to_string().as_bytes()).await;
-        assert!(result.unwrap().is_none());
-    }
-}

src/catalog/src/remote/client.rs

@@ -17,15 +17,18 @@ use std::fmt::Debug;
 use std::sync::Arc;
 use std::time::Duration;
 
-use async_stream::stream;
 use common_error::prelude::BoxedError;
 use common_meta::error::Error::{CacheNotGet, GetKvCache};
 use common_meta::error::{CacheNotGetSnafu, Error, MetaSrvSnafu, Result};
-use common_meta::kv_backend::{Kv, KvBackend, KvBackendRef, ValueIter};
+use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
 use common_meta::rpc::store::{
-    CompareAndPutRequest, DeleteRangeRequest, MoveValueRequest, PutRequest, RangeRequest,
+    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
+    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
+    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+    RangeRequest, RangeResponse,
 };
-use common_telemetry::{info, timer};
+use common_meta::rpc::KeyValue;
+use common_telemetry::timer;
 use meta_client::client::MetaClient;
 use moka::future::{Cache, CacheBuilder};
 use snafu::{OptionExt, ResultExt};
@@ -37,24 +40,29 @@ const CACHE_MAX_CAPACITY: u64 = 10000;
 const CACHE_TTL_SECOND: u64 = 10 * 60;
 const CACHE_TTI_SECOND: u64 = 5 * 60;
 
-pub type CacheBackendRef = Arc<Cache<Vec<u8>, Kv>>;
+pub type CacheBackendRef = Arc<Cache<Vec<u8>, KeyValue>>;
+
 pub struct CachedMetaKvBackend {
     kv_backend: KvBackendRef,
     cache: CacheBackendRef,
+    name: String,
+}
+
+impl TxnService for CachedMetaKvBackend {
+    type Error = Error;
 }
 
 #[async_trait::async_trait]
 impl KvBackend for CachedMetaKvBackend {
-    type Error = Error;
+    fn name(&self) -> &str {
-
+        &self.name
-    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
-    where
-        'a: 'b,
-    {
-        self.kv_backend.range(key)
     }
 
-    async fn get(&self, key: &[u8]) -> Result<Option<Kv>> {
+    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
+        self.kv_backend.range(req).await
+    }
+
+    async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
         let _timer = timer!(METRIC_CATALOG_KV_GET);
 
         let init = async {
@@ -81,8 +89,10 @@ impl KvBackend for CachedMetaKvBackend {
         })
     }
 
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<()> {
+    async fn put(&self, req: PutRequest) -> Result<PutResponse> {
-        let ret = self.kv_backend.set(key, val).await;
+        let key = &req.key.clone();
+
+        let ret = self.kv_backend.put(req).await;
 
         if ret.is_ok() {
             self.invalidate_key(key).await;
@@ -91,8 +101,72 @@ impl KvBackend for CachedMetaKvBackend {
         ret
     }
 
-    async fn delete(&self, key: &[u8]) -> Result<()> {
+    async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse> {
-        let ret = self.kv_backend.delete_range(key, &[]).await;
+        let keys = req
+            .kvs
+            .iter()
+            .map(|kv| kv.key().to_vec())
+            .collect::<Vec<_>>();
+
+        let resp = self.kv_backend.batch_put(req).await;
+
+        if resp.is_ok() {
+            for key in keys {
+                self.invalidate_key(&key).await;
+            }
+        }
+
+        resp
+    }
+
+    async fn delete_range(&self, mut req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
+        let prev_kv = req.prev_kv;
+
+        req.prev_kv = true;
+        let resp = self.kv_backend.delete_range(req).await;
+        match resp {
+            Ok(mut resp) => {
+                for prev_kv in resp.prev_kvs.iter() {
+                    self.invalidate_key(prev_kv.key()).await;
+                }
+
+                if !prev_kv {
+                    resp.prev_kvs = vec![];
+                }
+                Ok(resp)
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn batch_delete(&self, mut req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
+        let prev_kv = req.prev_kv;
+
+        req.prev_kv = true;
+        let resp = self.kv_backend.batch_delete(req).await;
+        match resp {
+            Ok(mut resp) => {
+                for prev_kv in resp.prev_kvs.iter() {
+                    self.invalidate_key(prev_kv.key()).await;
+                }
+
+                if !prev_kv {
+                    resp.prev_kvs = vec![];
+                }
+                Ok(resp)
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
+        self.kv_backend.batch_get(req).await
+    }
+
+    async fn compare_and_put(&self, req: CompareAndPutRequest) -> Result<CompareAndPutResponse> {
+        let key = &req.key.clone();
+
+        let ret = self.kv_backend.compare_and_put(req).await;
 
         if ret.is_ok() {
             self.invalidate_key(key).await;
@@ -101,28 +175,11 @@ impl KvBackend for CachedMetaKvBackend {
         ret
     }
 
-    async fn delete_range(&self, _key: &[u8], _end: &[u8]) -> Result<()> {
+    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
-        // TODO(fys): implement it
+        let from_key = &req.from_key.clone();
-        unimplemented!()
+        let to_key = &req.to_key.clone();
-    }
 
-    async fn compare_and_set(
+        let ret = self.kv_backend.move_value(req).await;
-        &self,
-        key: &[u8],
-        expect: &[u8],
-        val: &[u8],
-    ) -> Result<std::result::Result<(), Option<Vec<u8>>>> {
-        let ret = self.kv_backend.compare_and_set(key, expect, val).await;
-
-        if ret.is_ok() {
-            self.invalidate_key(key).await;
-        }
-
-        ret
-    }
-
-    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<()> {
-        let ret = self.kv_backend.move_value(from_key, to_key).await;
 
         if ret.is_ok() {
             self.invalidate_key(from_key).await;
@@ -146,15 +203,8 @@ impl KvCacheInvalidator for CachedMetaKvBackend {
 
 impl CachedMetaKvBackend {
     pub fn new(client: Arc<MetaClient>) -> Self {
-        let cache = Arc::new(
-            CacheBuilder::new(CACHE_MAX_CAPACITY)
-                .time_to_live(Duration::from_secs(CACHE_TTL_SECOND))
-                .time_to_idle(Duration::from_secs(CACHE_TTI_SECOND))
-                .build(),
-        );
         let kv_backend = Arc::new(MetaKvBackend { client });
-
+        Self::wrap(kv_backend)
-        Self { kv_backend, cache }
     }
 
     pub fn wrap(kv_backend: KvBackendRef) -> Self {
@@ -165,7 +215,12 @@ impl CachedMetaKvBackend {
                 .build(),
         );
 
-        Self { kv_backend, cache }
+        let name = format!("CachedKvBackend({})", kv_backend.name());
+        Self {
+            kv_backend,
+            cache,
+            name,
+        }
     }
 
     pub fn cache(&self) -> &CacheBackendRef {
@@ -178,108 +233,97 @@ pub struct MetaKvBackend {
     pub client: Arc<MetaClient>,
 }
 
+impl TxnService for MetaKvBackend {
+    type Error = Error;
+}
+
 /// Implement `KvBackend` trait for `MetaKvBackend` instead of opendal's `Accessor` since
 /// `MetaClient`'s range method can return both keys and values, which can reduce IO overhead
 /// comparing to `Accessor`'s list and get method.
 #[async_trait::async_trait]
 impl KvBackend for MetaKvBackend {
-    type Error = Error;
+    fn name(&self) -> &str {
-
+        "MetaKvBackend"
-    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
-    where
-        'a: 'b,
-    {
-        let key = key.to_vec();
-        Box::pin(stream!({
-            let mut resp = self
-                .client
-                .range(RangeRequest::new().with_prefix(key))
-                .await
-                .map_err(BoxedError::new)
-                .context(MetaSrvSnafu)?;
-            let kvs = resp.take_kvs();
-            for mut kv in kvs.into_iter() {
-                yield Ok(Kv(kv.take_key(), kv.take_value()))
-            }
-        }))
     }
 
-    async fn get(&self, key: &[u8]) -> Result<Option<Kv>> {
+    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
+        self.client
+            .range(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
+    }
+
+    async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
         let mut response = self
             .client
             .range(RangeRequest::new().with_key(key))
             .await
             .map_err(BoxedError::new)
             .context(MetaSrvSnafu)?;
-        Ok(response
+        Ok(response.take_kvs().get_mut(0).map(|kv| KeyValue {
-            .take_kvs()
+            key: kv.take_key(),
-            .get_mut(0)
+            value: kv.take_value(),
-            .map(|kv| Kv(kv.take_key(), kv.take_value())))
+        }))
     }
 
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<()> {
+    async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse> {
-        let req = PutRequest::new()
+        self.client
-            .with_key(key.to_vec())
+            .batch_put(req)
-            .with_value(val.to_vec());
+            .await
-        let _ = self
+            .map_err(BoxedError::new)
-            .client
+            .context(MetaSrvSnafu)
+    }
+
+    async fn put(&self, req: PutRequest) -> Result<PutResponse> {
+        self.client
             .put(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)?;
+            .context(MetaSrvSnafu)
-        Ok(())
     }
 
-    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<()> {
+    async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
-        let req = DeleteRangeRequest::new().with_range(key.to_vec(), end.to_vec());
+        self.client
-        let resp = self
-            .client
             .delete_range(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)?;
+            .context(MetaSrvSnafu)
-        info!(
-            "Delete range, key: {}, end: {}, deleted: {}",
-            String::from_utf8_lossy(key),
-            String::from_utf8_lossy(end),
-            resp.deleted()
-        );
-
-        Ok(())
     }
 
-    async fn compare_and_set(
+    async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
+        self.client
+            .batch_delete(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
+    }
+
+    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
+        self.client
+            .batch_get(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
+    }
+
+    async fn compare_and_put(
         &self,
-        key: &[u8],
+        request: CompareAndPutRequest,
-        expect: &[u8],
+    ) -> Result<CompareAndPutResponse> {
-        val: &[u8],
+        self.client
-    ) -> Result<std::result::Result<(), Option<Vec<u8>>>> {
-        let request = CompareAndPutRequest::new()
-            .with_key(key.to_vec())
-            .with_expect(expect.to_vec())
-            .with_value(val.to_vec());
-        let mut response = self
-            .client
             .compare_and_put(request)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)?;
+            .context(MetaSrvSnafu)
-        if response.is_success() {
-            Ok(Ok(()))
-        } else {
-            Ok(Err(response.take_prev_kv().map(|v| v.value().to_vec())))
-        }
     }
 
-    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<()> {
+    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
-        let req = MoveValueRequest::new(from_key, to_key);
+        self.client
-        let _ = self
-            .client
             .move_value(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)?;
+            .context(MetaSrvSnafu)
-        Ok(())
     }
 
     fn as_any(&self) -> &dyn Any {

src/catalog/src/remote/manager.rs

@@ -14,17 +14,15 @@
 
 use std::any::Any;
 use std::collections::HashSet;
-use std::pin::Pin;
 use std::sync::Arc;
 
-use async_stream::stream;
 use async_trait::async_trait;
 use common_catalog::consts::{MAX_SYS_TABLE_ID, MITO_ENGINE};
 use common_meta::ident::TableIdent;
-use common_meta::kv_backend::{Kv, KvBackendRef};
+use common_meta::kv_backend::KvBackendRef;
+use common_meta::rpc::store::{PutRequest, RangeRequest};
+use common_meta::rpc::KeyValue;
 use common_telemetry::{debug, error, info, warn};
-use futures::Stream;
-use futures_util::{StreamExt, TryStreamExt};
 use metrics::{decrement_gauge, increment_gauge};
 use snafu::ResultExt;
 use table::engine::manager::TableEngineManagerRef;
@@ -74,35 +72,39 @@ impl RemoteCatalogManager {
         }
     }
 
-    async fn iter_remote_catalogs(
+    async fn iter_remote_catalogs(&self) -> Result<Vec<CatalogKey>> {
-        &self,
-    ) -> Pin<Box<dyn Stream<Item = Result<CatalogKey>> + Send + '_>> {
         let catalog_range_prefix = build_catalog_prefix();
-        let mut catalogs = self.backend.range(catalog_range_prefix.as_bytes());
+        let req = RangeRequest::new().with_prefix(catalog_range_prefix.as_bytes());
-        Box::pin(stream!({
-            while let Some(r) = catalogs.next().await {
-                let Kv(k, _) = r.context(TableMetadataManagerSnafu)?;
-                if !k.starts_with(catalog_range_prefix.as_bytes()) {
-                    debug!("Ignoring non-catalog key: {}", String::from_utf8_lossy(&k));
-                    continue;
-                }
 
-                let catalog_key = String::from_utf8_lossy(&k);
+        let kvs = self
-                if let Ok(key) = CatalogKey::parse(&catalog_key) {
+            .backend
-                    yield Ok(key)
+            .range(req)
-                } else {
+            .await
-                    error!("Invalid catalog key: {:?}", catalog_key);
+            .context(TableMetadataManagerSnafu)?
+            .kvs;
+
+        let catalogs = kvs
+            .into_iter()
+            .filter_map(|kv| {
+                let catalog_key = String::from_utf8_lossy(kv.key());
+
+                match CatalogKey::parse(&catalog_key) {
+                    Ok(x) => Some(x),
+                    Err(e) => {
+                        error!(e; "Ignore invalid catalog key {:?}", catalog_key);
+                        None
+                    }
                 }
-            }
+            })
-        }))
+            .collect();
+        Ok(catalogs)
     }
 
     /// Fetch catalogs/schemas/tables from remote catalog manager along with max table id allocated.
     async fn initiate_catalogs(&self) -> Result<()> {
-        let mut catalogs = self.iter_remote_catalogs().await;
+        let catalogs = self.iter_remote_catalogs().await?;
         let mut joins = Vec::new();
-        while let Some(r) = catalogs.next().await {
+        for CatalogKey { catalog_name } in catalogs {
-            let CatalogKey { catalog_name, .. } = r?;
             info!("Fetch catalog from metasrv: {}", catalog_name);
 
             let node_id = self.node_id;
@@ -128,13 +130,11 @@ impl RemoteCatalogManager {
             schema_name: schema_name.to_string(),
         }
         .to_string();
+        let req = PutRequest::new()
+            .with_key(schema_key.as_bytes())
+            .with_value(SchemaValue.as_bytes().context(InvalidCatalogValueSnafu)?);
         self.backend
-            .set(
+            .put(req)
-                schema_key.as_bytes(),
-                &SchemaValue {}
-                    .as_bytes()
-                    .context(InvalidCatalogValueSnafu)?,
-            )
             .await
             .context(TableMetadataManagerSnafu)?;
         info!("Created schema '{schema_key}'");
@@ -143,13 +143,11 @@ impl RemoteCatalogManager {
             catalog_name: catalog_name.to_string(),
         }
         .to_string();
+        let req = PutRequest::new()
+            .with_key(catalog_key.as_bytes())
+            .with_value(CatalogValue.as_bytes().context(InvalidCatalogValueSnafu)?);
         self.backend
-            .set(
+            .put(req)
-                catalog_key.as_bytes(),
-                &CatalogValue {}
-                    .as_bytes()
-                    .context(InvalidCatalogValueSnafu)?,
-            )
             .await
             .context(TableMetadataManagerSnafu)?;
         info!("Created catalog '{catalog_key}");
@@ -174,9 +172,7 @@ impl RemoteCatalogManager {
         schema_name: String,
     ) -> Result<u32> {
         info!("initializing tables in {}.{}", catalog_name, schema_name);
-        let tables = iter_remote_tables(node_id, &backend, &catalog_name, &schema_name).await;
+        let kvs = iter_remote_tables(node_id, &backend, &catalog_name, &schema_name).await?;
-
-        let kvs = tables.try_collect::<Vec<_>>().await?;
         let table_num = kvs.len();
         let joins = kvs
             .into_iter()
@@ -253,15 +249,14 @@ impl RemoteCatalogManager {
         engine_manager: TableEngineManagerRef,
         catalog_name: String,
     ) -> Result<()> {
-        let mut schemas = iter_remote_schemas(&backend, &catalog_name).await;
+        let schemas = iter_remote_schemas(&backend, &catalog_name).await?;
-        let mut joins = Vec::new();
-        while let Some(r) = schemas.next().await {
-            let SchemaKey {
-                catalog_name,
-                schema_name,
-                ..
-            } = r?;
 
+        let mut joins = Vec::new();
+        for SchemaKey {
+            catalog_name,
+            schema_name,
+        } in schemas
+        {
             info!(
                 "Fetch schema from metasrv: {}.{}",
                 &catalog_name, &schema_name
@@ -314,11 +309,11 @@ impl RemoteCatalogManager {
         let table_key = self
             .build_regional_table_key(catalog_name, schema_name, table_name)
             .to_string();
+        let req = PutRequest::new()
+            .with_key(table_key.as_bytes())
+            .with_value(table_value.as_bytes().context(InvalidCatalogValueSnafu)?);
         self.backend
-            .set(
+            .put(req)
-                table_key.as_bytes(),
-                &table_value.as_bytes().context(InvalidCatalogValueSnafu)?,
-            )
             .await
             .context(TableMetadataManagerSnafu)?;
         debug!(
@@ -349,7 +344,7 @@ impl RemoteCatalogManager {
             .get(table_key.as_bytes())
             .await
             .context(TableMetadataManagerSnafu)?
-            .map(|Kv(_, v)| {
+            .map(|KeyValue { key: _, value: v }| {
                 let TableRegionalValue {
                     table_id,
                     engine_name,
@@ -367,7 +362,7 @@ impl RemoteCatalogManager {
             };
 
         self.backend
-            .delete(table_key.as_bytes())
+            .delete(table_key.as_bytes(), false)
             .await
             .context(TableMetadataManagerSnafu)?;
         debug!(
@@ -430,23 +425,30 @@ impl RemoteCatalogManager {
 async fn iter_remote_schemas<'a>(
     backend: &'a KvBackendRef,
     catalog_name: &'a str,
-) -> Pin<Box<dyn Stream<Item = Result<SchemaKey>> + Send + 'a>> {
+) -> Result<Vec<SchemaKey>> {
     let schema_prefix = build_schema_prefix(catalog_name);
-    let mut schemas = backend.range(schema_prefix.as_bytes());
+    let req = RangeRequest::new().with_prefix(schema_prefix.as_bytes());
 
-    Box::pin(stream!({
+    let kvs = backend
-        while let Some(r) = schemas.next().await {
+        .range(req)
-            let Kv(k, _) = r.context(TableMetadataManagerSnafu)?;
+        .await
-            if !k.starts_with(schema_prefix.as_bytes()) {
+        .context(TableMetadataManagerSnafu)?
-                debug!("Ignoring non-schema key: {}", String::from_utf8_lossy(&k));
+        .kvs;
-                continue;
+
+    let schemas = kvs
+        .into_iter()
+        .filter_map(|kv| {
+            let schema_key = String::from_utf8_lossy(kv.key());
+            match SchemaKey::parse(&schema_key) {
+                Ok(x) => Some(x),
+                Err(e) => {
+                    warn!("Ignore invalid schema key {:?}: {e}", schema_key);
+                    None
+                }
             }
-
+        })
-            let schema_key =
+        .collect();
-                SchemaKey::parse(&String::from_utf8_lossy(&k)).context(InvalidCatalogValueSnafu)?;
+    Ok(schemas)
-            yield Ok(schema_key)
-        }
-    }))
 }
 
 /// Iterate over all table entries on metasrv
@@ -455,35 +457,42 @@ async fn iter_remote_tables<'a>(
     backend: &'a KvBackendRef,
     catalog_name: &'a str,
     schema_name: &'a str,
-) -> Pin<Box<dyn Stream<Item = Result<(TableGlobalKey, TableGlobalValue)>> + Send + 'a>> {
+) -> Result<Vec<(TableGlobalKey, TableGlobalValue)>> {
     let table_prefix = build_table_global_prefix(catalog_name, schema_name);
-    let mut tables = backend.range(table_prefix.as_bytes());
+    let req = RangeRequest::new().with_prefix(table_prefix.as_bytes());
-    Box::pin(stream!({
-        while let Some(r) = tables.next().await {
-            let Kv(k, v) = r.context(TableMetadataManagerSnafu)?;
-            if !k.starts_with(table_prefix.as_bytes()) {
-                debug!("Ignoring non-table prefix: {}", String::from_utf8_lossy(&k));
-                continue;
-            }
-            let table_key = TableGlobalKey::parse(&String::from_utf8_lossy(&k))
-                .context(InvalidCatalogValueSnafu)?;
-            let table_value = TableGlobalValue::from_bytes(&v).context(InvalidCatalogValueSnafu)?;
 
-            info!(
+    let kvs = backend
-                "Found catalog table entry, key: {}, value: {:?}",
+        .range(req)
-                table_key, table_value
+        .await
-            );
+        .context(TableMetadataManagerSnafu)?
-            // metasrv has allocated region ids to current datanode
+        .kvs;
-            if table_value
+
-                .regions_id_map
+    let mut tables = Vec::with_capacity(kvs.len());
-                .get(&node_id)
+    for kv in kvs {
-                .map(|v| !v.is_empty())
+        let tgk = &String::from_utf8_lossy(kv.key());
-                .unwrap_or(false)
+        let Ok(table_key) = TableGlobalKey::parse(tgk) else {
-            {
+            warn!("Ignore invalid table global key {:?}", tgk);
-                yield Ok((table_key, table_value))
+            continue;
-            }
+        };
+
+        let Ok(table_value) = TableGlobalValue::from_bytes(kv.value()) else {
+            warn!("Ignore invalid table global value {:?}", String::from_utf8_lossy(kv.value()));
+            continue;
+        };
+
+        info!("Found catalog table entry, key: {table_key}, value: {table_value:?}");
+
+        // metasrv has allocated region ids to current datanode
+        if table_value
+            .regions_id_map
+            .get(&node_id)
+            .map(|v| !v.is_empty())
+            .unwrap_or(false)
+        {
+            tables.push((table_key, table_value))
         }
-    }))
+    }
+    Ok(tables)
 }
 
 async fn print_regional_key_debug_info(
@@ -500,7 +509,10 @@ async fn print_regional_key_debug_info(
     .to_string();
 
     match backend.get(regional_key.as_bytes()).await {
-        Ok(Some(Kv(_, values_bytes))) => {
+        Ok(Some(KeyValue {
+            key: _,
+            value: values_bytes,
+        })) => {
             debug!(
                 "Node id: {}, TableRegionalKey: {}, value: {},",
                 node_id,
@@ -702,13 +714,11 @@ impl CatalogManager for RemoteCatalogManager {
         let catalog_name = request.catalog;
         let schema_name = request.schema;
         let key = self.build_schema_key(catalog_name, schema_name).to_string();
+        let req = PutRequest::new()
+            .with_key(key.as_bytes())
+            .with_value(SchemaValue.as_bytes().context(InvalidCatalogValueSnafu)?);
         self.backend
-            .set(
+            .put(req)
-                key.as_bytes(),
-                &SchemaValue {}
-                    .as_bytes()
-                    .context(InvalidCatalogValueSnafu)?,
-            )
             .await
             .context(TableMetadataManagerSnafu)?;
 
@@ -729,21 +739,27 @@ impl CatalogManager for RemoteCatalogManager {
             node_id: self.node_id,
         }
         .to_string();
-        let Some(Kv(_, value_bytes)) = self.backend.get(old_table_key.as_bytes()).await.context(TableMetadataManagerSnafu)? else {
+        let Some(KeyValue{ key: _, value }) = self.backend
-            return Ok(false)
+            .get(old_table_key.as_bytes())
-        };
+            .await
+            .context(TableMetadataManagerSnafu)? else {
+                return Ok(false)
+            };
         let new_table_key = TableRegionalKey {
             catalog_name: request.catalog.clone(),
             schema_name: request.schema.clone(),
             table_name: request.new_table_name,
             node_id: self.node_id,
         };
+        let req = PutRequest::new()
+            .with_key(new_table_key.to_string().as_bytes())
+            .with_value(value);
         self.backend
-            .set(new_table_key.to_string().as_bytes(), &value_bytes)
+            .put(req)
             .await
             .context(TableMetadataManagerSnafu)?;
         self.backend
-            .delete(old_table_key.to_string().as_bytes())
+            .delete(old_table_key.to_string().as_bytes(), false)
             .await
             .context(TableMetadataManagerSnafu)?;
         Ok(true)
@@ -796,7 +812,7 @@ impl CatalogManager for RemoteCatalogManager {
             .get(key.as_bytes())
             .await
             .context(TableMetadataManagerSnafu)?
-            .map(|Kv(_, v)| {
+            .map(|KeyValue { key: _, value: v }| {
                 let TableRegionalValue {
                     table_id,
                     engine_name,
@@ -863,16 +879,19 @@ impl CatalogManager for RemoteCatalogManager {
     }
 
     async fn catalog_names(&self) -> Result<Vec<String>> {
-        let mut stream = self.backend.range(CATALOG_KEY_PREFIX.as_bytes());
+        let req = RangeRequest::new().with_prefix(CATALOG_KEY_PREFIX.as_bytes());
+        let kvs = self
+            .backend
+            .range(req)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .kvs;
         let mut catalogs = HashSet::new();
 
-        while let Some(catalog) = stream.next().await {
+        for catalog in kvs {
-            if let Ok(catalog) = catalog {
+            let catalog_key = String::from_utf8_lossy(catalog.key());
-                let catalog_key = String::from_utf8_lossy(&catalog.0);
+            if let Ok(key) = CatalogKey::parse(&catalog_key) {
-
+                let _ = catalogs.insert(key.catalog_name);
-                if let Ok(key) = CatalogKey::parse(&catalog_key) {
-                    let _ = catalogs.insert(key.catalog_name);
-                }
             }
         }
 
@@ -880,18 +899,19 @@ impl CatalogManager for RemoteCatalogManager {
     }
 
     async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
-        let mut stream = self
+        let req = RangeRequest::new().with_prefix(build_schema_prefix(catalog_name).as_bytes());
+        let kvs = self
             .backend
-            .range(build_schema_prefix(catalog_name).as_bytes());
+            .range(req)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .kvs;
         let mut schemas = HashSet::new();
 
-        while let Some(schema) = stream.next().await {
+        for schema in kvs {
-            if let Ok(schema) = schema {
+            let schema_key = String::from_utf8_lossy(schema.key());
-                let schema_key = String::from_utf8_lossy(&schema.0);
+            if let Ok(key) = SchemaKey::parse(&schema_key) {
-
+                let _ = schemas.insert(key.schema_name);
-                if let Ok(key) = SchemaKey::parse(&schema_key) {
-                    let _ = schemas.insert(key.schema_name);
-                }
             }
         }
         Ok(schemas.into_iter().collect())
@@ -901,18 +921,20 @@ impl CatalogManager for RemoteCatalogManager {
         self.check_catalog_schema_exist(catalog_name, schema_name)
             .await?;
 
-        let mut stream = self
+        let req = RangeRequest::new()
+            .with_prefix(build_table_regional_prefix(catalog_name, schema_name).as_bytes());
+        let kvs = self
             .backend
-            .range(build_table_regional_prefix(catalog_name, schema_name).as_bytes());
+            .range(req)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .kvs;
         let mut tables = HashSet::new();
 
-        while let Some(table) = stream.next().await {
+        for table in kvs {
-            if let Ok(table) = table {
+            let table_key = String::from_utf8_lossy(table.key());
-                let table_key = String::from_utf8_lossy(&table.0);
+            if let Ok(key) = TableRegionalKey::parse(&table_key) {
-
+                let _ = tables.insert(key.table_name);
-                if let Ok(key) = TableRegionalKey::parse(&table_key) {
-                    let _ = tables.insert(key.table_name);
-                }
             }
         }
         Ok(tables.into_iter().collect())
@@ -921,13 +943,11 @@ impl CatalogManager for RemoteCatalogManager {
     async fn register_catalog(&self, name: String) -> Result<bool> {
         let key = CatalogKey { catalog_name: name }.to_string();
         // TODO(hl): use compare_and_swap to prevent concurrent update
+        let req = PutRequest::new()
+            .with_key(key.as_bytes())
+            .with_value(CatalogValue.as_bytes().context(InvalidCatalogValueSnafu)?);
         self.backend
-            .set(
+            .put(req)
-                key.as_bytes(),
-                &CatalogValue {}
-                    .as_bytes()
-                    .context(InvalidCatalogValueSnafu)?,
-            )
             .await
             .context(TableMetadataManagerSnafu)?;
         increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);

src/catalog/tests/remote_catalog_tests.rs

@@ -21,6 +21,7 @@ mod tests {
     use std::sync::Arc;
     use std::time::Duration;
 
+    use catalog::error::Error;
     use catalog::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
     use catalog::remote::mock::MockTableEngine;
     use catalog::remote::region_alive_keeper::RegionAliveKeepers;
@@ -30,8 +31,8 @@ mod tests {
     use common_meta::ident::TableIdent;
     use common_meta::kv_backend::memory::MemoryKvBackend;
     use common_meta::kv_backend::KvBackend;
+    use common_meta::rpc::store::{CompareAndPutRequest, PutRequest, RangeRequest};
     use datatypes::schema::RawSchema;
-    use futures_util::StreamExt;
     use table::engine::manager::{MemoryTableEngineManager, TableEngineManagerRef};
     use table::engine::{EngineContext, TableEngineRef};
     use table::requests::CreateTableRequest;
@@ -52,38 +53,35 @@ mod tests {
 
     #[tokio::test]
     async fn test_backend() {
-        common_telemetry::init_default_ut_logging();
+        let backend = MemoryKvBackend::<Error>::default();
-        let backend = MemoryKvBackend::default();
 
         let default_catalog_key = CatalogKey {
             catalog_name: DEFAULT_CATALOG_NAME.to_string(),
         }
         .to_string();
-
+        let req = PutRequest::new()
-        backend
+            .with_key(default_catalog_key.as_bytes())
-            .set(
+            .with_value(CatalogValue.as_bytes().unwrap());
-                default_catalog_key.as_bytes(),
+        backend.put(req).await.unwrap();
-                &CatalogValue {}.as_bytes().unwrap(),
-            )
-            .await
-            .unwrap();
 
         let schema_key = SchemaKey {
             catalog_name: DEFAULT_CATALOG_NAME.to_string(),
             schema_name: DEFAULT_SCHEMA_NAME.to_string(),
         }
         .to_string();
-        backend
+        let req = PutRequest::new()
-            .set(schema_key.as_bytes(), &SchemaValue {}.as_bytes().unwrap())
+            .with_key(schema_key.as_bytes())
-            .await
+            .with_value(SchemaValue.as_bytes().unwrap());
-            .unwrap();
+        backend.put(req).await.unwrap();
 
-        let mut iter = backend.range("__c-".as_bytes());
+        let req = RangeRequest::new().with_prefix(b"__c-".to_vec());
-        let mut res = HashSet::new();
+        let res = backend
-        while let Some(r) = iter.next().await {
+            .range(req)
-            let kv = r.unwrap();
+            .await
-            let _ = res.insert(String::from_utf8_lossy(&kv.0).to_string());
+            .unwrap()
-        }
+            .kvs
+            .into_iter()
+            .map(|kv| String::from_utf8_lossy(kv.key()).to_string());
         assert_eq!(
             vec!["__c-greptime".to_string()],
             res.into_iter().collect::<Vec<_>>()
@@ -98,36 +96,32 @@ mod tests {
             catalog_name: DEFAULT_CATALOG_NAME.to_string(),
         }
         .to_string();
-
+        let req = PutRequest::new()
-        backend
+            .with_key(default_catalog_key.as_bytes())
-            .set(
+            .with_value(CatalogValue.as_bytes().unwrap());
-                default_catalog_key.as_bytes(),
+        backend.put(req).await.unwrap();
-                &CatalogValue {}.as_bytes().unwrap(),
-            )
-            .await
-            .unwrap();
 
         let ret = backend.get(b"__c-greptime").await.unwrap();
         let _ = ret.unwrap();
 
-        let _ = backend
+        let req = CompareAndPutRequest::new()
-            .compare_and_set(
+            .with_key(b"__c-greptime".to_vec())
-                b"__c-greptime",
+            .with_expect(CatalogValue.as_bytes().unwrap())
-                &CatalogValue {}.as_bytes().unwrap(),
+            .with_value(b"123".to_vec());
-                b"123",
+        let _ = backend.compare_and_put(req).await.unwrap();
-            )
-            .await
-            .unwrap();
 
         let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert_eq!(&b"123"[..], &(ret.as_ref().unwrap().1));
+        assert_eq!(b"123", ret.as_ref().unwrap().value.as_slice());
 
-        let _ = backend.set(b"__c-greptime", b"1234").await;
+        let req = PutRequest::new()
+            .with_key(b"__c-greptime".to_vec())
+            .with_value(b"1234".to_vec());
+        let _ = backend.put(req).await;
 
         let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert_eq!(&b"1234"[..], &(ret.as_ref().unwrap().1));
+        assert_eq!(b"1234", ret.unwrap().value.as_slice());
 
-        backend.delete(b"__c-greptime").await.unwrap();
+        backend.delete(b"__c-greptime", false).await.unwrap();
 
         let ret = backend.get(b"__c-greptime").await.unwrap();
         assert!(ret.is_none());
@@ -135,8 +129,16 @@ mod tests {
 
     async fn prepare_components(node_id: u64) -> TestingComponents {
         let backend = Arc::new(MemoryKvBackend::default());
-        backend.set(b"__c-greptime", b"").await.unwrap();
+
-        backend.set(b"__s-greptime-public", b"").await.unwrap();
+        let req = PutRequest::new()
+            .with_key(b"__c-greptime".to_vec())
+            .with_value(b"".to_vec());
+        backend.put(req).await.unwrap();
+
+        let req = PutRequest::new()
+            .with_key(b"__s-greptime-public".to_vec())
+            .with_value(b"".to_vec());
+        backend.put(req).await.unwrap();
 
         let cached_backend = Arc::new(CachedMetaKvBackend::wrap(backend));
 

src/common/meta/Cargo.toml

@@ -13,6 +13,7 @@ common-error = { path = "../error" }
 common-runtime = { path = "../runtime" }
 common-telemetry = { path = "../telemetry" }
 common-time = { path = "../time" }
+etcd-client.workspace = true
 futures.workspace = true
 lazy_static.workspace = true
 prost.workspace = true

src/common/meta/src/error.rs

@@ -15,6 +15,8 @@
 use common_error::prelude::*;
 use serde_json::error::Error as JsonError;
 use snafu::Location;
+use store_api::storage::RegionNumber;
+use table::metadata::TableId;
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
@@ -73,6 +75,22 @@ pub enum Error {
         source: BoxedError,
         location: Location,
     },
+
+    #[snafu(display("Etcd txn error: {err_msg}"))]
+    EtcdTxnOpResponse { err_msg: String, location: Location },
+
+    #[snafu(display(
+        "Failed to move region {} in table {}, err: {}",
+        region,
+        table_id,
+        err_msg
+    ))]
+    MoveRegion {
+        table_id: TableId,
+        region: RegionNumber,
+        err_msg: String,
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -81,12 +99,13 @@ impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         use Error::*;
         match self {
-            IllegalServerState { .. } => StatusCode::Internal,
+            IllegalServerState { .. } | EtcdTxnOpResponse { .. } => StatusCode::Internal,
 
             SerdeJson { .. }
             | RouteInfoCorrupted { .. }
             | InvalidProtoMsg { .. }
-            | InvalidTableMetadata { .. } => StatusCode::Unexpected,
+            | InvalidTableMetadata { .. }
+            | MoveRegion { .. } => StatusCode::Unexpected,
 
             SendMessage { .. }
             | GetKvCache { .. }

src/common/meta/src/key/datanode_table.rs

@@ -12,16 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use futures::{StreamExt, TryStreamExt};
 use serde::{Deserialize, Serialize};
-use snafu::OptionExt;
+use snafu::{ensure, OptionExt};
 use store_api::storage::RegionNumber;
 use table::metadata::TableId;
 
 use super::{DATANODE_TABLE_KEY_PATTERN, DATANODE_TABLE_KEY_PREFIX};
-use crate::error::{ConcurrentModifyRegionsPlacementSnafu, InvalidTableMetadataSnafu, Result};
+use crate::error::{
+    ConcurrentModifyRegionsPlacementSnafu, InvalidTableMetadataSnafu, MoveRegionSnafu, Result,
+};
 use crate::key::{to_removed_key, TableMetaKey};
+use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp};
 use crate::kv_backend::KvBackendRef;
+use crate::rpc::store::{CompareAndPutRequest, MoveValueRequest, RangeRequest};
 use crate::DatanodeId;
 
 struct DatanodeTableKey {
@@ -100,7 +103,7 @@ impl DatanodeTableManager {
         self.kv_backend
             .get(&key.as_raw_key())
             .await?
-            .map(|kv| DatanodeTableValue::try_from_raw_value(kv.1))
+            .map(|kv| DatanodeTableValue::try_from_raw_value(kv.value))
             .transpose()
     }
 
@@ -112,72 +115,123 @@ impl DatanodeTableManager {
     ) -> Result<()> {
         let key = DatanodeTableKey::new(datanode_id, table_id).as_raw_key();
         let val = DatanodeTableValue::new(table_id, regions).try_as_raw_value()?;
-        self.kv_backend
+        let req = CompareAndPutRequest::new().with_key(key).with_value(val);
-            .compare_and_set(&key, &[], &val)
+
-            .await?
+        let resp = self.kv_backend.compare_and_put(req).await?;
-            .map_err(|curr| {
+        if !resp.success {
-                let curr = if let Some(curr) = curr {
+            let curr = resp.prev_kv.map_or_else(
-                    DatanodeTableValue::try_from_raw_value(curr).map_or_else(
+                || "empty".to_string(),
-                        |e| format!("invalid DatanodeTableValue for Datanode {datanode_id}: {e}"),
+                |kv| {
+                    DatanodeTableValue::try_from_raw_value(kv.value).map_or_else(
+                        |e| format!("Invalid DatanodeTableValue for Datanode {datanode_id}: {e}"),
                         |v| format!("{v:?}"),
                     )
-                } else {
+                },
-                    "empty".to_string()
+            );
-                };
+            return ConcurrentModifyRegionsPlacementSnafu {
-                ConcurrentModifyRegionsPlacementSnafu {
+                err_msg: format!("Datanode {datanode_id} already existed {curr}"),
-                    err_msg: format!("Datanode {datanode_id} already existed {curr}"),
+            }
-                }
+            .fail();
-                .build()
+        }
-            })
+        Ok(())
     }
 
     pub async fn remove(&self, datanode_id: DatanodeId, table_id: TableId) -> Result<()> {
         let key = DatanodeTableKey::new(datanode_id, table_id);
         let removed_key = to_removed_key(&String::from_utf8_lossy(&key.as_raw_key()));
-        self.kv_backend
+        let req = MoveValueRequest::new(key.as_raw_key(), removed_key.as_bytes());
-            .move_value(&key.as_raw_key(), removed_key.as_bytes())
+        let _ = self.kv_backend.move_value(req).await?;
-            .await
+        Ok(())
     }
 
-    // TODO(LFC): Use transaction to move region once the KvBackend and KvStore are merged into one.
     pub async fn move_region(
         &self,
         from_datanode: DatanodeId,
         to_datanode: DatanodeId,
         table_id: TableId,
         region: RegionNumber,
-    ) -> Result<()> {
+    ) -> Result<bool> {
         let from_key = DatanodeTableKey::new(from_datanode, table_id);
-        let from_value = self.get(&from_key).await?;
+        let mut from_value = self.get(&from_key).await?.context(MoveRegionSnafu {
-        if let Some(mut from_value) = from_value {
+            table_id,
-            from_value.regions.retain(|x| *x != region);
+            region,
-            from_value.version += 1;
+            err_msg: format!("DatanodeTableKey not found for Datanode {from_datanode}"),
-            self.kv_backend
+        })?;
-                .set(&from_key.as_raw_key(), &from_value.try_as_raw_value()?)
+
-                .await?;
+        ensure!(
-        }
+            from_value.regions.contains(&region),
+            MoveRegionSnafu {
+                table_id,
+                region,
+                err_msg: format!("target region not found in Datanode {from_datanode}"),
+            }
+        );
 
         let to_key = DatanodeTableKey::new(to_datanode, table_id);
         let to_value = self.get(&to_key).await?;
-        if let Some(mut to_value) = to_value {
+
-            to_value.regions.push(region);
+        if let Some(v) = to_value.as_ref() {
-            to_value.version += 1;
+            ensure!(
-            self.kv_backend
+                !v.regions.contains(&region),
-                .set(&to_key.as_raw_key(), &to_value.try_as_raw_value()?)
+                MoveRegionSnafu {
-                .await?;
+                    table_id,
+                    region,
+                    err_msg: format!("target region already existed in Datanode {to_datanode}"),
+                }
+            );
         }
-        Ok(())
+
+        let compares = vec![
+            Compare::with_value(
+                from_key.as_raw_key(),
+                CompareOp::Equal,
+                from_value.try_as_raw_value()?,
+            ),
+            Compare::new(
+                to_key.as_raw_key(),
+                CompareOp::Equal,
+                to_value
+                    .as_ref()
+                    .map(|x| x.try_as_raw_value())
+                    .transpose()?,
+            ),
+        ];
+
+        let mut operations = Vec::with_capacity(2);
+
+        from_value.regions.retain(|x| *x != region);
+        if from_value.regions.is_empty() {
+            operations.push(TxnOp::Delete(from_key.as_raw_key()));
+        } else {
+            from_value.version += 1;
+            operations.push(TxnOp::Put(
+                from_key.as_raw_key(),
+                from_value.try_as_raw_value()?,
+            ));
+        }
+
+        if let Some(mut v) = to_value {
+            v.regions.push(region);
+            v.version += 1;
+            operations.push(TxnOp::Put(to_key.as_raw_key(), v.try_as_raw_value()?));
+        } else {
+            let v = DatanodeTableValue::new(table_id, vec![region]);
+            operations.push(TxnOp::Put(to_key.as_raw_key(), v.try_as_raw_value()?));
+        }
+
+        let txn = Txn::new().when(compares).and_then(operations);
+        let resp = self.kv_backend.txn(txn).await?;
+        Ok(resp.succeeded)
     }
 
     pub async fn tables(&self, datanode_id: DatanodeId) -> Result<Vec<DatanodeTableValue>> {
         let prefix = DatanodeTableKey::prefix(datanode_id);
-        let table_ids = self
+        let req = RangeRequest::new().with_prefix(prefix.as_bytes());
-            .kv_backend
+        let resp = self.kv_backend.range(req).await?;
-            .range(prefix.as_bytes())
+        let table_ids = resp
-            .map(|result| result.map(|kv| DatanodeTableValue::try_from_raw_value(kv.1)))
+            .kvs
-            .try_collect::<Vec<_>>()
-            .await?
             .into_iter()
+            .map(|kv| DatanodeTableValue::try_from_raw_value(kv.value))
             .collect::<Result<Vec<_>>>()?;
         Ok(table_ids)
     }
@@ -195,11 +249,20 @@ mod tests {
     async fn test_move_region() {
         let manager = DatanodeTableManager::new(Arc::new(MemoryKvBackend::default()));
 
-        assert!(manager.create(1, 1, vec![1, 2]).await.is_ok());
+        let result = manager.move_region(1, 2, 1, 1).await;
-        assert!(manager.create(2, 1, vec![3, 4]).await.is_ok());
+        assert!(result.unwrap_err().to_string().contains(
+            "Failed to move region 1 in table 1, err: DatanodeTableKey not found for Datanode 1"
+        ));
 
-        assert!(manager.move_region(1, 2, 1, 1).await.is_ok());
+        assert!(manager.create(1, 1, vec![1, 2, 3]).await.is_ok());
+        let result = manager.move_region(1, 2, 1, 100).await;
+        assert!(result.unwrap_err().to_string().contains(
+            "Failed to move region 100 in table 1, err: target region not found in Datanode 1"
+        ));
 
+        // Move region 1 from datanode 1 to datanode 2.
+        // Note that the DatanodeTableValue is not existed for datanode 2 now.
+        assert!(manager.move_region(1, 2, 1, 1).await.unwrap());
         let value = manager
             .get(&DatanodeTableKey::new(1, 1))
             .await
@@ -209,11 +272,10 @@ mod tests {
             value,
             DatanodeTableValue {
                 table_id: 1,
-                regions: vec![2],
+                regions: vec![2, 3],
                 version: 1,
             }
         );
-
         let value = manager
             .get(&DatanodeTableKey::new(2, 1))
             .await
@@ -223,10 +285,57 @@ mod tests {
             value,
             DatanodeTableValue {
                 table_id: 1,
-                regions: vec![3, 4, 1],
+                regions: vec![1],
+                version: 0,
+            }
+        );
+
+        // Move region 2 from datanode 1 to datanode 2.
+        assert!(manager.move_region(1, 2, 1, 2).await.is_ok());
+        let value = manager
+            .get(&DatanodeTableKey::new(1, 1))
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(
+            value,
+            DatanodeTableValue {
+                table_id: 1,
+                regions: vec![3],
+                version: 2,
+            }
+        );
+        let value = manager
+            .get(&DatanodeTableKey::new(2, 1))
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(
+            value,
+            DatanodeTableValue {
+                table_id: 1,
+                regions: vec![1, 2],
                 version: 1,
             }
         );
+
+        // Move region 3 (the last region) from datanode 1 to datanode 2.
+        assert!(manager.move_region(1, 2, 1, 3).await.is_ok());
+        let value = manager.get(&DatanodeTableKey::new(1, 1)).await.unwrap();
+        assert!(value.is_none());
+        let value = manager
+            .get(&DatanodeTableKey::new(2, 1))
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(
+            value,
+            DatanodeTableValue {
+                table_id: 1,
+                regions: vec![1, 2, 3],
+                version: 2,
+            }
+        );
     }
 
     #[tokio::test]
@@ -262,8 +371,8 @@ mod tests {
             .await
             .unwrap()
             .unwrap();
-        assert_eq!(b"__removed-__dn_table/2/1", kv.0.as_slice());
+        assert_eq!(b"__removed-__dn_table/2/1", kv.key());
-        let value = DatanodeTableValue::try_from_raw_value(kv.1).unwrap();
+        let value = DatanodeTableValue::try_from_raw_value(kv.value).unwrap();
         assert_eq!(value, expected_value);
 
         let values = manager.tables(1).await.unwrap();

src/common/meta/src/key/table_info.rs

@@ -19,6 +19,7 @@ use super::TABLE_INFO_KEY_PREFIX;
 use crate::error::Result;
 use crate::key::{to_removed_key, TableMetaKey};
 use crate::kv_backend::KvBackendRef;
+use crate::rpc::store::{CompareAndPutRequest, MoveValueRequest};
 
 pub struct TableInfoKey {
     table_id: TableId,
@@ -57,11 +58,18 @@ impl TableInfoManager {
         self.kv_backend
             .get(&raw_key)
             .await?
-            .map(|x| TableInfoValue::try_from_raw_value(x.1))
+            .map(|x| TableInfoValue::try_from_raw_value(x.value))
             .transpose()
     }
 
-    pub async fn compare_and_set(
+    /// Compare and put value of key. `expect` is the expected value, if backend's current value associated
+    /// with key is the same as `expect`, the value will be updated to `val`.
+    ///
+    /// - If the compare-and-set operation successfully updated value, this method will return an `Ok(Ok())`
+    /// - If associated value is not the same as `expect`, no value will be updated and an `Ok(Err(Vec<u8>))`
+    /// will be returned, the `Err(Vec<u8>)` indicates the current associated value of key.
+    /// - If any error happens during operation, an `Err(Error)` will be returned.
+    pub async fn compare_and_put(
         &self,
         table_id: TableId,
         expect: Option<TableInfoValue>,
@@ -82,17 +90,24 @@ impl TableInfoManager {
         };
         let raw_value = value.try_as_raw_value()?;
 
-        self.kv_backend
+        let req = CompareAndPutRequest::new()
-            .compare_and_set(&raw_key, &expect, &raw_value)
+            .with_key(raw_key)
-            .await
+            .with_expect(expect)
+            .with_value(raw_value);
+        let resp = self.kv_backend.compare_and_put(req).await?;
+        Ok(if resp.success {
+            Ok(())
+        } else {
+            Err(resp.prev_kv.map(|x| x.value))
+        })
     }
 
     pub async fn remove(&self, table_id: TableId) -> Result<()> {
-        let key = TableInfoKey::new(table_id);
+        let key = TableInfoKey::new(table_id).as_raw_key();
-        let removed_key = to_removed_key(&String::from_utf8_lossy(key.as_raw_key().as_slice()));
+        let removed_key = to_removed_key(&String::from_utf8_lossy(&key));
-        self.kv_backend
+        let req = MoveValueRequest::new(key, removed_key.as_bytes());
-            .move_value(&key.as_raw_key(), removed_key.as_bytes())
+        self.kv_backend.move_value(req).await?;
-            .await
+        Ok(())
     }
 }
 
@@ -107,6 +122,7 @@ mod tests {
     use super::*;
     use crate::kv_backend::memory::MemoryKvBackend;
     use crate::kv_backend::KvBackend;
+    use crate::rpc::store::PutRequest;
 
     #[tokio::test]
     async fn test_table_info_manager() {
@@ -120,7 +136,8 @@ mod tests {
             }
             .try_as_raw_value()
             .unwrap();
-            backend.set(&key, &val).await.unwrap();
+            let req = PutRequest::new().with_key(key).with_value(val);
+            backend.put(req).await.unwrap();
         }
 
         let manager = TableInfoManager::new(backend.clone());
@@ -137,7 +154,7 @@ mod tests {
 
         let table_info = new_table_info(4);
         let result = manager
-            .compare_and_set(4, None, table_info.clone())
+            .compare_and_put(4, None, table_info.clone())
             .await
             .unwrap();
         assert!(result.is_ok());
@@ -145,7 +162,7 @@ mod tests {
         // test cas failed, the new table info is not set
         let new_table_info = new_table_info(4);
         let result = manager
-            .compare_and_set(4, None, new_table_info.clone())
+            .compare_and_put(4, None, new_table_info.clone())
             .await
             .unwrap();
         let actual = TableInfoValue::try_from_raw_value(result.unwrap_err().unwrap()).unwrap();
@@ -159,7 +176,7 @@ mod tests {
 
         // test cas success
         let result = manager
-            .compare_and_set(4, Some(actual), new_table_info.clone())
+            .compare_and_put(4, Some(actual), new_table_info.clone())
             .await
             .unwrap();
         assert!(result.is_ok());
@@ -171,8 +188,8 @@ mod tests {
             .await
             .unwrap()
             .unwrap();
-        assert_eq!(b"__removed-__table_info/4", kv.0.as_slice());
+        assert_eq!(b"__removed-__table_info/4", kv.key.as_slice());
-        let value = TableInfoValue::try_from_raw_value(kv.1).unwrap();
+        let value = TableInfoValue::try_from_raw_value(kv.value).unwrap();
         assert_eq!(value.table_info, new_table_info);
         assert_eq!(value.version, 1);
     }

src/common/meta/src/key/table_name.rs

@@ -14,7 +14,6 @@
 
 use std::sync::Arc;
 
-use futures::{StreamExt, TryStreamExt};
 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use table::metadata::TableId;
@@ -24,6 +23,7 @@ use crate::error::{InvalidTableMetadataSnafu, Result};
 use crate::key::{to_removed_key, TableMetaKey};
 use crate::kv_backend::memory::MemoryKvBackend;
 use crate::kv_backend::KvBackendRef;
+use crate::rpc::store::{CompareAndPutRequest, MoveValueRequest, RangeRequest};
 
 #[derive(Debug)]
 pub struct TableNameKey<'a> {
@@ -111,11 +111,11 @@ impl TableNameManager {
         let raw_key = key.as_raw_key();
         let value = TableNameValue::new(table_id);
         let raw_value = value.try_as_raw_value()?;
-        let result = self
+        let req = CompareAndPutRequest::new()
-            .kv_backend
+            .with_key(raw_key)
-            .compare_and_set(&raw_key, &[], &raw_value)
+            .with_value(raw_value);
-            .await?;
+        let result = self.kv_backend.compare_and_put(req).await?;
-        Ok(matches!(result, Ok(())))
+        Ok(result.success)
     }
 
     pub async fn get(&self, key: &TableNameKey<'_>) -> Result<Option<TableNameValue>> {
@@ -123,19 +123,18 @@ impl TableNameManager {
         self.kv_backend
             .get(&raw_key)
             .await?
-            .map(|x| TableNameValue::try_from_raw_value(x.1))
+            .map(|x| TableNameValue::try_from_raw_value(x.value))
             .transpose()
     }
 
     pub async fn tables(&self, catalog: &str, schema: &str) -> Result<Vec<String>> {
         let key = TableNameKey::prefix_to_table(catalog, schema).into_bytes();
-        let table_names = self
+        let req = RangeRequest::new().with_prefix(key);
-            .kv_backend
+        let resp = self.kv_backend.range(req).await?;
-            .range(&key)
+        let table_names = resp
-            .map(|x| x.map(|kv| TableNameKey::strip_table_name(&kv.0)))
+            .kvs
-            .try_collect::<Vec<_>>()
-            .await?
             .into_iter()
+            .map(|kv| TableNameKey::strip_table_name(kv.key()))
             .collect::<Result<Vec<_>>>()?;
         Ok(table_names)
     }
@@ -143,9 +142,9 @@ impl TableNameManager {
     pub async fn remove(&self, key: &TableNameKey<'_>) -> Result<()> {
         let raw_key = key.as_raw_key();
         let removed_key = to_removed_key(&String::from_utf8_lossy(&raw_key));
-        self.kv_backend
+        let req = MoveValueRequest::new(raw_key, removed_key.as_bytes());
-            .move_value(&raw_key, removed_key.as_bytes())
+        let _ = self.kv_backend.move_value(req).await?;
-            .await
+        Ok(())
     }
 }
 
@@ -183,7 +182,7 @@ mod tests {
             .await
             .unwrap()
             .unwrap();
-        let value = TableNameValue::try_from_raw_value(kv.1).unwrap();
+        let value = TableNameValue::try_from_raw_value(kv.value).unwrap();
         assert_eq!(value.table_id(), 99);
 
         let tables = manager.tables("my_catalog", "my_schema").await.unwrap();

src/common/meta/src/key/table_region.rs

@@ -22,6 +22,7 @@ use super::TABLE_REGION_KEY_PREFIX;
 use crate::error::Result;
 use crate::key::{to_removed_key, TableMetaKey};
 use crate::kv_backend::KvBackendRef;
+use crate::rpc::store::{CompareAndPutRequest, MoveValueRequest};
 use crate::DatanodeId;
 
 pub type RegionDistribution = BTreeMap<DatanodeId, Vec<RegionNumber>>;
@@ -63,11 +64,18 @@ impl TableRegionManager {
         self.kv_backend
             .get(&raw_key)
             .await?
-            .map(|x| TableRegionValue::try_from_raw_value(x.1))
+            .map(|x| TableRegionValue::try_from_raw_value(x.value))
             .transpose()
     }
 
-    pub async fn compare_and_set(
+    /// Compare and put value of key. `expect` is the expected value, if backend's current value associated
+    /// with key is the same as `expect`, the value will be updated to `val`.
+    ///
+    /// - If the compare-and-set operation successfully updated value, this method will return an `Ok(Ok())`
+    /// - If associated value is not the same as `expect`, no value will be updated and an `Ok(Err(Vec<u8>))`
+    /// will be returned, the `Err(Vec<u8>)` indicates the current associated value of key.
+    /// - If any error happens during operation, an `Err(Error)` will be returned.
+    pub async fn compare_and_put(
         &self,
         table_id: TableId,
         expect: Option<TableRegionValue>,
@@ -88,17 +96,24 @@ impl TableRegionManager {
         };
         let raw_value = value.try_as_raw_value()?;
 
-        self.kv_backend
+        let req = CompareAndPutRequest::new()
-            .compare_and_set(&raw_key, &expect, &raw_value)
+            .with_key(raw_key)
-            .await
+            .with_expect(expect)
+            .with_value(raw_value);
+        let resp = self.kv_backend.compare_and_put(req).await?;
+        Ok(if resp.success {
+            Ok(())
+        } else {
+            Err(resp.prev_kv.map(|x| x.value))
+        })
     }
 
     pub async fn remove(&self, table_id: TableId) -> Result<()> {
-        let key = TableRegionKey::new(table_id);
+        let key = TableRegionKey::new(table_id).as_raw_key();
-        let remove_key = to_removed_key(&String::from_utf8_lossy(key.as_raw_key().as_slice()));
+        let remove_key = to_removed_key(&String::from_utf8_lossy(&key));
-        self.kv_backend
+        let req = MoveValueRequest::new(key, remove_key.as_bytes());
-            .move_value(&key.as_raw_key(), remove_key.as_bytes())
+        self.kv_backend.move_value(req).await?;
-            .await
+        Ok(())
     }
 }
 
@@ -118,7 +133,7 @@ mod tests {
         let region_distribution =
             RegionDistribution::from([(1, vec![1, 2, 3]), (2, vec![4, 5, 6])]);
         let result = manager
-            .compare_and_set(1, None, region_distribution.clone())
+            .compare_and_put(1, None, region_distribution.clone())
             .await
             .unwrap();
         assert!(result.is_ok());
@@ -126,7 +141,7 @@ mod tests {
         let new_region_distribution =
             RegionDistribution::from([(1, vec![4, 5, 6]), (2, vec![1, 2, 3])]);
         let curr = manager
-            .compare_and_set(1, None, new_region_distribution.clone())
+            .compare_and_put(1, None, new_region_distribution.clone())
             .await
             .unwrap()
             .unwrap_err()
@@ -141,7 +156,7 @@ mod tests {
         );
 
         assert!(manager
-            .compare_and_set(1, Some(curr), new_region_distribution.clone())
+            .compare_and_put(1, Some(curr), new_region_distribution.clone())
             .await
             .unwrap()
             .is_ok());
@@ -163,8 +178,8 @@ mod tests {
             .await
             .unwrap()
             .unwrap();
-        assert_eq!(b"__removed-__table_region/1", kv.0.as_slice());
+        assert_eq!(b"__removed-__table_region/1", kv.key.as_slice());
-        let value = TableRegionValue::try_from_raw_value(kv.1).unwrap();
+        let value = TableRegionValue::try_from_raw_value(kv.value).unwrap();
         assert_eq!(value.region_distribution, new_region_distribution);
         assert_eq!(value.version, 1);
     }

src/common/meta/src/kv_backend.rs

@@ -13,68 +13,84 @@
 // limitations under the License.
 
 pub mod memory;
+pub mod txn;
 
 use std::any::Any;
-use std::pin::Pin;
 use std::sync::Arc;
 
 use async_trait::async_trait;
 use common_error::ext::ErrorExt;
-use futures::{Stream, StreamExt};
+pub use txn::TxnService;
 
 use crate::error::Error;
+use crate::rpc::store::{
+    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
+    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
+    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+    RangeRequest, RangeResponse,
+};
+use crate::rpc::KeyValue;
 
-#[derive(Debug, Clone, PartialEq)]
+pub type KvBackendRef = Arc<dyn KvBackend<Error = Error> + Send + Sync>;
-pub struct Kv(pub Vec<u8>, pub Vec<u8>);
-
-pub type ValueIter<'a, E> = Pin<Box<dyn Stream<Item = Result<Kv, E>> + Send + 'a>>;
-
-pub type KvBackendRef = Arc<dyn KvBackend<Error = Error>>;
 
 #[async_trait]
-pub trait KvBackend: Send + Sync {
+pub trait KvBackend: TxnService
-    type Error: ErrorExt;
+where
+    Self::Error: ErrorExt,
+{
+    fn name(&self) -> &str;
 
-    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Self::Error>
+    async fn range(&self, req: RangeRequest) -> Result<RangeResponse, Self::Error>;
-    where
-        'a: 'b;
 
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Self::Error>;
+    async fn put(&self, req: PutRequest) -> Result<PutResponse, Self::Error>;
 
-    /// Compare and set value of key. `expect` is the expected value, if backend's current value associated
+    async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse, Self::Error>;
-    /// with key is the same as `expect`, the value will be updated to `val`.
+
-    ///
+    async fn compare_and_put(
-    /// - If the compare-and-set operation successfully updated value, this method will return an `Ok(Ok())`
-    /// - If associated value is not the same as `expect`, no value will be updated and an `Ok(Err(Vec<u8>))`
-    /// will be returned, the `Err(Vec<u8>)` indicates the current associated value of key.
-    /// - If any error happens during operation, an `Err(Error)` will be returned.
-    async fn compare_and_set(
         &self,
-        key: &[u8],
+        req: CompareAndPutRequest,
-        expect: &[u8],
+    ) -> Result<CompareAndPutResponse, Self::Error>;
-        val: &[u8],
-    ) -> Result<Result<(), Option<Vec<u8>>>, Self::Error>;
 
-    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Self::Error>;
+    async fn delete_range(
+        &self,
+        req: DeleteRangeRequest,
+    ) -> Result<DeleteRangeResponse, Self::Error>;
 
-    async fn delete(&self, key: &[u8]) -> Result<(), Self::Error> {
+    async fn delete(&self, key: &[u8], prev_kv: bool) -> Result<Option<KeyValue>, Self::Error> {
-        self.delete_range(key, &[]).await
+        let mut req = DeleteRangeRequest::new().with_key(key.to_vec());
+        if prev_kv {
+            req = req.with_prev_kv();
+        }
+
+        let resp = self.delete_range(req).await?;
+
+        if prev_kv {
+            Ok(resp.prev_kvs.into_iter().next())
+        } else {
+            Ok(None)
+        }
     }
 
+    async fn batch_delete(
+        &self,
+        req: BatchDeleteRequest,
+    ) -> Result<BatchDeleteResponse, Self::Error>;
+
     /// Default get is implemented based on `range` method.
-    async fn get(&self, key: &[u8]) -> Result<Option<Kv>, Self::Error> {
+    async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>, Self::Error> {
-        let mut iter = self.range(key);
+        let req = RangeRequest::new().with_key(key.to_vec());
-        while let Some(r) = iter.next().await {
+        let mut resp = self.range(req).await?;
-            let kv = r?;
+        Ok(if resp.kvs.is_empty() {
-            if kv.0 == key {
+            None
-                return Ok(Some(kv));
+        } else {
-            }
+            Some(resp.kvs.remove(0))
-        }
+        })
-        return Ok(None);
     }
 
+    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse, Self::Error>;
+
     /// MoveValue atomically renames the key to the given updated key.
-    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<(), Self::Error>;
+    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse, Self::Error>;
 
     fn as_any(&self) -> &dyn Any;
 }

src/common/meta/src/kv_backend/memory.rs

@@ -16,20 +16,32 @@ use std::any::Any;
 use std::collections::btree_map::Entry;
 use std::collections::BTreeMap;
 use std::fmt::{Display, Formatter};
+use std::marker::PhantomData;
+use std::ops::Range;
 use std::sync::RwLock;
 
-use async_stream::stream;
 use async_trait::async_trait;
+use common_error::prelude::ErrorExt;
+use common_telemetry::timer;
 use serde::Serializer;
 
-use crate::error::Error;
+use crate::kv_backend::txn::{Txn, TxnOp, TxnOpResponse, TxnRequest, TxnResponse};
-use crate::kv_backend::{Kv, KvBackend, ValueIter};
+use crate::kv_backend::{KvBackend, TxnService};
+use crate::metrics::METRIC_META_TXN_REQUEST;
+use crate::rpc::store::{
+    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
+    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
+    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+    RangeRequest, RangeResponse,
+};
+use crate::rpc::KeyValue;
 
-pub struct MemoryKvBackend {
+pub struct MemoryKvBackend<T> {
     kvs: RwLock<BTreeMap<Vec<u8>, Vec<u8>>>,
+    _phantom: PhantomData<T>,
 }
 
-impl Display for MemoryKvBackend {
+impl<T> Display for MemoryKvBackend<T> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         let kvs = self.kvs.read().unwrap();
         for (k, v) in kvs.iter() {
@@ -42,93 +54,239 @@ impl Display for MemoryKvBackend {
     }
 }
 
-impl Default for MemoryKvBackend {
+impl<T> Default for MemoryKvBackend<T> {
     fn default() -> Self {
         Self {
             kvs: RwLock::new(BTreeMap::new()),
+            _phantom: PhantomData,
         }
     }
 }
 
+impl<T> MemoryKvBackend<T> {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn clear(&self) {
+        let mut kvs = self.kvs.write().unwrap();
+        kvs.clear();
+    }
+}
+
 #[async_trait]
-impl KvBackend for MemoryKvBackend {
+impl<T: ErrorExt + Send + Sync + 'static> KvBackend for MemoryKvBackend<T> {
-    type Error = Error;
+    fn name(&self) -> &str {
+        "Memory"
+    }
+
+    async fn range(&self, req: RangeRequest) -> Result<RangeResponse, Self::Error> {
+        let RangeRequest {
+            key,
+            range_end,
+            limit,
+            keys_only,
+        } = req;
 
-    fn range<'a, 'b>(&'a self, prefix: &[u8]) -> ValueIter<'b, Error>
-    where
-        'a: 'b,
-    {
         let kvs = self.kvs.read().unwrap();
-        let kvs = kvs.clone();
 
-        let prefix = prefix.to_vec();
+        let iter: Box<dyn Iterator<Item = (&Vec<u8>, &Vec<u8>)>> = if range_end.is_empty() {
-        Box::pin(stream!({
+            Box::new(kvs.get_key_value(&key).into_iter())
-            for (k, v) in kvs.range(prefix.clone()..) {
+        } else {
-                if !k.starts_with(&prefix) {
+            Box::new(kvs.range(key..range_end))
-                    break;
+        };
+        let mut kvs = iter
+            .map(|(k, v)| {
+                let key = k.clone();
+                let value = if keys_only { vec![] } else { v.clone() };
+                KeyValue { key, value }
+            })
+            .collect::<Vec<_>>();
+
+        let more = if limit > 0 && kvs.len() > limit as usize {
+            kvs.truncate(limit as usize);
+            true
+        } else {
+            false
+        };
+
+        Ok(RangeResponse { kvs, more })
+    }
+
+    async fn put(&self, req: PutRequest) -> Result<PutResponse, Self::Error> {
+        let PutRequest {
+            key,
+            value,
+            prev_kv,
+        } = req;
+
+        let mut kvs = self.kvs.write().unwrap();
+
+        let prev_kv = if prev_kv {
+            kvs.insert(key.clone(), value)
+                .map(|value| KeyValue { key, value })
+        } else {
+            kvs.insert(key, value);
+            None
+        };
+
+        Ok(PutResponse { prev_kv })
+    }
+
+    async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse, Self::Error> {
+        let mut kvs = self.kvs.write().unwrap();
+
+        let mut prev_kvs = if req.prev_kv {
+            Vec::with_capacity(req.kvs.len())
+        } else {
+            vec![]
+        };
+
+        for kv in req.kvs {
+            if req.prev_kv {
+                if let Some(value) = kvs.insert(kv.key.clone(), kv.value) {
+                    prev_kvs.push(KeyValue { key: kv.key, value });
                 }
-                yield Ok(Kv(k.clone(), v.clone()));
+            } else {
+                kvs.insert(kv.key, kv.value);
             }
-        }))
+        }
+
+        Ok(BatchPutResponse { prev_kvs })
     }
 
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error> {
+    async fn compare_and_put(
-        let mut kvs = self.kvs.write().unwrap();
-        let _ = kvs.insert(key.to_vec(), val.to_vec());
-        Ok(())
-    }
-
-    async fn compare_and_set(
         &self,
-        key: &[u8],
+        req: CompareAndPutRequest,
-        expect: &[u8],
+    ) -> Result<CompareAndPutResponse, Self::Error> {
-        val: &[u8],
+        let CompareAndPutRequest { key, expect, value } = req;
-    ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
-        let key = key.to_vec();
-        let val = val.to_vec();
 
         let mut kvs = self.kvs.write().unwrap();
+
         let existed = kvs.entry(key);
-        Ok(match existed {
+        let (success, prev_kv) = match existed {
             Entry::Vacant(e) => {
-                if expect.is_empty() {
+                let expected = expect.is_empty();
-                    let _ = e.insert(val);
+                if expected {
-                    Ok(())
+                    let _ = e.insert(value);
-                } else {
-                    Err(None)
                 }
+                (expected, None)
             }
             Entry::Occupied(mut existed) => {
-                if existed.get() == expect {
+                let expected = existed.get() == &expect;
-                    let _ = existed.insert(val);
+                let prev_kv = if expected {
-                    Ok(())
+                    let _ = existed.insert(value);
+                    None
                 } else {
-                    Err(Some(existed.get().clone()))
+                    Some(KeyValue {
-                }
+                        key: existed.key().clone(),
+                        value: existed.get().clone(),
+                    })
+                };
+                (expected, prev_kv)
             }
+        };
+
+        Ok(CompareAndPutResponse { success, prev_kv })
+    }
+
+    async fn delete_range(
+        &self,
+        req: DeleteRangeRequest,
+    ) -> Result<DeleteRangeResponse, Self::Error> {
+        let DeleteRangeRequest {
+            key,
+            range_end,
+            prev_kv,
+        } = req;
+
+        let mut kvs = self.kvs.write().unwrap();
+
+        let prev_kvs = if range_end.is_empty() {
+            kvs.remove(&key)
+                .into_iter()
+                .map(|value| KeyValue {
+                    key: key.clone(),
+                    value,
+                })
+                .collect::<Vec<_>>()
+        } else {
+            let range = Range {
+                start: key,
+                end: range_end,
+            };
+            kvs.drain_filter(|key, _| range.contains(key))
+                .map(Into::into)
+                .collect::<Vec<_>>()
+        };
+
+        Ok(DeleteRangeResponse {
+            deleted: prev_kvs.len() as i64,
+            prev_kvs: if prev_kv { prev_kvs } else { vec![] },
         })
     }
 
-    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error> {
+    async fn batch_delete(
+        &self,
+        req: BatchDeleteRequest,
+    ) -> Result<BatchDeleteResponse, Self::Error> {
         let mut kvs = self.kvs.write().unwrap();
-        if end.is_empty() {
-            let _ = kvs.remove(key);
-        } else {
-            let start = key.to_vec();
-            let end = end.to_vec();
-            let range = start..end;
 
-            kvs.retain(|k, _| !range.contains(k));
+        let mut prev_kvs = if req.prev_kv {
+            Vec::with_capacity(req.keys.len())
+        } else {
+            vec![]
+        };
+
+        for key in req.keys {
+            if req.prev_kv {
+                if let Some(value) = kvs.remove(&key) {
+                    prev_kvs.push(KeyValue { key, value });
+                }
+            } else {
+                kvs.remove(&key);
+            }
         }
-        Ok(())
+
+        Ok(BatchDeleteResponse { prev_kvs })
     }
 
-    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<(), Error> {
+    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse, Self::Error> {
+        let kvs = self.kvs.read().unwrap();
+
+        let kvs = req
+            .keys
+            .into_iter()
+            .filter_map(|key| {
+                kvs.get_key_value(&key).map(|(k, v)| KeyValue {
+                    key: k.clone(),
+                    value: v.clone(),
+                })
+            })
+            .collect::<Vec<_>>();
+
+        Ok(BatchGetResponse { kvs })
+    }
+
+    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse, Self::Error> {
+        let MoveValueRequest { from_key, to_key } = req;
+
         let mut kvs = self.kvs.write().unwrap();
-        if let Some(v) = kvs.remove(from_key) {
+
-            let _ = kvs.insert(to_key.to_vec(), v);
+        let kv = if let Some(v) = kvs.remove(&from_key) {
-        }
+            kvs.insert(to_key, v.clone());
-        Ok(())
+            Some(KeyValue {
+                key: from_key,
+                value: v,
+            })
+        } else {
+            kvs.get(&to_key).map(|v| KeyValue {
+                key: to_key,
+                value: v.clone(),
+            })
+        };
+
+        Ok(MoveValueResponse(kv))
     }
 
     fn as_any(&self) -> &dyn Any {
@@ -136,62 +294,383 @@ impl KvBackend for MemoryKvBackend {
     }
 }
 
-#[cfg(test)]
+#[async_trait]
-mod tests {
+impl<T: ErrorExt + Send + Sync> TxnService for MemoryKvBackend<T> {
-    use futures::TryStreamExt;
+    type Error = T;
 
-    use super::*;
+    async fn txn(&self, txn: Txn) -> Result<TxnResponse, Self::Error> {
-
+        let _timer = timer!(
-    #[tokio::test]
+            METRIC_META_TXN_REQUEST,
-    async fn test_memory_kv_backend() {
+            &[("target", "memory"), ("op", "txn")]
-        let backend = MemoryKvBackend::default();
-
-        for i in 1..10 {
-            let key = format!("key{}", i);
-            let val = format!("val{}", i);
-            assert!(backend.set(key.as_bytes(), val.as_bytes()).await.is_ok());
-        }
-
-        let result = backend
-            .compare_and_set(b"hello", b"what", b"world")
-            .await
-            .unwrap();
-        assert!(result.unwrap_err().is_none());
-
-        let result = backend
-            .compare_and_set(b"hello", b"", b"world")
-            .await
-            .unwrap();
-        assert!(result.is_ok());
-
-        let result = backend
-            .compare_and_set(b"hello", b"world", b"greptime")
-            .await
-            .unwrap();
-        assert!(result.is_ok());
-
-        let result = backend
-            .compare_and_set(b"hello", b"world", b"what")
-            .await
-            .unwrap();
-        assert_eq!(result.unwrap_err().unwrap(), b"greptime");
-
-        assert!(backend.delete_range(b"key1", &[]).await.is_ok());
-        assert!(backend.delete_range(b"key3", b"key9").await.is_ok());
-
-        assert!(backend.move_value(b"key9", b"key10").await.is_ok());
-
-        assert_eq!(
-            backend.to_string(),
-            r#"hello -> greptime
-key10 -> val9
-key2 -> val2
-"#
         );
 
-        let range = backend.range(b"key").try_collect::<Vec<_>>().await.unwrap();
+        let TxnRequest {
-        assert_eq!(range.len(), 2);
+            compare,
-        assert_eq!(range[0], Kv(b"key10".to_vec(), b"val9".to_vec()));
+            success,
-        assert_eq!(range[1], Kv(b"key2".to_vec(), b"val2".to_vec()));
+            failure,
+        } = txn.into();
+
+        let mut kvs = self.kvs.write().unwrap();
+
+        let succeeded = compare
+            .iter()
+            .all(|x| x.compare_with_value(kvs.get(&x.key)));
+
+        let do_txn = |txn_op| match txn_op {
+            TxnOp::Put(key, value) => {
+                let prev_value = kvs.insert(key.clone(), value);
+                let prev_kv = prev_value.map(|value| KeyValue { key, value });
+                TxnOpResponse::ResponsePut(PutResponse { prev_kv })
+            }
+
+            TxnOp::Get(key) => {
+                let value = kvs.get(&key);
+                let kvs = value
+                    .into_iter()
+                    .map(|value| KeyValue {
+                        key: key.clone(),
+                        value: value.clone(),
+                    })
+                    .collect();
+                TxnOpResponse::ResponseGet(RangeResponse { kvs, more: false })
+            }
+
+            TxnOp::Delete(key) => {
+                let prev_value = kvs.remove(&key);
+                let deleted = prev_value.as_ref().map(|x| x.len()).unwrap_or(0) as i64;
+
+                let prev_kvs = prev_value
+                    .into_iter()
+                    .map(|value| KeyValue {
+                        key: key.clone(),
+                        value,
+                    })
+                    .collect();
+                TxnOpResponse::ResponseDelete(DeleteRangeResponse { deleted, prev_kvs })
+            }
+        };
+
+        let responses: Vec<_> = if succeeded { success } else { failure }
+            .into_iter()
+            .map(do_txn)
+            .collect();
+
+        Ok(TxnResponse {
+            succeeded,
+            responses,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::atomic::{AtomicU8, Ordering};
+    use std::sync::Arc;
+
+    use super::*;
+    use crate::error::Error;
+    use crate::kv_backend::KvBackend;
+    use crate::rpc::store::{BatchGetRequest, BatchPutRequest};
+    use crate::rpc::KeyValue;
+    use crate::util;
+
+    async fn mock_mem_store_with_data() -> MemoryKvBackend<Error> {
+        let kv_store = MemoryKvBackend::<Error>::new();
+        let kvs = mock_kvs();
+
+        assert!(kv_store
+            .batch_put(BatchPutRequest {
+                kvs,
+                ..Default::default()
+            })
+            .await
+            .is_ok());
+
+        assert!(kv_store
+            .put(PutRequest {
+                key: b"key11".to_vec(),
+                value: b"val11".to_vec(),
+                ..Default::default()
+            })
+            .await
+            .is_ok());
+
+        kv_store
+    }
+
+    fn mock_kvs() -> Vec<KeyValue> {
+        vec![
+            KeyValue {
+                key: b"key1".to_vec(),
+                value: b"val1".to_vec(),
+            },
+            KeyValue {
+                key: b"key2".to_vec(),
+                value: b"val2".to_vec(),
+            },
+            KeyValue {
+                key: b"key3".to_vec(),
+                value: b"val3".to_vec(),
+            },
+        ]
+    }
+
+    #[tokio::test]
+    async fn test_put() {
+        let kv_store = mock_mem_store_with_data().await;
+
+        let resp = kv_store
+            .put(PutRequest {
+                key: b"key11".to_vec(),
+                value: b"val12".to_vec(),
+                prev_kv: false,
+            })
+            .await
+            .unwrap();
+        assert!(resp.prev_kv.is_none());
+
+        let resp = kv_store
+            .put(PutRequest {
+                key: b"key11".to_vec(),
+                value: b"val13".to_vec(),
+                prev_kv: true,
+            })
+            .await
+            .unwrap();
+        let prev_kv = resp.prev_kv.unwrap();
+        assert_eq!(b"key11", prev_kv.key());
+        assert_eq!(b"val12", prev_kv.value());
+    }
+
+    #[tokio::test]
+    async fn test_range() {
+        let kv_store = mock_mem_store_with_data().await;
+
+        let key = b"key1".to_vec();
+        let range_end = util::get_prefix_end_key(b"key1");
+
+        let resp = kv_store
+            .range(RangeRequest {
+                key: key.clone(),
+                range_end: range_end.clone(),
+                limit: 0,
+                keys_only: false,
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(2, resp.kvs.len());
+        assert_eq!(b"key1", resp.kvs[0].key());
+        assert_eq!(b"val1", resp.kvs[0].value());
+        assert_eq!(b"key11", resp.kvs[1].key());
+        assert_eq!(b"val11", resp.kvs[1].value());
+
+        let resp = kv_store
+            .range(RangeRequest {
+                key: key.clone(),
+                range_end: range_end.clone(),
+                limit: 0,
+                keys_only: true,
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(2, resp.kvs.len());
+        assert_eq!(b"key1", resp.kvs[0].key());
+        assert_eq!(b"", resp.kvs[0].value());
+        assert_eq!(b"key11", resp.kvs[1].key());
+        assert_eq!(b"", resp.kvs[1].value());
+
+        let resp = kv_store
+            .range(RangeRequest {
+                key: key.clone(),
+                limit: 0,
+                keys_only: false,
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(1, resp.kvs.len());
+        assert_eq!(b"key1", resp.kvs[0].key());
+        assert_eq!(b"val1", resp.kvs[0].value());
+
+        let resp = kv_store
+            .range(RangeRequest {
+                key,
+                range_end,
+                limit: 1,
+                keys_only: false,
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(1, resp.kvs.len());
+        assert_eq!(b"key1", resp.kvs[0].key());
+        assert_eq!(b"val1", resp.kvs[0].value());
+    }
+
+    #[tokio::test]
+    async fn test_batch_get() {
+        let kv_store = mock_mem_store_with_data().await;
+
+        let keys = vec![];
+        let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
+
+        assert!(resp.kvs.is_empty());
+
+        let keys = vec![b"key10".to_vec()];
+        let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
+
+        assert!(resp.kvs.is_empty());
+
+        let keys = vec![b"key1".to_vec(), b"key3".to_vec(), b"key4".to_vec()];
+        let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
+
+        assert_eq!(2, resp.kvs.len());
+        assert_eq!(b"key1", resp.kvs[0].key());
+        assert_eq!(b"val1", resp.kvs[0].value());
+        assert_eq!(b"key3", resp.kvs[1].key());
+        assert_eq!(b"val3", resp.kvs[1].value());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_compare_and_put() {
+        let kv_store = Arc::new(MemoryKvBackend::<Error>::new());
+        let success = Arc::new(AtomicU8::new(0));
+
+        let mut joins = vec![];
+        for _ in 0..20 {
+            let kv_store_clone = kv_store.clone();
+            let success_clone = success.clone();
+            let join = tokio::spawn(async move {
+                let req = CompareAndPutRequest {
+                    key: b"key".to_vec(),
+                    expect: vec![],
+                    value: b"val_new".to_vec(),
+                };
+                let resp = kv_store_clone.compare_and_put(req).await.unwrap();
+                if resp.success {
+                    success_clone.fetch_add(1, Ordering::SeqCst);
+                }
+            });
+            joins.push(join);
+        }
+
+        for join in joins {
+            join.await.unwrap();
+        }
+
+        assert_eq!(1, success.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn test_delete_range() {
+        let kv_store = mock_mem_store_with_data().await;
+
+        let req = DeleteRangeRequest {
+            key: b"key3".to_vec(),
+            range_end: vec![],
+            prev_kv: true,
+        };
+
+        let resp = kv_store.delete_range(req).await.unwrap();
+        assert_eq!(1, resp.prev_kvs.len());
+        assert_eq!(b"key3", resp.prev_kvs[0].key());
+        assert_eq!(b"val3", resp.prev_kvs[0].value());
+
+        let resp = kv_store.get(b"key3").await.unwrap();
+        assert!(resp.is_none());
+
+        let req = DeleteRangeRequest {
+            key: b"key2".to_vec(),
+            range_end: vec![],
+            prev_kv: false,
+        };
+
+        let resp = kv_store.delete_range(req).await.unwrap();
+        assert!(resp.prev_kvs.is_empty());
+
+        let resp = kv_store.get(b"key2").await.unwrap();
+        assert!(resp.is_none());
+
+        let key = b"key1".to_vec();
+        let range_end = util::get_prefix_end_key(b"key1");
+
+        let req = DeleteRangeRequest {
+            key: key.clone(),
+            range_end: range_end.clone(),
+            prev_kv: true,
+        };
+        let resp = kv_store.delete_range(req).await.unwrap();
+        assert_eq!(2, resp.prev_kvs.len());
+
+        let req = RangeRequest {
+            key,
+            range_end,
+            ..Default::default()
+        };
+        let resp = kv_store.range(req).await.unwrap();
+        assert!(resp.kvs.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_move_value() {
+        let kv_store = mock_mem_store_with_data().await;
+
+        let req = MoveValueRequest {
+            from_key: b"key1".to_vec(),
+            to_key: b"key111".to_vec(),
+        };
+
+        let resp = kv_store.move_value(req).await.unwrap();
+        assert_eq!(b"key1", resp.0.as_ref().unwrap().key());
+        assert_eq!(b"val1", resp.0.as_ref().unwrap().value());
+
+        let kv_store = mock_mem_store_with_data().await;
+
+        let req = MoveValueRequest {
+            from_key: b"notexistkey".to_vec(),
+            to_key: b"key222".to_vec(),
+        };
+
+        let resp = kv_store.move_value(req).await.unwrap();
+        assert!(resp.0.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_batch_delete() {
+        let kv_store = mock_mem_store_with_data().await;
+
+        assert!(kv_store.get(b"key1").await.unwrap().is_some());
+        assert!(kv_store.get(b"key100").await.unwrap().is_none());
+
+        let req = BatchDeleteRequest {
+            keys: vec![b"key1".to_vec(), b"key100".to_vec()],
+            prev_kv: true,
+        };
+        let resp = kv_store.batch_delete(req).await.unwrap();
+        assert_eq!(1, resp.prev_kvs.len());
+        assert_eq!(
+            vec![KeyValue {
+                key: b"key1".to_vec(),
+                value: b"val1".to_vec()
+            }],
+            resp.prev_kvs
+        );
+        assert!(kv_store.get(b"key1").await.unwrap().is_none());
+
+        assert!(kv_store.get(b"key2").await.unwrap().is_some());
+        assert!(kv_store.get(b"key3").await.unwrap().is_some());
+
+        let req = BatchDeleteRequest {
+            keys: vec![b"key2".to_vec(), b"key3".to_vec()],
+            prev_kv: false,
+        };
+        let resp = kv_store.batch_delete(req).await.unwrap();
+        assert!(resp.prev_kvs.is_empty());
+
+        assert!(kv_store.get(b"key2").await.unwrap().is_none());
+        assert!(kv_store.get(b"key3").await.unwrap().is_none());
     }
 }

src/common/meta/src/kv_backend/txn.rs

@@ -12,15 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::meta::{DeleteRangeResponse, PutResponse, RangeResponse};
-
-use crate::error::Result;
-
 mod etcd;
 
+use common_error::prelude::ErrorExt;
+
+use crate::rpc::store::{DeleteRangeResponse, PutResponse, RangeResponse};
+
 #[async_trait::async_trait]
 pub trait TxnService: Sync + Send {
-    async fn txn(&self, _txn: Txn) -> Result<TxnResponse> {
+    type Error: ErrorExt;
+
+    async fn txn(&self, _txn: Txn) -> Result<TxnResponse, Self::Error> {
         unimplemented!("txn is not implemented")
     }
 }
@@ -169,11 +171,14 @@ impl From<Txn> for TxnRequest {
 
 #[cfg(test)]
 mod tests {
-    use api::v1::meta::{KeyValue, PutRequest};
+    use std::sync::Arc;
 
     use super::*;
-    use crate::service::store::ext::KvStoreExt;
+    use crate::error::Error;
-    use crate::service::store::kv::KvStoreRef;
+    use crate::kv_backend::memory::MemoryKvBackend;
+    use crate::kv_backend::KvBackendRef;
+    use crate::rpc::store::PutRequest;
+    use crate::rpc::KeyValue;
 
     #[test]
     fn test_compare() {
@@ -301,7 +306,7 @@ mod tests {
     async fn test_txn_compare_equal() {
         let kv_store = create_kv_store().await;
         let key = vec![101u8];
-        let _ = kv_store.delete(key.clone(), false).await.unwrap();
+        kv_store.delete(&key, false).await.unwrap();
 
         let txn = Txn::new()
             .when(vec![Compare::with_not_exist_value(
@@ -332,7 +337,7 @@ mod tests {
     async fn test_txn_compare_greater() {
         let kv_store = create_kv_store().await;
         let key = vec![102u8];
-        let _ = kv_store.delete(key.clone(), false).await.unwrap();
+        kv_store.delete(&key, false).await.unwrap();
 
         let txn = Txn::new()
             .when(vec![Compare::with_not_exist_value(
@@ -361,10 +366,9 @@ mod tests {
         assert_eq!(
             res,
             TxnOpResponse::ResponseGet(RangeResponse {
-                header: None,
                 kvs: vec![KeyValue {
                     key,
-                    value: vec![1],
+                    value: vec![1]
                 }],
                 more: false,
             })
@@ -375,7 +379,7 @@ mod tests {
     async fn test_txn_compare_less() {
         let kv_store = create_kv_store().await;
         let key = vec![103u8];
-        let _ = kv_store.delete(vec![3], false).await.unwrap();
+        kv_store.delete(&[3], false).await.unwrap();
 
         let txn = Txn::new()
             .when(vec![Compare::with_not_exist_value(
@@ -404,10 +408,9 @@ mod tests {
         assert_eq!(
             res,
             TxnOpResponse::ResponseGet(RangeResponse {
-                header: None,
                 kvs: vec![KeyValue {
                     key,
-                    value: vec![2],
+                    value: vec![2]
                 }],
                 more: false,
             })
@@ -418,7 +421,7 @@ mod tests {
     async fn test_txn_compare_not_equal() {
         let kv_store = create_kv_store().await;
         let key = vec![104u8];
-        let _ = kv_store.delete(key.clone(), false).await.unwrap();
+        kv_store.delete(&key, false).await.unwrap();
 
         let txn = Txn::new()
             .when(vec![Compare::with_not_exist_value(
@@ -447,18 +450,17 @@ mod tests {
         assert_eq!(
             res,
             TxnOpResponse::ResponseGet(RangeResponse {
-                header: None,
                 kvs: vec![KeyValue {
                     key,
-                    value: vec![1],
+                    value: vec![1]
                 }],
                 more: false,
             })
         );
     }
 
-    async fn create_kv_store() -> KvStoreRef {
+    async fn create_kv_store() -> KvBackendRef {
-        std::sync::Arc::new(crate::service::store::memory::MemStore::new())
+        Arc::new(MemoryKvBackend::<Error>::new())
         // TODO(jiachun): Add a feature to test against etcd in github CI
         //
         // The same test can be run against etcd by uncommenting the following line

src/common/meta/src/kv_backend/txn/etcd.rs

@@ -12,15 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::meta::{DeleteRangeResponse, PutResponse, RangeResponse};
 use etcd_client::{
     Compare as EtcdCompare, CompareOp as EtcdCompareOp, Txn as EtcdTxn, TxnOp as EtcdTxnOp,
     TxnOpResponse as EtcdTxnOpResponse, TxnResponse as EtcdTxnResponse,
 };
 
+use super::{Compare, CompareOp, Txn, TxnOp, TxnOpResponse, TxnResponse};
 use crate::error::{self, Result};
-use crate::service::store::etcd_util::KvPair;
+use crate::rpc::store::{DeleteRangeResponse, PutResponse, RangeResponse};
-use crate::service::store::txn::{Compare, CompareOp, Txn, TxnOp, TxnOpResponse, TxnResponse};
 
 impl From<Txn> for EtcdTxn {
     fn from(txn: Txn) -> Self {
@@ -88,31 +87,25 @@ impl TryFrom<EtcdTxnOpResponse> for TxnOpResponse {
     fn try_from(op_resp: EtcdTxnOpResponse) -> Result<Self> {
         match op_resp {
             EtcdTxnOpResponse::Put(res) => {
-                let prev_kv = res.prev_key().map(KvPair::from_etcd_kv);
+                let prev_kv = res.prev_key().cloned().map(Into::into);
-                let put_res = PutResponse {
+                let put_res = PutResponse { prev_kv };
-                    prev_kv,
-                    ..Default::default()
-                };
                 Ok(TxnOpResponse::ResponsePut(put_res))
             }
             EtcdTxnOpResponse::Get(res) => {
-                let kvs = res.kvs().iter().map(KvPair::from_etcd_kv).collect();
+                let kvs = res.kvs().iter().cloned().map(Into::into).collect();
-                let range_res = RangeResponse {
+                let range_res = RangeResponse { kvs, more: false };
-                    kvs,
-                    ..Default::default()
-                };
                 Ok(TxnOpResponse::ResponseGet(range_res))
             }
             EtcdTxnOpResponse::Delete(res) => {
                 let prev_kvs = res
                     .prev_kvs()
                     .iter()
-                    .map(KvPair::from_etcd_kv)
+                    .cloned()
+                    .map(Into::into)
                     .collect::<Vec<_>>();
                 let delete_res = DeleteRangeResponse {
                     prev_kvs,
                     deleted: res.deleted(),
-                    ..Default::default()
                 };
                 Ok(TxnOpResponse::ResponseDelete(delete_res))
             }

src/common/meta/src/lib.rs

@@ -12,15 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(btree_drain_filter)]
+
 pub mod error;
 pub mod heartbeat;
 pub mod ident;
 pub mod instruction;
 pub mod key;
 pub mod kv_backend;
+pub mod metrics;
 pub mod peer;
 pub mod rpc;
 pub mod table_name;
+pub mod util;
 
 pub type ClusterId = u64;
 pub type DatanodeId = u64;

src/common/meta/src/metrics.rs

@@ -0,0 +1,15 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const METRIC_META_TXN_REQUEST: &str = "meta.txn_request";

src/common/meta/src/rpc.rs

@@ -18,17 +18,14 @@ pub mod router;
 pub mod store;
 pub mod util;
 
+use std::fmt::{Display, Formatter};
+
 use api::v1::meta::{KeyValue as PbKeyValue, ResponseHeader as PbResponseHeader};
 
 #[derive(Debug, Clone)]
 pub struct ResponseHeader(PbResponseHeader);
 
 impl ResponseHeader {
-    #[inline]
-    pub(crate) fn new(header: PbResponseHeader) -> Self {
-        Self(header)
-    }
-
     #[inline]
     pub fn protocol_version(&self) -> u64 {
         self.0.protocol_version
@@ -56,33 +53,83 @@ impl ResponseHeader {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
-pub struct KeyValue(PbKeyValue);
+pub struct KeyValue {
+    pub key: Vec<u8>,
+    pub value: Vec<u8>,
+}
+
+impl From<KeyValue> for PbKeyValue {
+    fn from(kv: KeyValue) -> Self {
+        Self {
+            key: kv.key,
+            value: kv.value,
+        }
+    }
+}
+
+impl From<etcd_client::KeyValue> for KeyValue {
+    fn from(kv: etcd_client::KeyValue) -> Self {
+        Self {
+            key: kv.key().to_vec(),
+            value: kv.value().to_vec(),
+        }
+    }
+}
+
+impl From<KeyValue> for (Vec<u8>, Vec<u8>) {
+    fn from(kv: KeyValue) -> Self {
+        (kv.key, kv.value)
+    }
+}
+
+impl From<(Vec<u8>, Vec<u8>)> for KeyValue {
+    fn from(kv: (Vec<u8>, Vec<u8>)) -> Self {
+        Self {
+            key: kv.0,
+            value: kv.1,
+        }
+    }
+}
+
+impl Display for KeyValue {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "({}, {})",
+            String::from_utf8_lossy(&self.key),
+            String::from_utf8_lossy(&self.value)
+        )
+    }
+}
 
 impl KeyValue {
     #[inline]
-    pub(crate) fn new(kv: PbKeyValue) -> Self {
+    pub fn new(kv: PbKeyValue) -> Self {
-        Self(kv)
+        Self {
+            key: kv.key,
+            value: kv.value,
+        }
     }
 
     #[inline]
     pub fn key(&self) -> &[u8] {
-        &self.0.key
+        &self.key
     }
 
     #[inline]
     pub fn take_key(&mut self) -> Vec<u8> {
-        std::mem::take(&mut self.0.key)
+        std::mem::take(&mut self.key)
     }
 
     #[inline]
     pub fn value(&self) -> &[u8] {
-        &self.0.value
+        &self.value
     }
 
     #[inline]
     pub fn take_value(&mut self) -> Vec<u8> {
-        std::mem::take(&mut self.0.value)
+        std::mem::take(&mut self.value)
     }
 }
 
@@ -103,7 +150,7 @@ mod tests {
             }),
         };
 
-        let header = ResponseHeader::new(pb_header);
+        let header = ResponseHeader(pb_header);
         assert_eq!(101, header.protocol_version());
         assert_eq!(1, header.cluster_id());
         assert_eq!(100, header.error_code());

src/common/meta/src/rpc/store.rs

@@ -12,21 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::{Display, Formatter};
+
 use api::v1::meta::{
     BatchDeleteRequest as PbBatchDeleteRequest, BatchDeleteResponse as PbBatchDeleteResponse,
     BatchGetRequest as PbBatchGetRequest, BatchGetResponse as PbBatchGetResponse,
     BatchPutRequest as PbBatchPutRequest, BatchPutResponse as PbBatchPutResponse,
     CompareAndPutRequest as PbCompareAndPutRequest,
     CompareAndPutResponse as PbCompareAndPutResponse, DeleteRangeRequest as PbDeleteRangeRequest,
-    DeleteRangeResponse as PbDeleteRangeResponse, KeyValue as PbKeyValue,
+    DeleteRangeResponse as PbDeleteRangeResponse, MoveValueRequest as PbMoveValueRequest,
-    MoveValueRequest as PbMoveValueRequest, MoveValueResponse as PbMoveValueResponse,
+    MoveValueResponse as PbMoveValueResponse, PutRequest as PbPutRequest,
-    PutRequest as PbPutRequest, PutResponse as PbPutResponse, RangeRequest as PbRangeRequest,
+    PutResponse as PbPutResponse, RangeRequest as PbRangeRequest, RangeResponse as PbRangeResponse,
-    RangeResponse as PbRangeResponse,
+    ResponseHeader as PbResponseHeader,
 };
 
 use crate::error;
 use crate::error::Result;
-use crate::rpc::{util, KeyValue, ResponseHeader};
+use crate::rpc::{util, KeyValue};
 
 #[derive(Debug, Clone, Default)]
 pub struct RangeRequest {
@@ -59,6 +61,30 @@ impl From<RangeRequest> for PbRangeRequest {
     }
 }
 
+impl From<PbRangeRequest> for RangeRequest {
+    fn from(value: PbRangeRequest) -> Self {
+        Self {
+            key: value.key,
+            range_end: value.range_end,
+            limit: value.limit,
+            keys_only: value.keys_only,
+        }
+    }
+}
+
+impl Display for RangeRequest {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "RangeRequest{{key: '{}', range_end: '{}', limit: {}, keys_only: {}}}",
+            String::from_utf8_lossy(&self.key),
+            String::from_utf8_lossy(&self.range_end),
+            self.limit,
+            self.keys_only
+        )
+    }
+}
+
 impl RangeRequest {
     #[inline]
     pub fn new() -> Self {
@@ -119,8 +145,26 @@ impl RangeRequest {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
-pub struct RangeResponse(PbRangeResponse);
+pub struct RangeResponse {
+    pub kvs: Vec<KeyValue>,
+    pub more: bool,
+}
+
+impl Display for RangeResponse {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "RangeResponse{{kvs: [{}], more: {}}}",
+            self.kvs
+                .iter()
+                .map(|kv| kv.to_string())
+                .collect::<Vec<_>>()
+                .join(", "),
+            self.more
+        )
+    }
+}
 
 impl TryFrom<PbRangeResponse> for RangeResponse {
     type Error = error::Error;
@@ -128,29 +172,25 @@ impl TryFrom<PbRangeResponse> for RangeResponse {
     fn try_from(pb: PbRangeResponse) -> Result<Self> {
         util::check_response_header(pb.header.as_ref())?;
 
-        Ok(Self::new(pb))
+        Ok(Self {
+            kvs: pb.kvs.into_iter().map(KeyValue::new).collect(),
+            more: pb.more,
+        })
     }
 }
 
 impl RangeResponse {
-    #[inline]
+    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbRangeResponse {
-    pub fn new(res: PbRangeResponse) -> Self {
+        PbRangeResponse {
-        Self(res)
+            header: Some(header),
-    }
+            kvs: self.kvs.into_iter().map(Into::into).collect(),
-
+            more: self.more,
-    #[inline]
+        }
-    pub fn take_header(&mut self) -> Option<ResponseHeader> {
-        self.0.header.take().map(ResponseHeader::new)
     }
 
     #[inline]
     pub fn take_kvs(&mut self) -> Vec<KeyValue> {
-        self.0.kvs.drain(..).map(KeyValue::new).collect()
+        self.kvs.drain(..).collect()
-    }
-
-    #[inline]
-    pub fn more(&self) -> bool {
-        self.0.more
     }
 }
 
@@ -177,6 +217,16 @@ impl From<PutRequest> for PbPutRequest {
     }
 }
 
+impl From<PbPutRequest> for PutRequest {
+    fn from(value: PbPutRequest) -> Self {
+        Self {
+            key: value.key,
+            value: value.value,
+            prev_kv: value.prev_kv,
+        }
+    }
+}
+
 impl PutRequest {
     #[inline]
     pub fn new() -> Self {
@@ -211,8 +261,10 @@ impl PutRequest {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
-pub struct PutResponse(PbPutResponse);
+pub struct PutResponse {
+    pub prev_kv: Option<KeyValue>,
+}
 
 impl TryFrom<PbPutResponse> for PutResponse {
     type Error = error::Error;
@@ -220,27 +272,22 @@ impl TryFrom<PbPutResponse> for PutResponse {
     fn try_from(pb: PbPutResponse) -> Result<Self> {
         util::check_response_header(pb.header.as_ref())?;
 
-        Ok(Self::new(pb))
+        Ok(Self {
+            prev_kv: pb.prev_kv.map(KeyValue::new),
+        })
     }
 }
 
 impl PutResponse {
-    #[inline]
+    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbPutResponse {
-    pub fn new(res: PbPutResponse) -> Self {
+        PbPutResponse {
-        Self(res)
+            header: Some(header),
-    }
+            prev_kv: self.prev_kv.map(Into::into),
-
+        }
-    #[inline]
-    pub fn take_header(&mut self) -> Option<ResponseHeader> {
-        self.0.header.take().map(ResponseHeader::new)
-    }
-
-    #[inline]
-    pub fn take_prev_kv(&mut self) -> Option<KeyValue> {
-        self.0.prev_kv.take().map(KeyValue::new)
     }
 }
 
+#[derive(Clone)]
 pub struct BatchGetRequest {
     pub keys: Vec<Vec<u8>>,
 }
@@ -254,6 +301,12 @@ impl From<BatchGetRequest> for PbBatchGetRequest {
     }
 }
 
+impl From<PbBatchGetRequest> for BatchGetRequest {
+    fn from(value: PbBatchGetRequest) -> Self {
+        Self { keys: value.keys }
+    }
+}
+
 impl Default for BatchGetRequest {
     fn default() -> Self {
         Self::new()
@@ -274,7 +327,23 @@ impl BatchGetRequest {
 }
 
 #[derive(Debug, Clone)]
-pub struct BatchGetResponse(PbBatchGetResponse);
+pub struct BatchGetResponse {
+    pub kvs: Vec<KeyValue>,
+}
+
+impl Display for BatchGetResponse {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "[{}]",
+            self.kvs
+                .iter()
+                .map(|kv| kv.to_string())
+                .collect::<Vec<_>>()
+                .join(", "),
+        )
+    }
+}
 
 impl TryFrom<PbBatchGetResponse> for BatchGetResponse {
     type Error = error::Error;
@@ -282,30 +351,24 @@ impl TryFrom<PbBatchGetResponse> for BatchGetResponse {
     fn try_from(pb: PbBatchGetResponse) -> Result<Self> {
         util::check_response_header(pb.header.as_ref())?;
 
-        Ok(Self(pb))
+        Ok(Self {
+            kvs: pb.kvs.into_iter().map(KeyValue::new).collect(),
+        })
     }
 }
 
 impl BatchGetResponse {
-    #[inline]
+    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbBatchGetResponse {
-    pub fn new(res: PbBatchGetResponse) -> Self {
+        PbBatchGetResponse {
-        Self(res)
+            header: Some(header),
-    }
+            kvs: self.kvs.into_iter().map(Into::into).collect(),
-
+        }
-    #[inline]
-    pub fn take_header(&mut self) -> Option<ResponseHeader> {
-        self.0.header.take().map(ResponseHeader::new)
-    }
-
-    #[inline]
-    pub fn take_kvs(&mut self) -> Vec<KeyValue> {
-        self.0.kvs.drain(..).map(KeyValue::new).collect()
     }
 }
 
 #[derive(Debug, Clone, Default)]
 pub struct BatchPutRequest {
-    pub kvs: Vec<PbKeyValue>,
+    pub kvs: Vec<KeyValue>,
     /// If prev_kv is set, gets the previous key-value pairs before changing it.
     /// The previous key-value pairs will be returned in the batch put response.
     pub prev_kv: bool,
@@ -315,12 +378,21 @@ impl From<BatchPutRequest> for PbBatchPutRequest {
     fn from(req: BatchPutRequest) -> Self {
         Self {
             header: None,
-            kvs: req.kvs,
+            kvs: req.kvs.into_iter().map(Into::into).collect(),
             prev_kv: req.prev_kv,
         }
     }
 }
 
+impl From<PbBatchPutRequest> for BatchPutRequest {
+    fn from(value: PbBatchPutRequest) -> Self {
+        Self {
+            kvs: value.kvs.into_iter().map(KeyValue::new).collect(),
+            prev_kv: value.prev_kv,
+        }
+    }
+}
+
 impl BatchPutRequest {
     #[inline]
     pub fn new() -> Self {
@@ -332,7 +404,7 @@ impl BatchPutRequest {
 
     #[inline]
     pub fn add_kv(mut self, key: impl Into<Vec<u8>>, value: impl Into<Vec<u8>>) -> Self {
-        self.kvs.push(PbKeyValue {
+        self.kvs.push(KeyValue {
             key: key.into(),
             value: value.into(),
         });
@@ -349,7 +421,9 @@ impl BatchPutRequest {
 }
 
 #[derive(Debug, Clone)]
-pub struct BatchPutResponse(PbBatchPutResponse);
+pub struct BatchPutResponse {
+    pub prev_kvs: Vec<KeyValue>,
+}
 
 impl TryFrom<PbBatchPutResponse> for BatchPutResponse {
     type Error = error::Error;
@@ -357,24 +431,23 @@ impl TryFrom<PbBatchPutResponse> for BatchPutResponse {
     fn try_from(pb: PbBatchPutResponse) -> Result<Self> {
         util::check_response_header(pb.header.as_ref())?;
 
-        Ok(Self::new(pb))
+        Ok(Self {
+            prev_kvs: pb.prev_kvs.into_iter().map(KeyValue::new).collect(),
+        })
     }
 }
 
 impl BatchPutResponse {
-    #[inline]
+    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbBatchPutResponse {
-    pub fn new(res: PbBatchPutResponse) -> Self {
+        PbBatchPutResponse {
-        Self(res)
+            header: Some(header),
-    }
+            prev_kvs: self.prev_kvs.into_iter().map(Into::into).collect(),
-
+        }
-    #[inline]
-    pub fn take_header(&mut self) -> Option<ResponseHeader> {
-        self.0.header.take().map(ResponseHeader::new)
     }
 
     #[inline]
     pub fn take_prev_kvs(&mut self) -> Vec<KeyValue> {
-        self.0.prev_kvs.drain(..).map(KeyValue::new).collect()
+        self.prev_kvs.drain(..).collect()
     }
 }
 
@@ -396,6 +469,15 @@ impl From<BatchDeleteRequest> for PbBatchDeleteRequest {
     }
 }
 
+impl From<PbBatchDeleteRequest> for BatchDeleteRequest {
+    fn from(value: PbBatchDeleteRequest) -> Self {
+        Self {
+            keys: value.keys,
+            prev_kv: value.prev_kv,
+        }
+    }
+}
+
 impl BatchDeleteRequest {
     #[inline]
     pub fn new() -> Self {
@@ -421,7 +503,9 @@ impl BatchDeleteRequest {
 }
 
 #[derive(Debug, Clone)]
-pub struct BatchDeleteResponse(PbBatchDeleteResponse);
+pub struct BatchDeleteResponse {
+    pub prev_kvs: Vec<KeyValue>,
+}
 
 impl TryFrom<PbBatchDeleteResponse> for BatchDeleteResponse {
     type Error = error::Error;
@@ -429,24 +513,18 @@ impl TryFrom<PbBatchDeleteResponse> for BatchDeleteResponse {
     fn try_from(pb: PbBatchDeleteResponse) -> Result<Self> {
         util::check_response_header(pb.header.as_ref())?;
 
-        Ok(Self::new(pb))
+        Ok(Self {
+            prev_kvs: pb.prev_kvs.into_iter().map(KeyValue::new).collect(),
+        })
     }
 }
 
 impl BatchDeleteResponse {
-    #[inline]
+    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbBatchDeleteResponse {
-    pub fn new(res: PbBatchDeleteResponse) -> Self {
+        PbBatchDeleteResponse {
-        Self(res)
+            header: Some(header),
-    }
+            prev_kvs: self.prev_kvs.into_iter().map(Into::into).collect(),
-
+        }
-    #[inline]
-    pub fn take_header(&mut self) -> Option<ResponseHeader> {
-        self.0.header.take().map(ResponseHeader::new)
-    }
-
-    #[inline]
-    pub fn take_prev_kvs(&mut self) -> Vec<KeyValue> {
-        self.0.prev_kvs.drain(..).map(KeyValue::new).collect()
     }
 }
 
@@ -471,6 +549,16 @@ impl From<CompareAndPutRequest> for PbCompareAndPutRequest {
     }
 }
 
+impl From<PbCompareAndPutRequest> for CompareAndPutRequest {
+    fn from(value: PbCompareAndPutRequest) -> Self {
+        Self {
+            key: value.key,
+            expect: value.expect,
+            value: value.value,
+        }
+    }
+}
+
 impl CompareAndPutRequest {
     #[inline]
     pub fn new() -> Self {
@@ -504,8 +592,11 @@ impl CompareAndPutRequest {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
-pub struct CompareAndPutResponse(PbCompareAndPutResponse);
+pub struct CompareAndPutResponse {
+    pub success: bool,
+    pub prev_kv: Option<KeyValue>,
+}
 
 impl TryFrom<PbCompareAndPutResponse> for CompareAndPutResponse {
     type Error = error::Error;
@@ -513,29 +604,30 @@ impl TryFrom<PbCompareAndPutResponse> for CompareAndPutResponse {
     fn try_from(pb: PbCompareAndPutResponse) -> Result<Self> {
         util::check_response_header(pb.header.as_ref())?;
 
-        Ok(Self::new(pb))
+        Ok(Self {
+            success: pb.success,
+            prev_kv: pb.prev_kv.map(KeyValue::new),
+        })
     }
 }
 
 impl CompareAndPutResponse {
-    #[inline]
+    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbCompareAndPutResponse {
-    pub fn new(res: PbCompareAndPutResponse) -> Self {
+        PbCompareAndPutResponse {
-        Self(res)
+            header: Some(header),
-    }
+            success: self.success,
-
+            prev_kv: self.prev_kv.map(Into::into),
-    #[inline]
+        }
-    pub fn take_header(&mut self) -> Option<ResponseHeader> {
-        self.0.header.take().map(ResponseHeader::new)
     }
 
     #[inline]
     pub fn is_success(&self) -> bool {
-        self.0.success
+        self.success
     }
 
     #[inline]
     pub fn take_prev_kv(&mut self) -> Option<KeyValue> {
-        self.0.prev_kv.take().map(KeyValue::new)
+        self.prev_kv.take()
     }
 }
 
@@ -571,6 +663,16 @@ impl From<DeleteRangeRequest> for PbDeleteRangeRequest {
     }
 }
 
+impl From<PbDeleteRangeRequest> for DeleteRangeRequest {
+    fn from(value: PbDeleteRangeRequest) -> Self {
+        Self {
+            key: value.key,
+            range_end: value.range_end,
+            prev_kv: value.prev_kv,
+        }
+    }
+}
+
 impl DeleteRangeRequest {
     #[inline]
     pub fn new() -> Self {
@@ -624,8 +726,11 @@ impl DeleteRangeRequest {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
-pub struct DeleteRangeResponse(PbDeleteRangeResponse);
+pub struct DeleteRangeResponse {
+    pub deleted: i64,
+    pub prev_kvs: Vec<KeyValue>,
+}
 
 impl TryFrom<PbDeleteRangeResponse> for DeleteRangeResponse {
     type Error = error::Error;
@@ -633,29 +738,30 @@ impl TryFrom<PbDeleteRangeResponse> for DeleteRangeResponse {
     fn try_from(pb: PbDeleteRangeResponse) -> Result<Self> {
         util::check_response_header(pb.header.as_ref())?;
 
-        Ok(Self::new(pb))
+        Ok(Self {
+            deleted: pb.deleted,
+            prev_kvs: pb.prev_kvs.into_iter().map(KeyValue::new).collect(),
+        })
     }
 }
 
 impl DeleteRangeResponse {
-    #[inline]
+    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbDeleteRangeResponse {
-    pub fn new(res: PbDeleteRangeResponse) -> Self {
+        PbDeleteRangeResponse {
-        Self(res)
+            header: Some(header),
-    }
+            deleted: self.deleted,
-
+            prev_kvs: self.prev_kvs.into_iter().map(Into::into).collect(),
-    #[inline]
+        }
-    pub fn take_header(&mut self) -> Option<ResponseHeader> {
-        self.0.header.take().map(ResponseHeader::new)
     }
 
     #[inline]
     pub fn deleted(&self) -> i64 {
-        self.0.deleted
+        self.deleted
     }
 
     #[inline]
     pub fn take_prev_kvs(&mut self) -> Vec<KeyValue> {
-        self.0.prev_kvs.drain(..).map(KeyValue::new).collect()
+        self.prev_kvs.drain(..).collect()
     }
 }
 
@@ -678,6 +784,15 @@ impl From<MoveValueRequest> for PbMoveValueRequest {
     }
 }
 
+impl From<PbMoveValueRequest> for MoveValueRequest {
+    fn from(value: PbMoveValueRequest) -> Self {
+        Self {
+            from_key: value.from_key,
+            to_key: value.to_key,
+        }
+    }
+}
+
 impl MoveValueRequest {
     #[inline]
     pub fn new(from_key: impl Into<Vec<u8>>, to_key: impl Into<Vec<u8>>) -> Self {
@@ -689,7 +804,7 @@ impl MoveValueRequest {
 }
 
 #[derive(Debug, Clone)]
-pub struct MoveValueResponse(PbMoveValueResponse);
+pub struct MoveValueResponse(pub Option<KeyValue>);
 
 impl TryFrom<PbMoveValueResponse> for MoveValueResponse {
     type Error = error::Error;
@@ -697,24 +812,21 @@ impl TryFrom<PbMoveValueResponse> for MoveValueResponse {
     fn try_from(pb: PbMoveValueResponse) -> Result<Self> {
         util::check_response_header(pb.header.as_ref())?;
 
-        Ok(Self::new(pb))
+        Ok(Self(pb.kv.map(KeyValue::new)))
     }
 }
 
 impl MoveValueResponse {
-    #[inline]
+    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbMoveValueResponse {
-    pub fn new(res: PbMoveValueResponse) -> Self {
+        PbMoveValueResponse {
-        Self(res)
+            header: Some(header),
-    }
+            kv: self.0.map(Into::into),
-
+        }
-    #[inline]
-    pub fn take_header(&mut self) -> Option<ResponseHeader> {
-        self.0.header.take().map(ResponseHeader::new)
     }
 
     #[inline]
     pub fn take_kv(&mut self) -> Option<KeyValue> {
-        self.0.kv.take().map(KeyValue::new)
+        self.0.take()
     }
 }
 
@@ -784,9 +896,8 @@ mod tests {
             more: true,
         };
 
-        let mut res = RangeResponse::new(pb_res);
+        let mut res: RangeResponse = pb_res.try_into().unwrap();
-        assert!(res.take_header().is_none());
+        assert!(res.more);
-        assert!(res.more());
         let mut kvs = res.take_kvs();
         let kv0 = kvs.get_mut(0).unwrap();
         assert_eq!(b"k1".to_vec(), kv0.key().to_vec());
@@ -826,9 +937,8 @@ mod tests {
             }),
         };
 
-        let mut res = PutResponse::new(pb_res);
+        let res: PutResponse = pb_res.try_into().unwrap();
-        assert!(res.take_header().is_none());
+        let mut kv = res.prev_kv.unwrap();
-        let mut kv = res.take_prev_kv().unwrap();
         assert_eq!(b"k1".to_vec(), kv.key().to_vec());
         assert_eq!(b"k1".to_vec(), kv.take_key());
         assert_eq!(b"v1".to_vec(), kv.value().to_vec());
@@ -859,12 +969,8 @@ mod tests {
                 value: b"test_value1".to_vec(),
             }],
         };
-        let mut res = BatchGetResponse::new(pb_res);
+        let res: BatchGetResponse = pb_res.try_into().unwrap();
-
+        let kvs = res.kvs;
-        assert!(res.take_header().is_none());
-
-        let kvs = res.take_kvs();
-
         assert_eq!(b"test_key1".as_slice(), kvs[0].key());
         assert_eq!(b"test_value1".as_slice(), kvs[0].value());
     }
@@ -898,8 +1004,7 @@ mod tests {
             }],
         };
 
-        let mut res = BatchPutResponse::new(pb_res);
+        let mut res: BatchPutResponse = pb_res.try_into().unwrap();
-        assert!(res.take_header().is_none());
         let kvs = res.take_prev_kvs();
         assert_eq!(b"k1".to_vec(), kvs[0].key().to_vec());
         assert_eq!(b"v1".to_vec(), kvs[0].value().to_vec());
@@ -931,9 +1036,8 @@ mod tests {
             }],
         };
 
-        let mut res = BatchDeleteResponse::new(pb_res);
+        let res: BatchDeleteResponse = pb_res.try_into().unwrap();
-        assert!(res.take_header().is_none());
+        let kvs = res.prev_kvs;
-        let kvs = res.take_prev_kvs();
         assert_eq!(b"k1".to_vec(), kvs[0].key().to_vec());
         assert_eq!(b"v1".to_vec(), kvs[0].value().to_vec());
     }
@@ -969,8 +1073,7 @@ mod tests {
             }),
         };
 
-        let mut res = CompareAndPutResponse::new(pb_res);
+        let mut res: CompareAndPutResponse = pb_res.try_into().unwrap();
-        assert!(res.take_header().is_none());
         let mut kv = res.take_prev_kv().unwrap();
         assert_eq!(b"k1".to_vec(), kv.key().to_vec());
         assert_eq!(b"k1".to_vec(), kv.take_key());
@@ -1026,7 +1129,6 @@ mod tests {
         };
 
         let mut res: DeleteRangeResponse = pb_res.try_into().unwrap();
-        assert!(res.take_header().is_none());
         assert_eq!(2, res.deleted());
         let mut kvs = res.take_prev_kvs();
         let kv0 = kvs.get_mut(0).unwrap();
@@ -1064,8 +1166,7 @@ mod tests {
             }),
         };
 
-        let mut res = MoveValueResponse::new(pb_res);
+        let mut res: MoveValueResponse = pb_res.try_into().unwrap();
-        assert!(res.take_header().is_none());
         let mut kv = res.take_kv().unwrap();
         assert_eq!(b"k1".to_vec(), kv.key().to_vec());
         assert_eq!(b"k1".to_vec(), kv.take_key());

src/frontend/src/catalog.rs

@@ -34,11 +34,11 @@ use catalog::{
 use client::client_manager::DatanodeClients;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
 use common_error::prelude::BoxedError;
-use common_meta::kv_backend::{Kv, KvBackendRef};
+use common_meta::kv_backend::KvBackendRef;
+use common_meta::rpc::store::RangeRequest;
+use common_meta::rpc::KeyValue;
 use common_meta::table_name::TableName;
 use common_telemetry::warn;
-use futures::StreamExt;
-use futures_util::TryStreamExt;
 use partition::manager::PartitionRuleManagerRef;
 use snafu::prelude::*;
 use table::table::numbers::NumbersTable;
@@ -252,10 +252,17 @@ impl CatalogManager for FrontendCatalogManager {
 
     async fn catalog_names(&self) -> CatalogResult<Vec<String>> {
         let key = build_catalog_prefix();
-        let mut iter = self.backend.range(key.as_bytes());
+        let req = RangeRequest::new().with_prefix(key.as_bytes());
+
+        let kvs = self
+            .backend
+            .range(req)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .kvs;
+
         let mut res = HashSet::new();
-        while let Some(r) = iter.next().await {
+        for KeyValue { key: k, value: _ } in kvs {
-            let Kv(k, _) = r.context(TableMetadataManagerSnafu)?;
             let catalog_key = String::from_utf8_lossy(&k);
             if let Ok(key) = CatalogKey::parse(catalog_key.as_ref()) {
                 let _ = res.insert(key.catalog_name);
@@ -268,10 +275,17 @@ impl CatalogManager for FrontendCatalogManager {
 
     async fn schema_names(&self, catalog: &str) -> CatalogResult<Vec<String>> {
         let key = build_schema_prefix(catalog);
-        let mut iter = self.backend.range(key.as_bytes());
+        let req = RangeRequest::new().with_prefix(key.as_bytes());
+
+        let kvs = self
+            .backend
+            .range(req)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .kvs;
+
         let mut res = HashSet::new();
-        while let Some(r) = iter.next().await {
+        for KeyValue { key: k, value: _ } in kvs {
-            let Kv(k, _) = r.context(TableMetadataManagerSnafu)?;
             let key =
                 SchemaKey::parse(String::from_utf8_lossy(&k)).context(InvalidCatalogValueSnafu)?;
             let _ = res.insert(key.schema_name);
@@ -285,16 +299,23 @@ impl CatalogManager for FrontendCatalogManager {
             tables.push("numbers".to_string());
         }
         let key = build_table_global_prefix(catalog, schema);
-        let iter = self.backend.range(key.as_bytes());
+        let req = RangeRequest::new().with_prefix(key.as_bytes());
+
+        let iter = self
+            .backend
+            .range(req)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .kvs
+            .into_iter();
+
         let result = iter
-            .map(|r| {
+            .map(|KeyValue { key: k, value: _ }| {
-                let Kv(k, _) = r.context(TableMetadataManagerSnafu)?;
                 let key = TableGlobalKey::parse(String::from_utf8_lossy(&k))
                     .context(InvalidCatalogValueSnafu)?;
                 Ok(key.table_name)
             })
-            .try_collect::<Vec<_>>()
+            .collect::<CatalogResult<Vec<_>>>()?;
-            .await?;
         tables.extend(result);
         Ok(tables)
     }
@@ -377,7 +398,7 @@ impl CatalogManager for FrontendCatalogManager {
         let Some(kv) = self.backend().get(table_global_key.to_string().as_bytes()).await.context(TableMetadataManagerSnafu)? else {
             return Ok(None);
         };
-        let v = TableGlobalValue::from_bytes(kv.1).context(InvalidCatalogValueSnafu)?;
+        let v = TableGlobalValue::from_bytes(kv.value).context(InvalidCatalogValueSnafu)?;
         let table_info = Arc::new(
             v.table_info
                 .try_into()

src/frontend/src/instance/distributed/inserter.rs

@@ -183,6 +183,7 @@ mod tests {
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
     use common_meta::kv_backend::memory::MemoryKvBackend;
     use common_meta::kv_backend::{KvBackend, KvBackendRef};
+    use common_meta::rpc::store::PutRequest;
     use datatypes::prelude::{ConcreteDataType, VectorRef};
     use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, Schema};
     use datatypes::vectors::Int32Vector;
@@ -199,26 +200,20 @@ mod tests {
             catalog_name: DEFAULT_CATALOG_NAME.to_string(),
         }
         .to_string();
-        backend
+        let req = PutRequest::new()
-            .set(
+            .with_key(default_catalog.as_bytes())
-                default_catalog.as_bytes(),
+            .with_value(CatalogValue.as_bytes().unwrap());
-                CatalogValue.as_bytes().unwrap().as_slice(),
+        backend.put(req).await.unwrap();
-            )
-            .await
-            .unwrap();
 
         let default_schema = SchemaKey {
             catalog_name: DEFAULT_CATALOG_NAME.to_string(),
             schema_name: DEFAULT_SCHEMA_NAME.to_string(),
         }
         .to_string();
-        backend
+        let req = PutRequest::new()
-            .set(
+            .with_key(default_schema.as_bytes())
-                default_schema.as_bytes(),
+            .with_value(SchemaValue.as_bytes().unwrap());
-                SchemaValue.as_bytes().unwrap().as_slice(),
+        backend.put(req).await.unwrap();
-            )
-            .await
-            .unwrap();
 
         backend
     }
@@ -257,13 +252,10 @@ mod tests {
             table_info: table_info.into(),
         };
 
-        backend
+        let req = PutRequest::new()
-            .set(
+            .with_key(table_global_key.to_string().as_bytes())
-                table_global_key.to_string().as_bytes(),
+            .with_value(table_global_value.as_bytes().unwrap());
-                table_global_value.as_bytes().unwrap().as_slice(),
+        backend.put(req).await.unwrap();
-            )
-            .await
-            .unwrap();
     }
 
     #[tokio::test]

src/frontend/src/table.rs

@@ -23,6 +23,7 @@ use catalog::helper::{TableGlobalKey, TableGlobalValue};
 use client::Database;
 use common_error::prelude::BoxedError;
 use common_meta::key::TableRouteKey;
+use common_meta::rpc::store::{MoveValueRequest, PutRequest};
 use common_meta::table_name::TableName;
 use common_query::error::Result as QueryResult;
 use common_query::logical_plan::Expr;
@@ -260,7 +261,7 @@ impl DistTable {
             .await
             .context(TableMetadataManagerSnafu)?;
         Ok(if let Some(raw) = raw {
-            Some(TableGlobalValue::from_bytes(raw.1).context(error::CatalogEntrySerdeSnafu)?)
+            Some(TableGlobalValue::from_bytes(raw.value).context(error::CatalogEntrySerdeSnafu)?)
         } else {
             None
         })
@@ -272,19 +273,26 @@ impl DistTable {
         value: TableGlobalValue,
     ) -> Result<()> {
         let value = value.as_bytes().context(error::CatalogEntrySerdeSnafu)?;
-        self.catalog_manager
+        let req = PutRequest::new()
+            .with_key(key.to_string().as_bytes())
+            .with_value(value);
+        let _ = self
+            .catalog_manager
             .backend()
-            .set(key.to_string().as_bytes(), &value)
+            .put(req)
             .await
-            .context(TableMetadataManagerSnafu)
+            .context(TableMetadataManagerSnafu)?;
+        Ok(())
     }
 
     async fn delete_table_global_value(&self, key: TableGlobalKey) -> Result<()> {
-        self.catalog_manager
+        let _ = self
+            .catalog_manager
             .backend()
-            .delete(key.to_string().as_bytes())
+            .delete(key.to_string().as_bytes(), false)
             .await
-            .context(TableMetadataManagerSnafu)
+            .context(TableMetadataManagerSnafu)?;
+        Ok(())
     }
 
     async fn move_table_route_value(
@@ -311,9 +319,10 @@ impl DistTable {
         }
         .to_string();
 
+        let req = MoveValueRequest::new(old_key.as_bytes(), new_key.as_bytes());
         self.catalog_manager
             .backend()
-            .move_value(old_key.as_bytes(), new_key.as_bytes())
+            .move_value(req)
             .await
             .context(TableMetadataManagerSnafu)?;
 

src/meta-client/Cargo.toml

@@ -12,7 +12,7 @@ common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
 common-telemetry = { path = "../common/telemetry" }
 common-meta = { path = "../common/meta" }
-etcd-client = "0.11"
+etcd-client.workspace = true
 rand.workspace = true
 serde.workspace = true
 serde_json.workspace = true

src/meta-client/src/client.rs

@@ -739,7 +739,7 @@ mod tests {
             .with_key(tc.key("key"))
             .with_value(b"value".to_vec());
         let res = tc.client.put(req).await;
-        assert!(res.unwrap().take_prev_kv().is_none());
+        assert!(res.unwrap().prev_kv.is_none());
     }
 
     #[tokio::test]
@@ -752,14 +752,14 @@ mod tests {
             .with_value(b"value".to_vec())
             .with_prev_kv();
         let res = tc.client.put(req).await;
-        assert!(res.unwrap().take_prev_kv().is_none());
+        assert!(res.unwrap().prev_kv.is_none());
 
         let req = PutRequest::new()
             .with_key(key.as_slice())
             .with_value(b"value1".to_vec())
             .with_prev_kv();
         let res = tc.client.put(req).await;
-        let mut kv = res.unwrap().take_prev_kv().unwrap();
+        let mut kv = res.unwrap().prev_kv.unwrap();
         assert_eq!(key, kv.take_key());
         assert_eq!(b"value".to_vec(), kv.take_value());
     }
@@ -794,16 +794,14 @@ mod tests {
         for i in 0..256 {
             req = req.add_key(tc.key(&format!("key-{}", i)));
         }
-        let mut res = tc.client.batch_get(req).await.unwrap();
+        let res = tc.client.batch_get(req).await.unwrap();
-
+        assert_eq!(10, res.kvs.len());
-        assert_eq!(10, res.take_kvs().len());
 
         let req = BatchGetRequest::default()
             .add_key(tc.key("key-1"))
             .add_key(tc.key("key-999"));
-        let mut res = tc.client.batch_get(req).await.unwrap();
+        let res = tc.client.batch_get(req).await.unwrap();
-
+        assert_eq!(1, res.kvs.len());
-        assert_eq!(1, res.take_kvs().len());
     }
 
     #[tokio::test]
@@ -867,7 +865,9 @@ mod tests {
         let res = tc.client.compare_and_put(req).await;
         let mut res = res.unwrap();
         assert!(res.is_success());
-        assert_eq!(b"value".to_vec(), res.take_prev_kv().unwrap().take_value());
+
+        // If compare-and-put is success, previous value doesn't need to be returned.
+        assert!(res.take_prev_kv().is_none());
     }
 
     #[tokio::test]

src/meta-srv/Cargo.toml

@@ -25,7 +25,7 @@ common-telemetry = { path = "../common/telemetry" }
 common-time = { path = "../common/time" }
 dashmap = "5.4"
 derive_builder = "0.12"
-etcd-client = "0.11"
+etcd-client.workspace = true
 futures.workspace = true
 h2 = "0.3"
 http-body = "0.4"

src/meta-srv/examples/kv_store.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::meta::{DeleteRangeRequest, PutRequest, RangeRequest};
+use common_meta::rpc::store::{DeleteRangeRequest, PutRequest, RangeRequest};
 use meta_srv::service::store::etcd::EtcdStore;
 use tracing::{event, subscriber, Level};
 use tracing_subscriber::FmtSubscriber;
@@ -31,7 +31,6 @@ async fn run() {
         key: b"key1".to_vec(),
         value: b"value1".to_vec(),
         prev_kv: true,
-        ..Default::default()
     };
     let res = kv_store.put(put_req).await;
     event!(Level::INFO, "put result: {:#?}", res);

src/meta-srv/src/cluster.rs

@@ -18,18 +18,22 @@ use std::time::Duration;
 
 use api::v1::meta::cluster_client::ClusterClient;
 use api::v1::meta::{
-    BatchGetRequest, BatchGetResponse, KeyValue, RangeRequest, RangeResponse, ResponseHeader,
+    BatchGetRequest as PbBatchGetRequest, BatchGetResponse as PbBatchGetResponse,
+    RangeRequest as PbRangeRequest, RangeResponse as PbRangeResponse, ResponseHeader,
 };
 use common_grpc::channel_manager::ChannelManager;
+use common_meta::rpc::store::{BatchGetRequest, RangeRequest};
+use common_meta::rpc::KeyValue;
+use common_meta::util;
 use common_telemetry::warn;
 use derive_builder::Builder;
 use snafu::{ensure, OptionExt, ResultExt};
 
+use crate::error;
 use crate::error::{match_for_io_error, Result};
 use crate::keys::{StatKey, StatValue, DN_STAT_PREFIX};
 use crate::metasrv::ElectionRef;
 use crate::service::store::kv::ResettableKvStoreRef;
-use crate::{error, util};
 
 pub type MetaPeerClientRef = Arc<MetaPeerClient>;
 
@@ -112,13 +116,13 @@ impl MetaPeerClient {
             .get(&leader_addr)
             .context(error::CreateChannelSnafu)?;
 
-        let request = tonic::Request::new(RangeRequest {
+        let request = tonic::Request::new(PbRangeRequest {
             key,
             range_end,
             ..Default::default()
         });
 
-        let response: RangeResponse = ClusterClient::new(channel)
+        let response: PbRangeResponse = ClusterClient::new(channel)
             .range(request)
             .await
             .context(error::RangeSnafu)?
@@ -126,16 +130,13 @@ impl MetaPeerClient {
 
         check_resp_header(&response.header, Context { addr: &leader_addr })?;
 
-        Ok(response.kvs)
+        Ok(response.kvs.into_iter().map(KeyValue::new).collect())
     }
 
     // Get kv information from the leader's in_mem kv store
     pub async fn batch_get(&self, keys: Vec<Vec<u8>>) -> Result<Vec<KeyValue>> {
         if self.is_leader() {
-            let request = BatchGetRequest {
+            let request = BatchGetRequest { keys };
-                keys,
-                ..Default::default()
-            };
 
             return self.in_memory.batch_get(request).await.map(|resp| resp.kvs);
         }
@@ -175,12 +176,12 @@ impl MetaPeerClient {
             .get(&leader_addr)
             .context(error::CreateChannelSnafu)?;
 
-        let request = tonic::Request::new(BatchGetRequest {
+        let request = tonic::Request::new(PbBatchGetRequest {
             keys,
             ..Default::default()
         });
 
-        let response: BatchGetResponse = ClusterClient::new(channel)
+        let response: PbBatchGetResponse = ClusterClient::new(channel)
             .batch_get(request)
             .await
             .context(error::BatchGetSnafu)?
@@ -188,7 +189,7 @@ impl MetaPeerClient {
 
         check_resp_header(&response.header, Context { addr: &leader_addr })?;
 
-        Ok(response.kvs)
+        Ok(response.kvs.into_iter().map(KeyValue::new).collect())
     }
 
     // Check if the meta node is a leader node.
@@ -240,7 +241,8 @@ fn need_retry(error: &error::Error) -> bool {
 
 #[cfg(test)]
 mod tests {
-    use api::v1::meta::{Error, ErrorCode, KeyValue, ResponseHeader};
+    use api::v1::meta::{Error, ErrorCode, ResponseHeader};
+    use common_meta::rpc::KeyValue;
 
     use super::{check_resp_header, to_stat_kv_map, Context};
     use crate::error;

src/meta-srv/src/error.rs

@@ -409,8 +409,11 @@ pub enum Error {
         source: common_meta::error::Error,
     },
 
-    #[snafu(display("Etcd txn got an error: {err_msg}"))]
+    #[snafu(display("Failed to convert Etcd txn object: {source}"))]
-    EtcdTxnOpResponse { err_msg: String, location: Location },
+    ConvertEtcdTxnObject {
+        source: common_meta::error::Error,
+        location: Location,
+    },
 
     // this error is used for custom error mapping
     // please do not delete it
@@ -490,7 +493,6 @@ impl ErrorExt for Error {
             | Error::InvalidTxnResult { .. }
             | Error::InvalidUtf8Value { .. }
             | Error::UnexpectedInstructionReply { .. }
-            | Error::EtcdTxnOpResponse { .. }
             | Error::Unexpected { .. }
             | Error::Txn { .. }
             | Error::TableIdChanged { .. } => StatusCode::Unexpected,
@@ -507,9 +509,11 @@ impl ErrorExt for Error {
             Error::RegionFailoverCandidatesNotFound { .. } => StatusCode::RuntimeResourcesExhausted,
 
             Error::RegisterProcedureLoader { source, .. } => source.status_code(),
-            Error::TableRouteConversion { source, .. } | Error::ConvertProtoData { source, .. } => {
+
-                source.status_code()
+            Error::TableRouteConversion { source, .. }
-            }
+            | Error::ConvertProtoData { source, .. }
+            | Error::ConvertEtcdTxnObject { source, .. } => source.status_code(),
+
             Error::Other { source, .. } => source.status_code(),
         }
     }

src/meta-srv/src/handler/keep_lease_handler.rs

@@ -12,7 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::meta::{HeartbeatRequest, PutRequest, Role};
+use api::v1::meta::{HeartbeatRequest, Role};
+use common_meta::rpc::store::PutRequest;
 use common_telemetry::{trace, warn};
 use common_time::util as time_util;
 

src/meta-srv/src/handler/persist_stats_handler.rs

@@ -14,7 +14,8 @@
 
 use std::cmp::Ordering;
 
-use api::v1::meta::{HeartbeatRequest, PutRequest, Role};
+use api::v1::meta::{HeartbeatRequest, Role};
+use common_meta::rpc::store::PutRequest;
 use common_telemetry::warn;
 use dashmap::DashMap;
 
@@ -147,7 +148,6 @@ mod tests {
     use crate::keys::StatKey;
     use crate::sequence::Sequence;
     use crate::service::store::cached_kv::LeaderCachedKvStore;
-    use crate::service::store::ext::KvStoreExt;
     use crate::service::store::memory::MemStore;
 
     #[tokio::test]
@@ -184,7 +184,8 @@ mod tests {
             cluster_id: 3,
             node_id: 101,
         };
-        let res = ctx.in_memory.get(key.try_into().unwrap()).await.unwrap();
+        let key: Vec<u8> = key.try_into().unwrap();
+        let res = ctx.in_memory.get(&key).await.unwrap();
         let kv = res.unwrap();
         let key: StatKey = kv.key.clone().try_into().unwrap();
         assert_eq!(3, key.cluster_id);
@@ -195,7 +196,9 @@ mod tests {
         assert_eq!(Some(1), val.stats[0].region_num);
 
         handle_request_many_times(ctx.clone(), &handler, 10).await;
-        let res = ctx.in_memory.get(key.try_into().unwrap()).await.unwrap();
+
+        let key: Vec<u8> = key.try_into().unwrap();
+        let res = ctx.in_memory.get(&key).await.unwrap();
         let kv = res.unwrap();
         let val: StatValue = kv.value.try_into().unwrap();
         // refresh every 10 stats

src/meta-srv/src/lease.rs

@@ -14,12 +14,12 @@
 
 use std::collections::HashMap;
 
+use common_meta::util;
 use common_time::util as time_util;
 
 use crate::cluster::MetaPeerClientRef;
 use crate::error::Result;
 use crate::keys::{LeaseKey, LeaseValue, DN_LEASE_PREFIX};
-use crate::util;
 
 pub async fn alive_datanodes(
     cluster_id: u64,

src/meta-srv/src/lib.rs

@@ -40,6 +40,4 @@ pub mod table_routes;
 #[cfg(test)]
 mod test_util;
 
-pub mod util;
-
 pub use crate::error::Result;

src/meta-srv/src/metadata_service.rs

@@ -14,9 +14,9 @@
 
 use std::sync::Arc;
 
-use api::v1::meta::CompareAndPutRequest;
 use async_trait::async_trait;
 use catalog::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
+use common_meta::rpc::store::CompareAndPutRequest;
 use common_telemetry::{info, timer};
 use metrics::increment_counter;
 use snafu::{ensure, ResultExt};
@@ -80,7 +80,6 @@ impl MetadataService for DefaultMetadataService {
             value: CatalogValue {}
                 .as_bytes()
                 .context(error::InvalidCatalogValueSnafu)?,
-            ..Default::default()
         };
 
         let resp = kv_store.compare_and_put(req).await?;
@@ -96,7 +95,6 @@ impl MetadataService for DefaultMetadataService {
             value: SchemaValue {}
                 .as_bytes()
                 .context(error::InvalidCatalogValueSnafu)?,
-            ..Default::default()
         };
         let resp = kv_store.compare_and_put(req).await?;
 
@@ -124,7 +122,6 @@ mod tests {
     use catalog::helper::{CatalogKey, SchemaKey};
 
     use super::{DefaultMetadataService, MetadataService};
-    use crate::service::store::ext::KvStoreExt;
     use crate::service::store::kv::KvStoreRef;
     use crate::service::store::memory::MemStore;
 
@@ -156,11 +153,9 @@ mod tests {
         .to_string()
         .into();
 
-        let result = kv_store.get(key.clone()).await.unwrap();
+        let result = kv_store.get(&key).await.unwrap();
-
         let kv = result.unwrap();
-
+        assert_eq!(key, kv.key());
-        assert_eq!(key, kv.key);
 
         let key: Vec<u8> = SchemaKey {
             catalog_name: "catalog".to_string(),
@@ -169,10 +164,8 @@ mod tests {
         .to_string()
         .into();
 
-        let result = kv_store.get(key.clone()).await.unwrap();
+        let result = kv_store.get(&key).await.unwrap();
-
         let kv = result.unwrap();
-
+        assert_eq!(key, kv.key());
-        assert_eq!(key, kv.key);
     }
 }

src/meta-srv/src/metrics.rs

@@ -15,7 +15,6 @@
 pub(crate) const METRIC_META_CREATE_CATALOG: &str = "meta.create_catalog";
 pub(crate) const METRIC_META_CREATE_SCHEMA: &str = "meta.create_schema";
 pub(crate) const METRIC_META_KV_REQUEST: &str = "meta.kv_request";
-pub(crate) const METRIC_META_TXN_REQUEST: &str = "meta.txn_request";
 pub(crate) const METRIC_META_ROUTE_REQUEST: &str = "meta.route_request";
 pub(crate) const METRIC_META_HEARTBEAT_CONNECTION_NUM: &str = "meta.heartbeat_connection_num";
 pub(crate) const METRIC_META_HANDLER_EXECUTE: &str = "meta.handler_execute";

src/meta-srv/src/procedure/create_table.rs

@@ -19,6 +19,7 @@ use client::Database;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_meta::key::TableRouteKey;
+use common_meta::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp};
 use common_meta::rpc::ddl::CreateTableTask;
 use common_meta::rpc::router::TableRoute;
 use common_meta::table_name::TableName;
@@ -34,7 +35,6 @@ use super::utils::{handle_request_datanode_error, handle_retry_error};
 use crate::ddl::DdlContext;
 use crate::error::{self, Result};
 use crate::service::router::create_table_global_value;
-use crate::service::store::txn::{Compare, CompareOp, Txn, TxnOp};
 use crate::table_routes::get_table_global_value;
 
 // TODO(weny): removes in following PRs.

src/meta-srv/src/procedure/drop_table.rs

@@ -21,6 +21,7 @@ use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_meta::ident::TableIdent;
 use common_meta::instruction::Instruction;
+use common_meta::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp};
 use common_meta::rpc::ddl::DropTableTask;
 use common_meta::rpc::router::TableRoute;
 use common_meta::table_name::TableName;
@@ -40,7 +41,6 @@ use crate::error;
 use crate::error::Result;
 use crate::procedure::utils::{build_table_route_value, handle_request_datanode_error};
 use crate::service::mailbox::BroadcastChannel;
-use crate::service::store::txn::{Compare, CompareOp, Txn, TxnOp};
 use crate::table_routes::fetch_table;
 pub struct DropTableProcedure {
     context: DdlContext,

src/meta-srv/src/procedure/region_failover.rs

@@ -45,7 +45,6 @@ use crate::error::{Error, RegisterProcedureLoaderSnafu, Result};
 use crate::lock::DistLockRef;
 use crate::metasrv::{SelectorContext, SelectorRef};
 use crate::service::mailbox::MailboxRef;
-use crate::service::store::ext::KvStoreExt;
 
 const OPEN_REGION_MESSAGE_TIMEOUT: Duration = Duration::from_secs(30);
 const CLOSE_REGION_MESSAGE_TIMEOUT: Duration = Duration::from_secs(2);
@@ -208,7 +207,7 @@ impl RegionFailoverManager {
         let table_global_value = self
             .selector_ctx
             .kv_store
-            .get(table_global_key.to_raw_key())
+            .get(&table_global_key.to_raw_key())
             .await?;
         Ok(table_global_value.is_some())
     }

src/meta-srv/src/procedure/state_store.rs

@@ -12,10 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::meta::{BatchDeleteRequest, PutRequest, RangeRequest};
 use async_stream::try_stream;
 use async_trait::async_trait;
 use common_error::prelude::BoxedError;
+use common_meta::rpc::store::{BatchDeleteRequest, PutRequest, RangeRequest};
+use common_meta::util;
 use common_procedure::error::{
     CorruptedDataSnafu, DeleteStatesSnafu, ListStateSnafu, PutStateSnafu,
 };
@@ -24,7 +25,6 @@ use common_procedure::Result;
 use snafu::ResultExt;
 
 use crate::service::store::kv::KvStoreRef;
-use crate::util;
 
 const PROCEDURE_PREFIX: &str = "/__procedure__/";
 

src/meta-srv/src/sequence.rs

@@ -15,7 +15,7 @@
 use std::ops::Range;
 use std::sync::Arc;
 
-use api::v1::meta::CompareAndPutRequest;
+use common_meta::rpc::store::CompareAndPutRequest;
 use snafu::{ensure, OptionExt};
 use tokio::sync::Mutex;
 
@@ -120,7 +120,6 @@ impl Inner {
                 key: key.to_vec(),
                 expect,
                 value: value.to_vec(),
-                ..Default::default()
             };
 
             let res = self.generator.compare_and_put(req).await?;
@@ -156,16 +155,20 @@ impl Inner {
 
 #[cfg(test)]
 mod tests {
+    use std::any::Any;
     use std::sync::Arc;
 
-    use api::v1::meta::{
+    use common_meta::kv_backend::{KvBackend, TxnService};
+    use common_meta::rpc::store::{
         BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse,
+        BatchPutRequest, BatchPutResponse, CompareAndPutResponse, DeleteRangeRequest,
+        DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+        RangeRequest, RangeResponse,
     };
 
     use super::*;
-    use crate::service::store::kv::KvStore;
+    use crate::error::Error;
     use crate::service::store::memory::MemStore;
-    use crate::service::store::txn::TxnService;
 
     #[tokio::test]
     async fn test_sequence() {
@@ -200,28 +203,25 @@ mod tests {
     async fn test_sequence_force_quit() {
         struct Noop;
 
-        impl TxnService for Noop {}
+        impl TxnService for Noop {
+            type Error = Error;
+        }
 
         #[async_trait::async_trait]
-        impl KvStore for Noop {
+        impl KvBackend for Noop {
-            async fn range(
+            fn name(&self) -> &str {
-                &self,
+                "Noop"
-                _: api::v1::meta::RangeRequest,
+            }
-            ) -> Result<api::v1::meta::RangeResponse> {
+
+            async fn range(&self, _: RangeRequest) -> Result<RangeResponse> {
                 unreachable!()
             }
 
-            async fn put(
+            async fn put(&self, _: PutRequest) -> Result<PutResponse> {
-                &self,
-                _: api::v1::meta::PutRequest,
-            ) -> Result<api::v1::meta::PutResponse> {
                 unreachable!()
             }
 
-            async fn batch_put(
+            async fn batch_put(&self, _: BatchPutRequest) -> Result<BatchPutResponse> {
-                &self,
-                _: api::v1::meta::BatchPutRequest,
-            ) -> Result<api::v1::meta::BatchPutResponse> {
                 unreachable!()
             }
 
@@ -232,27 +232,25 @@ mod tests {
             async fn compare_and_put(
                 &self,
                 _: CompareAndPutRequest,
-            ) -> Result<api::v1::meta::CompareAndPutResponse> {
+            ) -> Result<CompareAndPutResponse> {
-                Ok(api::v1::meta::CompareAndPutResponse::default())
+                Ok(CompareAndPutResponse::default())
             }
 
-            async fn delete_range(
+            async fn delete_range(&self, _: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
-                &self,
-                _: api::v1::meta::DeleteRangeRequest,
-            ) -> Result<api::v1::meta::DeleteRangeResponse> {
                 unreachable!()
             }
 
-            async fn move_value(
+            async fn move_value(&self, _: MoveValueRequest) -> Result<MoveValueResponse> {
-                &self,
-                _: api::v1::meta::MoveValueRequest,
-            ) -> Result<api::v1::meta::MoveValueResponse> {
                 unreachable!()
             }
 
             async fn batch_delete(&self, _: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
                 unreachable!()
             }
+
+            fn as_any(&self) -> &dyn Any {
+                self
+            }
         }
 
         let kv_store = Arc::new(Noop {});

src/meta-srv/src/service/admin/meta.rs

@@ -14,18 +14,18 @@
 
 use std::collections::HashMap;
 
-use api::v1::meta::{RangeRequest, RangeResponse};
 use catalog::helper::{
     build_catalog_prefix, build_schema_prefix, build_table_global_prefix, TABLE_GLOBAL_KEY_PREFIX,
 };
+use common_meta::rpc::store::{RangeRequest, RangeResponse};
+use common_meta::util;
 use snafu::{OptionExt, ResultExt};
 use tonic::codegen::http;
 
+use crate::error;
 use crate::error::Result;
 use crate::service::admin::HttpHandler;
-use crate::service::store::ext::KvStoreExt;
 use crate::service::store::kv::KvStoreRef;
-use crate::{error, util};
 
 pub struct CatalogsHandler {
     pub kv_store: KvStoreRef,
@@ -104,7 +104,7 @@ impl HttpHandler for TableHandler {
             })?;
         let table_key = format!("{TABLE_GLOBAL_KEY_PREFIX}-{table_name}");
 
-        let response = self.kv_store.get(table_key.into_bytes()).await?;
+        let response = self.kv_store.get(table_key.as_bytes()).await?;
         let mut value: String = "Not found result".to_string();
         if let Some(key_value) = response {
             value = String::from_utf8(key_value.value).context(error::InvalidUtf8ValueSnafu)?;
@@ -161,11 +161,11 @@ async fn get_keys_by_prefix(key_prefix: String, kv_store: &KvStoreRef) -> Result
 mod tests {
     use std::sync::Arc;
 
-    use api::v1::meta::PutRequest;
     use catalog::helper::{
         build_catalog_prefix, build_schema_prefix, build_table_global_prefix, CatalogKey,
         SchemaKey, TableGlobalKey,
     };
+    use common_meta::rpc::store::PutRequest;
 
     use crate::service::admin::meta::get_keys_by_prefix;
     use crate::service::store::kv::KvStoreRef;
@@ -184,7 +184,7 @@ mod tests {
             .put(PutRequest {
                 key: catalog.to_string().as_bytes().to_vec(),
                 value: "".as_bytes().to_vec(),
-                ..Default::default()
+                prev_kv: false,
             })
             .await
             .is_ok());
@@ -197,7 +197,7 @@ mod tests {
             .put(PutRequest {
                 key: schema.to_string().as_bytes().to_vec(),
                 value: "".as_bytes().to_vec(),
-                ..Default::default()
+                prev_kv: false,
             })
             .await
             .is_ok());
@@ -216,7 +216,7 @@ mod tests {
             .put(PutRequest {
                 key: table1.to_string().as_bytes().to_vec(),
                 value: "".as_bytes().to_vec(),
-                ..Default::default()
+                prev_kv: false,
             })
             .await
             .is_ok());
@@ -224,7 +224,7 @@ mod tests {
             .put(PutRequest {
                 key: table2.to_string().as_bytes().to_vec(),
                 value: "".as_bytes().to_vec(),
-                ..Default::default()
+                prev_kv: false,
             })
             .await
             .is_ok());

src/meta-srv/src/service/admin/route.rs

@@ -14,16 +14,18 @@
 
 use std::collections::HashMap;
 
-use api::v1::meta::{RangeRequest, RangeResponse, TableRouteValue};
+use api::v1::meta::TableRouteValue;
 use common_meta::key::TABLE_ROUTE_PREFIX;
+use common_meta::rpc::store::{RangeRequest, RangeResponse};
+use common_meta::util;
 use prost::Message;
 use snafu::{OptionExt, ResultExt};
 use tonic::codegen::http;
 
 use super::HttpHandler;
+use crate::error;
 use crate::error::Result;
 use crate::service::store::kv::KvStoreRef;
-use crate::{error, util};
 
 pub struct RouteHandler {
     pub kv_store: KvStoreRef,

src/meta-srv/src/service/cluster.rs

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use api::v1::meta::{
-    cluster_server, BatchGetRequest, BatchGetResponse, Error, RangeRequest, RangeResponse,
+    cluster_server, BatchGetRequest as PbBatchGetRequest, BatchGetResponse as PbBatchGetResponse,
-    ResponseHeader,
+    Error, RangeRequest as PbRangeRequest, RangeResponse as PbRangeResponse, ResponseHeader,
 };
 use common_telemetry::warn;
 use tonic::{Request, Response};
@@ -24,10 +24,10 @@ use crate::service::GrpcResult;
 
 #[async_trait::async_trait]
 impl cluster_server::Cluster for MetaSrv {
-    async fn batch_get(&self, req: Request<BatchGetRequest>) -> GrpcResult<BatchGetResponse> {
+    async fn batch_get(&self, req: Request<PbBatchGetRequest>) -> GrpcResult<PbBatchGetResponse> {
         if !self.is_leader() {
             let is_not_leader = ResponseHeader::failed(0, Error::is_not_leader());
-            let resp = BatchGetResponse {
+            let resp = PbBatchGetResponse {
                 header: Some(is_not_leader),
                 ..Default::default()
             };
@@ -36,22 +36,17 @@ impl cluster_server::Cluster for MetaSrv {
             return Ok(Response::new(resp));
         }
 
-        let kvs = self.in_memory().batch_get(req.into_inner()).await?.kvs;
+        let req = req.into_inner().into();
+        let resp = self.in_memory().batch_get(req).await?;
 
-        let success = ResponseHeader::success(0);
+        let resp = resp.to_proto_resp(ResponseHeader::success(0));
-
+        Ok(Response::new(resp))
-        let get_resp = BatchGetResponse {
-            kvs,
-            header: Some(success),
-        };
-
-        Ok(Response::new(get_resp))
     }
 
-    async fn range(&self, req: Request<RangeRequest>) -> GrpcResult<RangeResponse> {
+    async fn range(&self, req: Request<PbRangeRequest>) -> GrpcResult<PbRangeResponse> {
         if !self.is_leader() {
             let is_not_leader = ResponseHeader::failed(0, Error::is_not_leader());
-            let resp = RangeResponse {
+            let resp = PbRangeResponse {
                 header: Some(is_not_leader),
                 ..Default::default()
             };
@@ -60,10 +55,11 @@ impl cluster_server::Cluster for MetaSrv {
             return Ok(Response::new(resp));
         }
 
-        let req = req.into_inner();
+        let req = req.into_inner().into();
         let res = self.in_memory().range(req).await?;
 
-        Ok(Response::new(res))
+        let resp = res.to_proto_resp(ResponseHeader::success(0));
+        Ok(Response::new(resp))
     }
 }
 

src/meta-srv/src/service/router.rs

@@ -15,12 +15,13 @@
 use std::collections::HashMap;
 
 use api::v1::meta::{
-    router_server, BatchPutRequest, CreateRequest, DeleteRequest, Error, KeyValue, Peer, PeerDict,
+    router_server, CreateRequest, DeleteRequest, Error, Peer, PeerDict, Region, RegionRoute,
-    Region, RegionRoute, ResponseHeader, RouteRequest, RouteResponse, Table, TableRoute,
+    ResponseHeader, RouteRequest, RouteResponse, Table, TableRoute, TableRouteValue,
-    TableRouteValue,
 };
 use catalog::helper::{TableGlobalKey, TableGlobalValue};
 use common_meta::key::TableRouteKey;
+use common_meta::rpc::store::BatchPutRequest;
+use common_meta::rpc::KeyValue;
 use common_meta::table_name::TableName;
 use common_telemetry::{timer, warn};
 use snafu::{ensure, OptionExt, ResultExt};
@@ -33,7 +34,6 @@ use crate::lock::{keys, DistLockGuard};
 use crate::metasrv::{Context, MetaSrv, SelectorContext, SelectorRef};
 use crate::metrics::METRIC_META_ROUTE_REQUEST;
 use crate::sequence::SequenceRef;
-use crate::service::store::ext::KvStoreExt;
 use crate::service::GrpcResult;
 use crate::table_routes::{
     fetch_tables, get_table_global_value, remove_table_global_value, remove_table_route_value,
@@ -66,7 +66,7 @@ impl router_server::Router for MetaSrv {
         .to_string()
         .into_bytes();
         ensure!(
-            self.kv_store().get(table_global_key).await?.is_none(),
+            self.kv_store().get(&table_global_key).await?.is_none(),
             error::TableAlreadyExistsSnafu {
                 table_name: table_name.to_string(),
             }
@@ -238,7 +238,6 @@ async fn handle_create(
             },
         ],
         prev_kv: true,
-        ..Default::default()
     };
 
     let resp = ctx.kv_store.batch_put(req).await?;

src/meta-srv/src/service/store.rs

@@ -15,85 +15,252 @@
 pub mod cached_kv;
 pub mod etcd;
 pub(crate) mod etcd_util;
-pub mod ext;
 pub mod kv;
 pub mod memory;
-pub mod txn;
 
 use api::v1::meta::{
-    store_server, BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse,
+    store_server, BatchDeleteRequest as PbBatchDeleteRequest,
-    BatchPutRequest, BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse,
+    BatchDeleteResponse as PbBatchDeleteResponse, BatchGetRequest as PbBatchGetRequest,
-    DeleteRangeRequest, DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest,
+    BatchGetResponse as PbBatchGetResponse, BatchPutRequest as PbBatchPutRequest,
-    PutResponse, RangeRequest, RangeResponse,
+    BatchPutResponse as PbBatchPutResponse, CompareAndPutRequest as PbCompareAndPutRequest,
+    CompareAndPutResponse as PbCompareAndPutResponse, DeleteRangeRequest as PbDeleteRangeRequest,
+    DeleteRangeResponse as PbDeleteRangeResponse, MoveValueRequest as PbMoveValueRequest,
+    MoveValueResponse as PbMoveValueResponse, PutRequest as PbPutRequest,
+    PutResponse as PbPutResponse, RangeRequest as PbRangeRequest, RangeResponse as PbRangeResponse,
+    ResponseHeader,
 };
+use common_meta::rpc::store::{
+    BatchDeleteRequest, BatchGetRequest, BatchPutRequest, CompareAndPutRequest, DeleteRangeRequest,
+    MoveValueRequest, PutRequest, RangeRequest,
+};
+use common_telemetry::timer;
+use snafu::OptionExt;
 use tonic::{Request, Response};
 
+use crate::error::MissingRequestHeaderSnafu;
 use crate::metasrv::MetaSrv;
+use crate::metrics::METRIC_META_KV_REQUEST;
 use crate::service::GrpcResult;
 
 #[async_trait::async_trait]
 impl store_server::Store for MetaSrv {
-    async fn range(&self, req: Request<RangeRequest>) -> GrpcResult<RangeResponse> {
+    async fn range(&self, req: Request<PbRangeRequest>) -> GrpcResult<PbRangeResponse> {
         let req = req.into_inner();
+
+        let cluster_id = req
+            .header
+            .as_ref()
+            .context(MissingRequestHeaderSnafu)?
+            .cluster_id;
+
+        let _timer = timer!(
+            METRIC_META_KV_REQUEST,
+            &[
+                ("target", self.kv_store().name().to_string()),
+                ("op", "range".to_string()),
+                ("cluster_id", cluster_id.to_string()),
+            ]
+        );
+
+        let req: RangeRequest = req.into();
+
         let res = self.kv_store().range(req).await?;
 
+        let res = res.to_proto_resp(ResponseHeader::success(cluster_id));
         Ok(Response::new(res))
     }
 
-    async fn put(&self, req: Request<PutRequest>) -> GrpcResult<PutResponse> {
+    async fn put(&self, req: Request<PbPutRequest>) -> GrpcResult<PbPutResponse> {
         let req = req.into_inner();
+
+        let cluster_id = req
+            .header
+            .as_ref()
+            .context(MissingRequestHeaderSnafu)?
+            .cluster_id;
+
+        let _timer = timer!(
+            METRIC_META_KV_REQUEST,
+            &[
+                ("target", self.kv_store().name().to_string()),
+                ("op", "put".to_string()),
+                ("cluster_id", cluster_id.to_string()),
+            ]
+        );
+
+        let req: PutRequest = req.into();
+
         let res = self.kv_store().put(req).await?;
 
+        let res = res.to_proto_resp(ResponseHeader::success(cluster_id));
         Ok(Response::new(res))
     }
 
-    async fn batch_get(&self, req: Request<BatchGetRequest>) -> GrpcResult<BatchGetResponse> {
+    async fn batch_get(&self, req: Request<PbBatchGetRequest>) -> GrpcResult<PbBatchGetResponse> {
         let req = req.into_inner();
+
+        let cluster_id = req
+            .header
+            .as_ref()
+            .context(MissingRequestHeaderSnafu)?
+            .cluster_id;
+
+        let _timer = timer!(
+            METRIC_META_KV_REQUEST,
+            &[
+                ("target", self.kv_store().name().to_string()),
+                ("op", "batch_get".to_string()),
+                ("cluster_id", cluster_id.to_string()),
+            ]
+        );
+
+        let req: BatchGetRequest = req.into();
+
         let res = self.kv_store().batch_get(req).await?;
 
+        let res = res.to_proto_resp(ResponseHeader::success(cluster_id));
         Ok(Response::new(res))
     }
 
-    async fn batch_put(&self, req: Request<BatchPutRequest>) -> GrpcResult<BatchPutResponse> {
+    async fn batch_put(&self, req: Request<PbBatchPutRequest>) -> GrpcResult<PbBatchPutResponse> {
         let req = req.into_inner();
+
+        let cluster_id = req
+            .header
+            .as_ref()
+            .context(MissingRequestHeaderSnafu)?
+            .cluster_id;
+
+        let _timer = timer!(
+            METRIC_META_KV_REQUEST,
+            &[
+                ("target", self.kv_store().name().to_string()),
+                ("op", "batch_pub".to_string()),
+                ("cluster_id", cluster_id.to_string()),
+            ]
+        );
+
+        let req: BatchPutRequest = req.into();
+
         let res = self.kv_store().batch_put(req).await?;
 
+        let res = res.to_proto_resp(ResponseHeader::success(cluster_id));
         Ok(Response::new(res))
     }
 
     async fn batch_delete(
         &self,
-        req: Request<BatchDeleteRequest>,
+        req: Request<PbBatchDeleteRequest>,
-    ) -> GrpcResult<BatchDeleteResponse> {
+    ) -> GrpcResult<PbBatchDeleteResponse> {
         let req = req.into_inner();
+
+        let cluster_id = req
+            .header
+            .as_ref()
+            .context(MissingRequestHeaderSnafu)?
+            .cluster_id;
+
+        let _timer = timer!(
+            METRIC_META_KV_REQUEST,
+            &[
+                ("target", self.kv_store().name().to_string()),
+                ("op", "batch_delete".to_string()),
+                ("cluster_id", cluster_id.to_string()),
+            ]
+        );
+
+        let req: BatchDeleteRequest = req.into();
+
         let res = self.kv_store().batch_delete(req).await?;
+
+        let res = res.to_proto_resp(ResponseHeader::success(cluster_id));
         Ok(Response::new(res))
     }
 
     async fn compare_and_put(
         &self,
-        req: Request<CompareAndPutRequest>,
+        req: Request<PbCompareAndPutRequest>,
-    ) -> GrpcResult<CompareAndPutResponse> {
+    ) -> GrpcResult<PbCompareAndPutResponse> {
         let req = req.into_inner();
+
+        let cluster_id = req
+            .header
+            .as_ref()
+            .context(MissingRequestHeaderSnafu)?
+            .cluster_id;
+
+        let _timer = timer!(
+            METRIC_META_KV_REQUEST,
+            &[
+                ("target", self.kv_store().name().to_string()),
+                ("op", "compare_and_put".to_string()),
+                ("cluster_id", cluster_id.to_string()),
+            ]
+        );
+
+        let req: CompareAndPutRequest = req.into();
+
         let res = self.kv_store().compare_and_put(req).await?;
 
+        let res = res.to_proto_resp(ResponseHeader::success(cluster_id));
         Ok(Response::new(res))
     }
 
     async fn delete_range(
         &self,
-        req: Request<DeleteRangeRequest>,
+        req: Request<PbDeleteRangeRequest>,
-    ) -> GrpcResult<DeleteRangeResponse> {
+    ) -> GrpcResult<PbDeleteRangeResponse> {
         let req = req.into_inner();
+
+        let cluster_id = req
+            .header
+            .as_ref()
+            .context(MissingRequestHeaderSnafu)?
+            .cluster_id;
+
+        let _timer = timer!(
+            METRIC_META_KV_REQUEST,
+            &[
+                ("target", self.kv_store().name().to_string()),
+                ("op", "delete_range".to_string()),
+                ("cluster_id", cluster_id.to_string()),
+            ]
+        );
+
+        let req: DeleteRangeRequest = req.into();
+
         let res = self.kv_store().delete_range(req).await?;
 
+        let res = res.to_proto_resp(ResponseHeader::success(cluster_id));
         Ok(Response::new(res))
     }
 
-    async fn move_value(&self, req: Request<MoveValueRequest>) -> GrpcResult<MoveValueResponse> {
+    async fn move_value(
+        &self,
+        req: Request<PbMoveValueRequest>,
+    ) -> GrpcResult<PbMoveValueResponse> {
         let req = req.into_inner();
+
+        let cluster_id = req
+            .header
+            .as_ref()
+            .context(MissingRequestHeaderSnafu)?
+            .cluster_id;
+
+        let _timer = timer!(
+            METRIC_META_KV_REQUEST,
+            &[
+                ("target", self.kv_store().name().to_string()),
+                ("op", "move_value".to_string()),
+                ("cluster_id", cluster_id.to_string()),
+            ]
+        );
+
+        let req: MoveValueRequest = req.into();
+
         let res = self.kv_store().move_value(req).await?;
 
+        let res = res.to_proto_resp(ResponseHeader::success(cluster_id));
         Ok(Response::new(res))
     }
 }
@@ -122,7 +289,8 @@ mod tests {
     async fn test_range() {
         let meta_srv = new_meta_srv().await;
 
-        let req = RangeRequest::default();
+        let mut req = RangeRequest::default();
+        req.set_header((1, 1), Role::Datanode);
         let res = meta_srv.range(req.into_request()).await;
 
         let _ = res.unwrap();
@@ -132,7 +300,8 @@ mod tests {
     async fn test_put() {
         let meta_srv = new_meta_srv().await;
 
-        let req = PutRequest::default();
+        let mut req = PutRequest::default();
+        req.set_header((1, 1), Role::Datanode);
         let res = meta_srv.put(req.into_request()).await;
 
         let _ = res.unwrap();
@@ -142,7 +311,8 @@ mod tests {
     async fn test_batch_get() {
         let meta_srv = new_meta_srv().await;
 
-        let req = BatchGetRequest::default();
+        let mut req = BatchGetRequest::default();
+        req.set_header((1, 1), Role::Datanode);
         let res = meta_srv.batch_get(req.into_request()).await;
 
         let _ = res.unwrap();
@@ -152,7 +322,8 @@ mod tests {
     async fn test_batch_put() {
         let meta_srv = new_meta_srv().await;
 
-        let req = BatchPutRequest::default();
+        let mut req = BatchPutRequest::default();
+        req.set_header((1, 1), Role::Datanode);
         let res = meta_srv.batch_put(req.into_request()).await;
 
         let _ = res.unwrap();
@@ -162,7 +333,8 @@ mod tests {
     async fn test_batch_delete() {
         let meta_srv = new_meta_srv().await;
 
-        let req = BatchDeleteRequest::default();
+        let mut req = BatchDeleteRequest::default();
+        req.set_header((1, 1), Role::Datanode);
         let res = meta_srv.batch_delete(req.into_request()).await;
 
         let _ = res.unwrap();
@@ -172,7 +344,8 @@ mod tests {
     async fn test_compare_and_put() {
         let meta_srv = new_meta_srv().await;
 
-        let req = CompareAndPutRequest::default();
+        let mut req = CompareAndPutRequest::default();
+        req.set_header((1, 1), Role::Datanode);
         let res = meta_srv.compare_and_put(req.into_request()).await;
 
         let _ = res.unwrap();
@@ -182,7 +355,8 @@ mod tests {
     async fn test_delete_range() {
         let meta_srv = new_meta_srv().await;
 
-        let req = DeleteRangeRequest::default();
+        let mut req = DeleteRangeRequest::default();
+        req.set_header((1, 1), Role::Datanode);
         let res = meta_srv.delete_range(req.into_request()).await;
 
         let _ = res.unwrap();
@@ -192,7 +366,8 @@ mod tests {
     async fn test_move_value() {
         let meta_srv = new_meta_srv().await;
 
-        let req = MoveValueRequest::default();
+        let mut req = MoveValueRequest::default();
+        req.set_header((1, 1), Role::Datanode);
         let res = meta_srv.move_value(req.into_request()).await;
 
         let _ = res.unwrap();

src/meta-srv/src/service/store/cached_kv.rs

@@ -12,22 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::any::Any;
 use std::collections::HashSet;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
-use api::v1::meta::{
+use common_meta::kv_backend::txn::{Txn, TxnOp, TxnRequest, TxnResponse};
+use common_meta::kv_backend::{KvBackend, TxnService};
+use common_meta::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
     BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
-    DeleteRangeResponse, KeyValue, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
     RangeRequest, RangeResponse,
 };
+use common_meta::rpc::KeyValue;
 
-use crate::error::Result;
+use crate::error::{Error, Result};
-use crate::service::store::ext::KvStoreExt;
+use crate::service::store::kv::{KvStoreRef, ResettableKvStore, ResettableKvStoreRef};
-use crate::service::store::kv::{KvStore, KvStoreRef, ResettableKvStore, ResettableKvStoreRef};
 use crate::service::store::memory::MemStore;
-use crate::service::store::txn::{Txn, TxnOp, TxnRequest, TxnResponse, TxnService};
 
 pub type CheckLeaderRef = Arc<dyn CheckLeader>;
 
@@ -57,15 +59,18 @@ pub struct LeaderCachedKvStore {
     store: KvStoreRef,
     cache: ResettableKvStoreRef,
     version: AtomicUsize,
+    name: String,
 }
 
 impl LeaderCachedKvStore {
     pub fn new(check_leader: CheckLeaderRef, store: KvStoreRef) -> Self {
+        let name = format!("LeaderCached({})", store.name());
         Self {
             check_leader,
             store,
             cache: Arc::new(MemStore::new()),
             version: AtomicUsize::new(0),
+            name,
         }
     }
 
@@ -82,7 +87,7 @@ impl LeaderCachedKvStore {
 
     #[inline]
     async fn invalid_key(&self, key: Vec<u8>) -> Result<()> {
-        let _ = self.cache.delete(key, false).await?;
+        let _ = self.cache.delete(&key, false).await?;
         Ok(())
     }
 
@@ -110,7 +115,11 @@ impl LeaderCachedKvStore {
 }
 
 #[async_trait::async_trait]
-impl KvStore for LeaderCachedKvStore {
+impl KvBackend for LeaderCachedKvStore {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
     async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
         if !self.is_leader() {
             return self.store.range(req).await;
@@ -186,16 +195,13 @@ impl KvStore for LeaderCachedKvStore {
             .filter(|key| !hit_keys.contains(*key))
             .cloned()
             .collect::<Vec<_>>();
-        let remote_req = BatchGetRequest {
+        let remote_req = BatchGetRequest { keys: missed_keys };
-            keys: missed_keys,
-            ..Default::default()
-        };
 
         let ver = self.get_version();
 
         let remote_res = self.store.batch_get(remote_req).await?;
         let put_req = BatchPutRequest {
-            kvs: remote_res.kvs.clone(),
+            kvs: remote_res.kvs.clone().into_iter().map(Into::into).collect(),
             ..Default::default()
         };
         let _ = self.cache.batch_put(put_req).await?;
@@ -204,7 +210,7 @@ impl KvStore for LeaderCachedKvStore {
             let keys = remote_res
                 .kvs
                 .iter()
-                .map(|kv| kv.key.clone())
+                .map(|kv| kv.key().to_vec())
                 .collect::<Vec<_>>();
             self.invalid_keys(keys).await?;
         }
@@ -291,10 +297,16 @@ impl KvStore for LeaderCachedKvStore {
         self.invalid_keys(vec![from_key, to_key]).await?;
         Ok(res)
     }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
 
 #[async_trait::async_trait]
 impl TxnService for LeaderCachedKvStore {
+    type Error = Error;
+
     async fn txn(&self, txn: Txn) -> Result<TxnResponse> {
         if !self.is_leader() {
             return self.store.txn(txn).await;
@@ -338,7 +350,7 @@ impl ResettableKvStore for LeaderCachedKvStore {
 
 #[cfg(test)]
 mod tests {
-    use api::v1::meta::KeyValue;
+    use common_meta::rpc::KeyValue;
 
     use super::*;
     use crate::service::store::memory::MemStore;
@@ -364,23 +376,19 @@ mod tests {
         };
         let _ = inner_store.put(put_req).await.unwrap();
 
-        let cached_value = inner_cache.get(key.clone()).await.unwrap();
+        let cached_value = inner_cache.get(&key).await.unwrap();
         assert!(cached_value.is_none());
 
-        let cached_value = cached_store.get(key.clone()).await.unwrap().unwrap();
+        let cached_value = cached_store.get(&key).await.unwrap().unwrap();
-        assert_eq!(cached_value.value, value);
+        assert_eq!(cached_value.value(), value);
 
-        let cached_value = inner_cache.get(key.clone()).await.unwrap().unwrap();
+        let cached_value = inner_cache.get(&key).await.unwrap().unwrap();
-        assert_eq!(cached_value.value, value);
+        assert_eq!(cached_value.value(), value);
 
-        let res = cached_store
+        let res = cached_store.delete(&key, true).await.unwrap().unwrap();
-            .delete(key.clone(), true)
+        assert_eq!(res.value(), value);
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(res.value, value);
 
-        let cached_value = inner_cache.get(key.clone()).await.unwrap();
+        let cached_value = inner_cache.get(&key).await.unwrap();
         assert!(cached_value.is_none());
     }
 
@@ -409,10 +417,7 @@ mod tests {
             .map(|i| format!("test_key_{}", i).into_bytes())
             .collect::<Vec<_>>();
 
-        let batch_get_req = BatchGetRequest {
+        let batch_get_req = BatchGetRequest { keys };
-            keys,
-            ..Default::default()
-        };
 
         let cached_values = inner_cache.batch_get(batch_get_req.clone()).await.unwrap();
         assert!(cached_values.kvs.is_empty());
@@ -451,10 +456,7 @@ mod tests {
         let keys = (1..5)
             .map(|i| format!("test_key_{}", i).into_bytes())
             .collect::<Vec<_>>();
-        let batch_get_req = BatchGetRequest {
+        let batch_get_req = BatchGetRequest { keys };
-            keys,
-            ..Default::default()
-        };
         let cached_values = inner_cache.batch_get(batch_get_req.clone()).await.unwrap();
         assert_eq!(cached_values.kvs.len(), 4);
 

src/meta-srv/src/service/store/etcd.rs

@@ -12,15 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::any::Any;
 use std::sync::Arc;
 
-use api::v1::meta::{
+use common_error::prelude::*;
+use common_meta::kv_backend::txn::{Txn as KvTxn, TxnResponse as KvTxnResponse};
+use common_meta::kv_backend::{KvBackend, TxnService};
+use common_meta::metrics::METRIC_META_TXN_REQUEST;
+use common_meta::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
     BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
-    DeleteRangeResponse, KeyValue, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
-    RangeRequest, RangeResponse, ResponseHeader,
+    RangeRequest, RangeResponse,
 };
-use common_error::prelude::*;
+use common_meta::rpc::KeyValue;
 use common_telemetry::{timer, warn};
 use etcd_client::{
     Client, Compare, CompareOp, DeleteOptions, GetOptions, PutOptions, Txn, TxnOp, TxnOpResponse,
@@ -28,11 +33,9 @@ use etcd_client::{
 };
 
 use crate::error;
-use crate::error::Result;
+use crate::error::{ConvertEtcdTxnObjectSnafu, Error, Result};
-use crate::metrics::{METRIC_META_KV_REQUEST, METRIC_META_TXN_REQUEST};
 use crate::service::store::etcd_util::KvPair;
-use crate::service::store::kv::{KvStore, KvStoreRef};
+use crate::service::store::kv::KvStoreRef;
-use crate::service::store::txn::TxnService;
 
 // Maximum number of operations permitted in a transaction.
 // The etcd default configuration's `--max-txn-ops` is 128.
@@ -89,22 +92,13 @@ impl EtcdStore {
 }
 
 #[async_trait::async_trait]
-impl KvStore for EtcdStore {
+impl KvBackend for EtcdStore {
-    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
+    fn name(&self) -> &str {
-        let Get {
+        "Etcd"
-            cluster_id,
+    }
-            key,
-            options,
-        } = req.try_into()?;
 
-        let _timer = timer!(
+    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
-            METRIC_META_KV_REQUEST,
+        let Get { key, options } = req.try_into()?;
-            &[
-                ("target", "etcd".to_string()),
-                ("op", "range".to_string()),
-                ("cluster_id", cluster_id.to_string())
-            ]
-        );
 
         let res = self
             .client
@@ -119,9 +113,7 @@ impl KvStore for EtcdStore {
             .map(KvPair::from_etcd_kv)
             .collect::<Vec<_>>();
 
-        let header = Some(ResponseHeader::success(cluster_id));
         Ok(RangeResponse {
-            header,
             kvs,
             more: res.more(),
         })
@@ -129,21 +121,11 @@ impl KvStore for EtcdStore {
 
     async fn put(&self, req: PutRequest) -> Result<PutResponse> {
         let Put {
-            cluster_id,
             key,
             value,
             options,
         } = req.try_into()?;
 
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[
-                ("target", "etcd".to_string()),
-                ("op", "put".to_string()),
-                ("cluster_id", cluster_id.to_string())
-            ]
-        );
-
         let res = self
             .client
             .kv_client()
@@ -152,26 +134,11 @@ impl KvStore for EtcdStore {
             .context(error::EtcdFailedSnafu)?;
 
         let prev_kv = res.prev_key().map(KvPair::from_etcd_kv);
-
+        Ok(PutResponse { prev_kv })
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(PutResponse { header, prev_kv })
     }
 
     async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
-        let BatchGet {
+        let BatchGet { keys, options } = req.try_into()?;
-            cluster_id,
-            keys,
-            options,
-        } = req.try_into()?;
-
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[
-                ("target", "etcd".to_string()),
-                ("op", "batch_get".to_string()),
-                ("cluster_id", cluster_id.to_string())
-            ]
-        );
 
         let get_ops: Vec<_> = keys
             .into_iter()
@@ -192,25 +159,11 @@ impl KvStore for EtcdStore {
             }
         }
 
-        let header = Some(ResponseHeader::success(cluster_id));
+        Ok(BatchGetResponse { kvs })
-        Ok(BatchGetResponse { header, kvs })
     }
 
     async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse> {
-        let BatchPut {
+        let BatchPut { kvs, options } = req.try_into()?;
-            cluster_id,
-            kvs,
-            options,
-        } = req.try_into()?;
-
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[
-                ("target", "etcd".to_string()),
-                ("op", "batch_put".to_string()),
-                ("cluster_id", cluster_id.to_string())
-            ]
-        );
 
         let put_ops = kvs
             .into_iter()
@@ -233,25 +186,11 @@ impl KvStore for EtcdStore {
             }
         }
 
-        let header = Some(ResponseHeader::success(cluster_id));
+        Ok(BatchPutResponse { prev_kvs })
-        Ok(BatchPutResponse { header, prev_kvs })
     }
 
     async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
-        let BatchDelete {
+        let BatchDelete { keys, options } = req.try_into()?;
-            cluster_id,
-            keys,
-            options,
-        } = req.try_into()?;
-
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[
-                ("target", "etcd".to_string()),
-                ("op", "batch_delete".to_string()),
-                ("cluster_id", cluster_id.to_string())
-            ]
-        );
 
         let mut prev_kvs = Vec::with_capacity(keys.len());
 
@@ -275,28 +214,17 @@ impl KvStore for EtcdStore {
             }
         }
 
-        let header = Some(ResponseHeader::success(cluster_id));
+        Ok(BatchDeleteResponse { prev_kvs })
-        Ok(BatchDeleteResponse { header, prev_kvs })
     }
 
     async fn compare_and_put(&self, req: CompareAndPutRequest) -> Result<CompareAndPutResponse> {
         let CompareAndPut {
-            cluster_id,
             key,
             expect,
             value,
             put_options,
         } = req.try_into()?;
 
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[
-                ("target", "etcd".to_string()),
-                ("op", "compare_and_put".to_string()),
-                ("cluster_id", cluster_id.to_string())
-            ]
-        );
-
         let compare = if expect.is_empty() {
             // create if absent
             // revision 0 means key was not exist
@@ -333,29 +261,11 @@ impl KvStore for EtcdStore {
             _ => unreachable!(),
         };
 
-        let header = Some(ResponseHeader::success(cluster_id));
+        Ok(CompareAndPutResponse { success, prev_kv })
-        Ok(CompareAndPutResponse {
-            header,
-            success,
-            prev_kv,
-        })
     }
 
     async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
-        let Delete {
+        let Delete { key, options } = req.try_into()?;
-            cluster_id,
-            key,
-            options,
-        } = req.try_into()?;
-
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[
-                ("target", "etcd".to_string()),
-                ("op", "delete_range".to_string()),
-                ("cluster_id", cluster_id.to_string())
-            ]
-        );
 
         let res = self
             .client
@@ -370,9 +280,7 @@ impl KvStore for EtcdStore {
             .map(KvPair::from_etcd_kv)
             .collect::<Vec<_>>();
 
-        let header = Some(ResponseHeader::success(cluster_id));
         Ok(DeleteRangeResponse {
-            header,
             deleted: res.deleted(),
             prev_kvs,
         })
@@ -380,24 +288,13 @@ impl KvStore for EtcdStore {
 
     async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
         let MoveValue {
-            cluster_id,
             from_key,
             to_key,
             delete_options,
         } = req.try_into()?;
 
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[
-                ("target", "etcd".to_string()),
-                ("op", "move_value".to_string()),
-                ("cluster_id", cluster_id.to_string())
-            ]
-        );
-
         let mut client = self.client.kv_client();
 
-        let header = Some(ResponseHeader::success(cluster_id));
         // TODO(jiachun): Maybe it's better to let the users control it in the request
         const MAX_RETRIES: usize = 8;
         for _ in 0..MAX_RETRIES {
@@ -442,16 +339,14 @@ impl KvStore for EtcdStore {
             for op_res in txn_res.op_responses() {
                 match op_res {
                     TxnOpResponse::Get(res) => {
-                        return Ok(MoveValueResponse {
+                        return Ok(MoveValueResponse(
-                            header,
+                            res.kvs().first().map(KvPair::from_etcd_kv),
-                            kv: res.kvs().first().map(KvPair::from_etcd_kv),
+                        ));
-                        });
                     }
                     TxnOpResponse::Delete(res) => {
-                        return Ok(MoveValueResponse {
+                        return Ok(MoveValueResponse(
-                            header,
+                            res.prev_kvs().first().map(KvPair::from_etcd_kv),
-                            kv: res.prev_kvs().first().map(KvPair::from_etcd_kv),
+                        ));
-                        });
                     }
                     _ => {}
                 }
@@ -463,14 +358,17 @@ impl KvStore for EtcdStore {
         }
         .fail()
     }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
 
 #[async_trait::async_trait]
 impl TxnService for EtcdStore {
-    async fn txn(
+    type Error = Error;
-        &self,
+
-        txn: crate::service::store::txn::Txn,
+    async fn txn(&self, txn: KvTxn) -> Result<KvTxnResponse> {
-    ) -> Result<crate::service::store::txn::TxnResponse> {
         let _timer = timer!(
             METRIC_META_TXN_REQUEST,
             &[("target", "etcd".to_string()), ("op", "txn".to_string()),]
@@ -483,12 +381,11 @@ impl TxnService for EtcdStore {
             .txn(etcd_txn)
             .await
             .context(error::EtcdFailedSnafu)?;
-        txn_res.try_into()
+        txn_res.try_into().context(ConvertEtcdTxnObjectSnafu)
     }
 }
 
 struct Get {
-    cluster_id: u64,
     key: Vec<u8>,
     options: Option<GetOptions>,
 }
@@ -498,7 +395,6 @@ impl TryFrom<RangeRequest> for Get {
 
     fn try_from(req: RangeRequest) -> Result<Self> {
         let RangeRequest {
-            header,
             key,
             range_end,
             limit,
@@ -519,7 +415,6 @@ impl TryFrom<RangeRequest> for Get {
         }
 
         Ok(Get {
-            cluster_id: header.map_or(0, |h| h.cluster_id),
             key,
             options: Some(options),
         })
@@ -527,7 +422,6 @@ impl TryFrom<RangeRequest> for Get {
 }
 
 struct Put {
-    cluster_id: u64,
     key: Vec<u8>,
     value: Vec<u8>,
     options: Option<PutOptions>,
@@ -538,7 +432,6 @@ impl TryFrom<PutRequest> for Put {
 
     fn try_from(req: PutRequest) -> Result<Self> {
         let PutRequest {
-            header,
             key,
             value,
             prev_kv,
@@ -550,7 +443,6 @@ impl TryFrom<PutRequest> for Put {
         }
 
         Ok(Put {
-            cluster_id: header.map_or(0, |h| h.cluster_id),
             key,
             value,
             options: Some(options),
@@ -559,7 +451,6 @@ impl TryFrom<PutRequest> for Put {
 }
 
 struct BatchGet {
-    cluster_id: u64,
     keys: Vec<Vec<u8>>,
     options: Option<GetOptions>,
 }
@@ -568,12 +459,11 @@ impl TryFrom<BatchGetRequest> for BatchGet {
     type Error = error::Error;
 
     fn try_from(req: BatchGetRequest) -> Result<Self> {
-        let BatchGetRequest { header, keys } = req;
+        let BatchGetRequest { keys } = req;
 
         let options = GetOptions::default();
 
         Ok(BatchGet {
-            cluster_id: header.map_or(0, |h| h.cluster_id),
             keys,
             options: Some(options),
         })
@@ -581,7 +471,6 @@ impl TryFrom<BatchGetRequest> for BatchGet {
 }
 
 struct BatchPut {
-    cluster_id: u64,
     kvs: Vec<KeyValue>,
     options: Option<PutOptions>,
 }
@@ -590,11 +479,7 @@ impl TryFrom<BatchPutRequest> for BatchPut {
     type Error = error::Error;
 
     fn try_from(req: BatchPutRequest) -> Result<Self> {
-        let BatchPutRequest {
+        let BatchPutRequest { kvs, prev_kv } = req;
-            header,
-            kvs,
-            prev_kv,
-        } = req;
 
         let mut options = PutOptions::default();
         if prev_kv {
@@ -602,7 +487,6 @@ impl TryFrom<BatchPutRequest> for BatchPut {
         }
 
         Ok(BatchPut {
-            cluster_id: header.map_or(0, |h| h.cluster_id),
             kvs,
             options: Some(options),
         })
@@ -610,7 +494,6 @@ impl TryFrom<BatchPutRequest> for BatchPut {
 }
 
 struct BatchDelete {
-    cluster_id: u64,
     keys: Vec<Vec<u8>>,
     options: Option<DeleteOptions>,
 }
@@ -619,11 +502,7 @@ impl TryFrom<BatchDeleteRequest> for BatchDelete {
     type Error = error::Error;
 
     fn try_from(req: BatchDeleteRequest) -> Result<Self> {
-        let BatchDeleteRequest {
+        let BatchDeleteRequest { keys, prev_kv } = req;
-            header,
-            keys,
-            prev_kv,
-        } = req;
 
         let mut options = DeleteOptions::default();
         if prev_kv {
@@ -631,7 +510,6 @@ impl TryFrom<BatchDeleteRequest> for BatchDelete {
         }
 
         Ok(BatchDelete {
-            cluster_id: header.map_or(0, |h| h.cluster_id),
             keys,
             options: Some(options),
         })
@@ -639,7 +517,6 @@ impl TryFrom<BatchDeleteRequest> for BatchDelete {
 }
 
 struct CompareAndPut {
-    cluster_id: u64,
     key: Vec<u8>,
     expect: Vec<u8>,
     value: Vec<u8>,
@@ -650,15 +527,9 @@ impl TryFrom<CompareAndPutRequest> for CompareAndPut {
     type Error = error::Error;
 
     fn try_from(req: CompareAndPutRequest) -> Result<Self> {
-        let CompareAndPutRequest {
+        let CompareAndPutRequest { key, expect, value } = req;
-            header,
-            key,
-            expect,
-            value,
-        } = req;
 
         Ok(CompareAndPut {
-            cluster_id: header.map_or(0, |h| h.cluster_id),
             key,
             expect,
             value,
@@ -668,7 +539,6 @@ impl TryFrom<CompareAndPutRequest> for CompareAndPut {
 }
 
 struct Delete {
-    cluster_id: u64,
     key: Vec<u8>,
     options: Option<DeleteOptions>,
 }
@@ -678,7 +548,6 @@ impl TryFrom<DeleteRangeRequest> for Delete {
 
     fn try_from(req: DeleteRangeRequest) -> Result<Self> {
         let DeleteRangeRequest {
-            header,
             key,
             range_end,
             prev_kv,
@@ -695,7 +564,6 @@ impl TryFrom<DeleteRangeRequest> for Delete {
         }
 
         Ok(Delete {
-            cluster_id: header.map_or(0, |h| h.cluster_id),
             key,
             options: Some(options),
         })
@@ -703,7 +571,6 @@ impl TryFrom<DeleteRangeRequest> for Delete {
 }
 
 struct MoveValue {
-    cluster_id: u64,
     from_key: Vec<u8>,
     to_key: Vec<u8>,
     delete_options: Option<DeleteOptions>,
@@ -713,14 +580,9 @@ impl TryFrom<MoveValueRequest> for MoveValue {
     type Error = error::Error;
 
     fn try_from(req: MoveValueRequest) -> Result<Self> {
-        let MoveValueRequest {
+        let MoveValueRequest { from_key, to_key } = req;
-            header,
-            from_key,
-            to_key,
-        } = req;
 
         Ok(MoveValue {
-            cluster_id: header.map_or(0, |h| h.cluster_id),
             from_key,
             to_key,
             delete_options: Some(DeleteOptions::default().with_prev_key()),
@@ -739,7 +601,6 @@ mod tests {
             range_end: b"test_range_end".to_vec(),
             limit: 64,
             keys_only: true,
-            ..Default::default()
         };
 
         let get: Get = req.try_into().unwrap();
@@ -754,7 +615,6 @@ mod tests {
             key: b"test_key".to_vec(),
             value: b"test_value".to_vec(),
             prev_kv: true,
-            ..Default::default()
         };
 
         let put: Put = req.try_into().unwrap();
@@ -768,7 +628,6 @@ mod tests {
     fn test_parse_batch_get() {
         let req = BatchGetRequest {
             keys: vec![b"k1".to_vec(), b"k2".to_vec(), b"k3".to_vec()],
-            ..Default::default()
         };
 
         let batch_get: BatchGet = req.try_into().unwrap();
@@ -787,13 +646,13 @@ mod tests {
                 value: b"test_value".to_vec(),
             }],
             prev_kv: true,
-            ..Default::default()
         };
 
         let batch_put: BatchPut = req.try_into().unwrap();
 
-        assert_eq!(b"test_key".to_vec(), batch_put.kvs.get(0).unwrap().key);
+        let kv = batch_put.kvs.get(0).unwrap();
-        assert_eq!(b"test_value".to_vec(), batch_put.kvs.get(0).unwrap().value);
+        assert_eq!(b"test_key", kv.key());
+        assert_eq!(b"test_value", kv.value());
         let _ = batch_put.options.unwrap();
     }
 
@@ -802,7 +661,6 @@ mod tests {
         let req = BatchDeleteRequest {
             keys: vec![b"k1".to_vec(), b"k2".to_vec(), b"k3".to_vec()],
             prev_kv: true,
-            ..Default::default()
         };
 
         let batch_delete: BatchDelete = req.try_into().unwrap();
@@ -820,7 +678,6 @@ mod tests {
             key: b"test_key".to_vec(),
             expect: b"test_expect".to_vec(),
             value: b"test_value".to_vec(),
-            ..Default::default()
         };
 
         let compare_and_put: CompareAndPut = req.try_into().unwrap();
@@ -837,7 +694,6 @@ mod tests {
             key: b"test_key".to_vec(),
             range_end: b"test_range_end".to_vec(),
             prev_kv: true,
-            ..Default::default()
         };
 
         let delete: Delete = req.try_into().unwrap();
@@ -851,7 +707,6 @@ mod tests {
         let req = MoveValueRequest {
             from_key: b"test_from_key".to_vec(),
             to_key: b"test_to_key".to_vec(),
-            ..Default::default()
         };
 
         let move_value: MoveValue = req.try_into().unwrap();

src/meta-srv/src/service/store/etcd_util.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::meta::KeyValue;
+use common_meta::rpc::KeyValue;
 
 pub struct KvPair<'a>(&'a etcd_client::KeyValue);
 

src/meta-srv/src/service/store/ext.rs

@@ -1,177 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use api::v1::meta::{DeleteRangeRequest, KeyValue, RangeRequest};
-
-use crate::error::Result;
-use crate::service::store::kv::KvStore;
-
-#[async_trait::async_trait]
-pub trait KvStoreExt {
-    /// Get the value by the given key.
-    async fn get(&self, key: Vec<u8>) -> Result<Option<KeyValue>>;
-
-    /// Check if a key exists, it does not return the value.
-    async fn exists(&self, key: Vec<u8>) -> Result<bool>;
-
-    /// Delete the value by the given key. If prev_kv is true,
-    /// the previous key-value pairs will be returned.
-    async fn delete(&self, key: Vec<u8>, prev_kv: bool) -> Result<Option<KeyValue>>;
-}
-
-#[async_trait::async_trait]
-impl<T> KvStoreExt for T
-where
-    T: KvStore + ?Sized,
-{
-    async fn get(&self, key: Vec<u8>) -> Result<Option<KeyValue>> {
-        let req = RangeRequest {
-            key,
-            ..Default::default()
-        };
-
-        let mut kvs = self.range(req).await?.kvs;
-
-        Ok(kvs.pop())
-    }
-
-    async fn exists(&self, key: Vec<u8>) -> Result<bool> {
-        let req = RangeRequest {
-            key,
-            keys_only: true,
-            ..Default::default()
-        };
-
-        let kvs = self.range(req).await?.kvs;
-
-        Ok(!kvs.is_empty())
-    }
-
-    async fn delete(&self, key: Vec<u8>, prev_kv: bool) -> Result<Option<KeyValue>> {
-        let req = DeleteRangeRequest {
-            key,
-            prev_kv,
-            ..Default::default()
-        };
-
-        let mut prev_kvs = self.delete_range(req).await?.prev_kvs;
-
-        Ok(prev_kvs.pop())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use api::v1::meta::PutRequest;
-
-    use crate::service::store::ext::KvStoreExt;
-    use crate::service::store::kv::KvStoreRef;
-    use crate::service::store::memory::MemStore;
-
-    #[tokio::test]
-    async fn test_get() {
-        let mut in_mem = Arc::new(MemStore::new()) as KvStoreRef;
-
-        put_stats_to_store(&mut in_mem).await;
-
-        let kv = in_mem
-            .get("test_key1".as_bytes().to_vec())
-            .await
-            .unwrap()
-            .unwrap();
-
-        assert_eq!("test_key1".as_bytes(), kv.key);
-        assert_eq!("test_val1".as_bytes(), kv.value);
-
-        let kv = in_mem
-            .get("test_key2".as_bytes().to_vec())
-            .await
-            .unwrap()
-            .unwrap();
-
-        assert_eq!("test_key2".as_bytes(), kv.key);
-        assert_eq!("test_val2".as_bytes(), kv.value);
-
-        let may_kv = in_mem.get("test_key3".as_bytes().to_vec()).await.unwrap();
-
-        assert!(may_kv.is_none());
-    }
-
-    #[tokio::test]
-    async fn test_exists() {
-        let mut in_mem = Arc::new(MemStore::new()) as KvStoreRef;
-
-        put_stats_to_store(&mut in_mem).await;
-
-        assert!(in_mem
-            .exists("test_key1".as_bytes().to_vec())
-            .await
-            .unwrap());
-        assert!(in_mem
-            .exists("test_key2".as_bytes().to_vec())
-            .await
-            .unwrap());
-        assert!(!in_mem
-            .exists("test_key3".as_bytes().to_vec())
-            .await
-            .unwrap());
-        assert!(!in_mem.exists("test_key".as_bytes().to_vec()).await.unwrap());
-    }
-
-    #[tokio::test]
-    async fn test_delete() {
-        let mut in_mem = Arc::new(MemStore::new()) as KvStoreRef;
-
-        let mut prev_kv = in_mem
-            .delete("test_key1".as_bytes().to_vec(), true)
-            .await
-            .unwrap();
-        assert!(prev_kv.is_none());
-
-        put_stats_to_store(&mut in_mem).await;
-
-        assert!(in_mem
-            .exists("test_key1".as_bytes().to_vec())
-            .await
-            .unwrap());
-
-        prev_kv = in_mem
-            .delete("test_key1".as_bytes().to_vec(), true)
-            .await
-            .unwrap();
-        assert_eq!("test_key1".as_bytes(), prev_kv.unwrap().key);
-    }
-
-    async fn put_stats_to_store(store: &mut KvStoreRef) {
-        assert!(store
-            .put(PutRequest {
-                key: "test_key1".as_bytes().to_vec(),
-                value: "test_val1".as_bytes().to_vec(),
-                ..Default::default()
-            })
-            .await
-            .is_ok());
-
-        assert!(store
-            .put(PutRequest {
-                key: "test_key2".as_bytes().to_vec(),
-                value: "test_val2".as_bytes().to_vec(),
-                ..Default::default()
-            })
-            .await
-            .is_ok());
-    }
-}

src/meta-srv/src/service/store/kv.rs

@@ -14,38 +14,13 @@
 
 use std::sync::Arc;
 
-use api::v1::meta::{
+use common_meta::kv_backend::KvBackend;
-    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
-    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
-    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
-    RangeRequest, RangeResponse,
-};
 
-use crate::error::Result;
+use crate::error::Error;
-use crate::service::store::txn::TxnService;
 
-pub type KvStoreRef = Arc<dyn KvStore>;
+pub type KvStoreRef = Arc<dyn KvBackend<Error = Error>>;
 pub type ResettableKvStoreRef = Arc<dyn ResettableKvStore>;
 
-#[async_trait::async_trait]
+pub trait ResettableKvStore: KvBackend<Error = Error> {
-pub trait KvStore: TxnService {
-    async fn range(&self, req: RangeRequest) -> Result<RangeResponse>;
-
-    async fn put(&self, req: PutRequest) -> Result<PutResponse>;
-
-    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse>;
-
-    async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse>;
-
-    async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse>;
-
-    async fn compare_and_put(&self, req: CompareAndPutRequest) -> Result<CompareAndPutResponse>;
-
-    async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse>;
-
-    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse>;
-}
-
-pub trait ResettableKvStore: KvStore {
     fn reset(&self);
 }

src/meta-srv/src/service/store/memory.rs

@@ -12,697 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::btree_map::Entry;
+use common_meta::kv_backend::memory::MemoryKvBackend;
-use std::collections::BTreeMap;
-use std::ops::Range;
 
-use api::v1::meta::{
+use crate::error::Error;
-    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
+use crate::service::store::kv::ResettableKvStore;
-    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
-    DeleteRangeResponse, KeyValue, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
-    RangeRequest, RangeResponse, ResponseHeader,
-};
-use common_telemetry::timer;
-use parking_lot::RwLock;
 
-use super::ext::KvStoreExt;
+pub type MemStore = MemoryKvBackend<Error>;
-use crate::error::Result;
-use crate::metrics::{METRIC_META_KV_REQUEST, METRIC_META_TXN_REQUEST};
-use crate::service::store::kv::{KvStore, ResettableKvStore};
-use crate::service::store::txn::{Txn, TxnOp, TxnOpResponse, TxnRequest, TxnResponse, TxnService};
-
-pub struct MemStore {
-    inner: RwLock<BTreeMap<Vec<u8>, Vec<u8>>>,
-}
-
-impl Default for MemStore {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl MemStore {
-    pub fn new() -> Self {
-        Self {
-            inner: RwLock::new(Default::default()),
-        }
-    }
-}
 
 impl ResettableKvStore for MemStore {
     fn reset(&self) {
-        self.inner.write().clear();
+        self.clear();
-    }
-}
-
-#[async_trait::async_trait]
-impl KvStore for MemStore {
-    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[("target", "memory"), ("op", "range"),]
-        );
-
-        let RangeRequest {
-            header,
-            key,
-            range_end,
-            limit,
-            keys_only,
-        } = req;
-
-        let memory = self.inner.read();
-
-        let mut kvs = if range_end.is_empty() {
-            memory.get_key_value(&key).map_or(vec![], |(k, v)| {
-                vec![KeyValue {
-                    key: k.clone(),
-                    value: if keys_only { vec![] } else { v.clone() },
-                }]
-            })
-        } else {
-            let range = Range {
-                start: key,
-                end: range_end,
-            };
-            memory
-                .range(range)
-                .map(|kv| KeyValue {
-                    key: kv.0.clone(),
-                    value: if keys_only { vec![] } else { kv.1.clone() },
-                })
-                .collect::<Vec<_>>()
-        };
-
-        let more = if limit > 0 {
-            kvs.truncate(limit as usize);
-            true
-        } else {
-            false
-        };
-
-        let cluster_id = header.map_or(0, |h| h.cluster_id);
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(RangeResponse { header, kvs, more })
-    }
-
-    async fn put(&self, req: PutRequest) -> Result<PutResponse> {
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[("target", "memory"), ("op", "put"),]
-        );
-
-        let PutRequest {
-            header,
-            key,
-            value,
-            prev_kv,
-        } = req;
-
-        let mut memory = self.inner.write();
-
-        let prev_value = memory.insert(key.clone(), value);
-        let prev_kv = if prev_kv {
-            prev_value.map(|value| KeyValue { key, value })
-        } else {
-            None
-        };
-
-        let cluster_id = header.map_or(0, |h| h.cluster_id);
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(PutResponse { header, prev_kv })
-    }
-
-    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[("target", "memory"), ("op", "batch_get"),]
-        );
-
-        let BatchGetRequest { header, keys } = req;
-
-        let mut kvs = Vec::with_capacity(keys.len());
-        for key in keys {
-            if let Some(kv) = self.get(key).await? {
-                kvs.push(kv);
-            }
-        }
-
-        let cluster_id = header.map_or(0, |h| h.cluster_id);
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(BatchGetResponse { header, kvs })
-    }
-
-    async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse> {
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[("target", "memory"), ("op", "batch_put"),]
-        );
-
-        let BatchPutRequest {
-            header,
-            kvs,
-            prev_kv,
-        } = req;
-
-        let mut memory = self.inner.write();
-
-        let prev_kvs = if prev_kv {
-            kvs.into_iter()
-                .map(|kv| (kv.key.clone(), memory.insert(kv.key, kv.value)))
-                .filter(|(_, v)| v.is_some())
-                .map(|(key, value)| KeyValue {
-                    key,
-                    value: value.unwrap(),
-                })
-                .collect()
-        } else {
-            for kv in kvs.into_iter() {
-                let _ = memory.insert(kv.key, kv.value);
-            }
-            vec![]
-        };
-
-        let cluster_id = header.map_or(0, |h| h.cluster_id);
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(BatchPutResponse { header, prev_kvs })
-    }
-
-    async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[("target", "memory"), ("op", "batch_delete"),]
-        );
-
-        let BatchDeleteRequest {
-            header,
-            keys,
-            prev_kv,
-        } = req;
-
-        let mut memory = self.inner.write();
-
-        let prev_kvs = if prev_kv {
-            keys.into_iter()
-                .filter_map(|key| memory.remove(&key).map(|value| KeyValue { key, value }))
-                .collect()
-        } else {
-            for key in keys.into_iter() {
-                let _ = memory.remove(&key);
-            }
-            vec![]
-        };
-        let cluster_id = header.map_or(0, |h| h.cluster_id);
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(BatchDeleteResponse { header, prev_kvs })
-    }
-
-    async fn compare_and_put(&self, req: CompareAndPutRequest) -> Result<CompareAndPutResponse> {
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[("target", "memory"), ("op", "compare_and_put"),]
-        );
-
-        let CompareAndPutRequest {
-            header,
-            key,
-            expect,
-            value,
-        } = req;
-
-        let mut memory = self.inner.write();
-
-        let (success, prev_kv) = match memory.entry(key) {
-            Entry::Vacant(e) => {
-                let success = expect.is_empty();
-                if success {
-                    let _ = e.insert(value);
-                }
-                (success, None)
-            }
-            Entry::Occupied(mut e) => {
-                let key = e.key().clone();
-                let prev_val = e.get().clone();
-                let success = prev_val == expect;
-                if success {
-                    let _ = e.insert(value);
-                }
-                (success, Some((key, prev_val)))
-            }
-        };
-
-        let prev_kv = prev_kv.map(|(key, value)| KeyValue { key, value });
-
-        let cluster_id = header.map_or(0, |h| h.cluster_id);
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(CompareAndPutResponse {
-            header,
-            success,
-            prev_kv,
-        })
-    }
-
-    async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[("target", "memory"), ("op", "deleteL_range"),]
-        );
-
-        let DeleteRangeRequest {
-            header,
-            key,
-            range_end,
-            prev_kv,
-        } = req;
-
-        let mut memory = self.inner.write();
-
-        let prev_kvs = if range_end.is_empty() {
-            let prev_val = memory.remove(&key);
-            prev_val.map_or(vec![], |value| vec![KeyValue { key, value }])
-        } else {
-            let range = Range {
-                start: key,
-                end: range_end,
-            };
-            memory
-                .drain_filter(|key, _| range.contains(key))
-                .map(|(key, value)| KeyValue { key, value })
-                .collect::<Vec<_>>()
-        };
-
-        let cluster_id = header.map_or(0, |h| h.cluster_id);
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(DeleteRangeResponse {
-            header,
-            deleted: prev_kvs.len() as i64,
-            prev_kvs: if prev_kv {
-                prev_kvs
-            } else {
-                Default::default()
-            },
-        })
-    }
-
-    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
-        let _timer = timer!(
-            METRIC_META_KV_REQUEST,
-            &[("target", "memory"), ("op", "move_value"),]
-        );
-
-        let MoveValueRequest {
-            header,
-            from_key,
-            to_key,
-        } = req;
-
-        let mut memory = self.inner.write();
-
-        let kv = match memory.remove(&from_key) {
-            Some(v) => {
-                let _ = memory.insert(to_key, v.clone());
-                Some((from_key, v))
-            }
-            None => memory.get(&to_key).map(|v| (to_key, v.clone())),
-        };
-
-        let kv = kv.map(|(key, value)| KeyValue { key, value });
-
-        let cluster_id = header.map_or(0, |h| h.cluster_id);
-        let header = Some(ResponseHeader::success(cluster_id));
-        Ok(MoveValueResponse { header, kv })
-    }
-}
-
-#[async_trait::async_trait]
-impl TxnService for MemStore {
-    async fn txn(&self, txn: Txn) -> Result<TxnResponse> {
-        let _timer = timer!(
-            METRIC_META_TXN_REQUEST,
-            &[("target", "memory".to_string()), ("op", "txn".to_string()),]
-        );
-
-        let TxnRequest {
-            compare,
-            success,
-            failure,
-        } = txn.into();
-
-        let mut memory = self.inner.write();
-
-        let succeeded = compare
-            .iter()
-            .all(|x| x.compare_with_value(memory.get(&x.key)));
-
-        let do_txn = |txn_op| match txn_op {
-            TxnOp::Put(key, value) => {
-                let prev_value = memory.insert(key.clone(), value);
-                let prev_kv = prev_value.map(|value| KeyValue { key, value });
-                let put_res = PutResponse {
-                    prev_kv,
-                    ..Default::default()
-                };
-                TxnOpResponse::ResponsePut(put_res)
-            }
-            TxnOp::Get(key) => {
-                let value = memory.get(&key);
-                let kv = value.map(|value| KeyValue {
-                    key,
-                    value: value.clone(),
-                });
-                let get_res = RangeResponse {
-                    kvs: kv.map(|kv| vec![kv]).unwrap_or(vec![]),
-                    ..Default::default()
-                };
-                TxnOpResponse::ResponseGet(get_res)
-            }
-            TxnOp::Delete(key) => {
-                let prev_value = memory.remove(&key);
-                let prev_kv = prev_value.map(|value| KeyValue { key, value });
-                let delete_res = DeleteRangeResponse {
-                    prev_kvs: prev_kv.map(|kv| vec![kv]).unwrap_or(vec![]),
-                    ..Default::default()
-                };
-                TxnOpResponse::ResponseDelete(delete_res)
-            }
-        };
-
-        let responses: Vec<_> = if succeeded {
-            success.into_iter().map(do_txn).collect()
-        } else {
-            failure.into_iter().map(do_txn).collect()
-        };
-
-        Ok(TxnResponse {
-            succeeded,
-            responses,
-        })
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::atomic::{AtomicU8, Ordering};
-    use std::sync::Arc;
-
-    use api::v1::meta::{
-        BatchGetRequest, BatchPutRequest, CompareAndPutRequest, DeleteRangeRequest, KeyValue,
-        MoveValueRequest, PutRequest, RangeRequest,
-    };
-
-    use super::MemStore;
-    use crate::service::store::ext::KvStoreExt;
-    use crate::service::store::kv::KvStore;
-    use crate::util;
-
-    async fn mock_mem_store_with_data() -> MemStore {
-        let kv_store = MemStore::new();
-        let kvs = mock_kvs();
-
-        assert!(kv_store
-            .batch_put(BatchPutRequest {
-                kvs,
-                ..Default::default()
-            })
-            .await
-            .is_ok());
-
-        assert!(kv_store
-            .put(PutRequest {
-                key: b"key11".to_vec(),
-                value: b"val11".to_vec(),
-                ..Default::default()
-            })
-            .await
-            .is_ok());
-
-        kv_store
-    }
-
-    fn mock_kvs() -> Vec<KeyValue> {
-        vec![
-            KeyValue {
-                key: b"key1".to_vec(),
-                value: b"val1".to_vec(),
-            },
-            KeyValue {
-                key: b"key2".to_vec(),
-                value: b"val2".to_vec(),
-            },
-            KeyValue {
-                key: b"key3".to_vec(),
-                value: b"val3".to_vec(),
-            },
-        ]
-    }
-
-    #[tokio::test]
-    async fn test_put() {
-        let kv_store = mock_mem_store_with_data().await;
-
-        let resp = kv_store
-            .put(PutRequest {
-                key: b"key11".to_vec(),
-                value: b"val12".to_vec(),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-        assert!(resp.prev_kv.is_none());
-
-        let resp = kv_store
-            .put(PutRequest {
-                key: b"key11".to_vec(),
-                value: b"val13".to_vec(),
-                prev_kv: true,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-        assert_eq!(b"key11".as_slice(), resp.prev_kv.as_ref().unwrap().key);
-        assert_eq!(b"val12".as_slice(), resp.prev_kv.as_ref().unwrap().value);
-    }
-
-    #[tokio::test]
-    async fn test_range() {
-        let kv_store = mock_mem_store_with_data().await;
-
-        let key = b"key1".to_vec();
-        let range_end = util::get_prefix_end_key(b"key1");
-
-        let resp = kv_store
-            .range(RangeRequest {
-                key: key.clone(),
-                range_end: range_end.clone(),
-                limit: 0,
-                keys_only: false,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(2, resp.kvs.len());
-        assert_eq!(b"key1".as_slice(), resp.kvs[0].key);
-        assert_eq!(b"val1".as_slice(), resp.kvs[0].value);
-        assert_eq!(b"key11".as_slice(), resp.kvs[1].key);
-        assert_eq!(b"val11".as_slice(), resp.kvs[1].value);
-
-        let resp = kv_store
-            .range(RangeRequest {
-                key: key.clone(),
-                range_end: range_end.clone(),
-                limit: 0,
-                keys_only: true,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(2, resp.kvs.len());
-        assert_eq!(b"key1".as_slice(), resp.kvs[0].key);
-        assert_eq!(b"".as_slice(), resp.kvs[0].value);
-        assert_eq!(b"key11".as_slice(), resp.kvs[1].key);
-        assert_eq!(b"".as_slice(), resp.kvs[1].value);
-
-        let resp = kv_store
-            .range(RangeRequest {
-                key: key.clone(),
-                limit: 0,
-                keys_only: false,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(1, resp.kvs.len());
-        assert_eq!(b"key1".as_slice(), resp.kvs[0].key);
-        assert_eq!(b"val1".as_slice(), resp.kvs[0].value);
-
-        let resp = kv_store
-            .range(RangeRequest {
-                key,
-                range_end,
-                limit: 1,
-                keys_only: false,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(1, resp.kvs.len());
-        assert_eq!(b"key1".as_slice(), resp.kvs[0].key);
-        assert_eq!(b"val1".as_slice(), resp.kvs[0].value);
-    }
-
-    #[tokio::test]
-    async fn test_batch_get() {
-        let kv_store = mock_mem_store_with_data().await;
-
-        let keys = vec![];
-        let batch_resp = kv_store
-            .batch_get(BatchGetRequest {
-                keys,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert!(batch_resp.kvs.is_empty());
-
-        let keys = vec![b"key10".to_vec()];
-        let batch_resp = kv_store
-            .batch_get(BatchGetRequest {
-                keys,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert!(batch_resp.kvs.is_empty());
-
-        let keys = vec![b"key1".to_vec(), b"key3".to_vec(), b"key4".to_vec()];
-        let batch_resp = kv_store
-            .batch_get(BatchGetRequest {
-                keys,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(2, batch_resp.kvs.len());
-        assert_eq!(b"key1".as_slice(), batch_resp.kvs[0].key);
-        assert_eq!(b"val1".as_slice(), batch_resp.kvs[0].value);
-        assert_eq!(b"key3".as_slice(), batch_resp.kvs[1].key);
-        assert_eq!(b"val3".as_slice(), batch_resp.kvs[1].value);
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_compare_and_put() {
-        let kv_store = Arc::new(MemStore::new());
-        let success = Arc::new(AtomicU8::new(0));
-
-        let mut joins = vec![];
-        for _ in 0..20 {
-            let kv_store_clone = kv_store.clone();
-            let success_clone = success.clone();
-            let join = tokio::spawn(async move {
-                let req = CompareAndPutRequest {
-                    key: b"key".to_vec(),
-                    expect: vec![],
-                    value: b"val_new".to_vec(),
-                    ..Default::default()
-                };
-                let resp = kv_store_clone.compare_and_put(req).await.unwrap();
-                if resp.success {
-                    let _ = success_clone.fetch_add(1, Ordering::SeqCst);
-                }
-            });
-            joins.push(join);
-        }
-
-        for join in joins {
-            join.await.unwrap();
-        }
-
-        assert_eq!(1, success.load(Ordering::SeqCst));
-    }
-
-    #[tokio::test]
-    async fn test_delete_range() {
-        let kv_store = mock_mem_store_with_data().await;
-
-        let req = DeleteRangeRequest {
-            key: b"key3".to_vec(),
-            range_end: vec![],
-            prev_kv: true,
-            ..Default::default()
-        };
-
-        let resp = kv_store.delete_range(req).await.unwrap();
-        assert_eq!(1, resp.prev_kvs.len());
-        assert_eq!(b"key3".as_slice(), resp.prev_kvs[0].key);
-        assert_eq!(b"val3".as_slice(), resp.prev_kvs[0].value);
-
-        let get_resp = kv_store.get(b"key3".to_vec()).await.unwrap();
-        assert!(get_resp.is_none());
-
-        let req = DeleteRangeRequest {
-            key: b"key2".to_vec(),
-            range_end: vec![],
-            prev_kv: false,
-            ..Default::default()
-        };
-
-        let resp = kv_store.delete_range(req).await.unwrap();
-        assert!(resp.prev_kvs.is_empty());
-
-        let get_resp = kv_store.get(b"key2".to_vec()).await.unwrap();
-        assert!(get_resp.is_none());
-
-        let key = b"key1".to_vec();
-        let range_end = util::get_prefix_end_key(b"key1");
-
-        let req = DeleteRangeRequest {
-            key: key.clone(),
-            range_end: range_end.clone(),
-            prev_kv: true,
-            ..Default::default()
-        };
-        let resp = kv_store.delete_range(req).await.unwrap();
-        assert_eq!(2, resp.prev_kvs.len());
-
-        let req = RangeRequest {
-            key,
-            range_end,
-            ..Default::default()
-        };
-        let resp = kv_store.range(req).await.unwrap();
-        assert!(resp.kvs.is_empty());
-    }
-
-    #[tokio::test]
-    async fn test_move_value() {
-        let kv_store = mock_mem_store_with_data().await;
-
-        let req = MoveValueRequest {
-            from_key: b"key1".to_vec(),
-            to_key: b"key111".to_vec(),
-            ..Default::default()
-        };
-
-        let resp = kv_store.move_value(req).await.unwrap();
-        assert_eq!(b"key1".as_slice(), resp.kv.as_ref().unwrap().key);
-        assert_eq!(b"val1".as_slice(), resp.kv.as_ref().unwrap().value);
-
-        let kv_store = mock_mem_store_with_data().await;
-
-        let req = MoveValueRequest {
-            from_key: b"notexistkey".to_vec(),
-            to_key: b"key222".to_vec(),
-            ..Default::default()
-        };
-
-        let resp = kv_store.move_value(req).await.unwrap();
-        assert!(resp.kv.is_none());
     }
 }

src/meta-srv/src/table_routes.rs

@@ -14,26 +14,25 @@
 
 use std::collections::HashMap;
 
-use api::v1::meta::{MoveValueRequest, PutRequest, TableRouteValue};
+use api::v1::meta::TableRouteValue;
 use catalog::helper::{TableGlobalKey, TableGlobalValue};
 use common_meta::key::TableRouteKey;
-use common_meta::rpc::store::{BatchGetRequest, BatchGetResponse};
+use common_meta::rpc::store::{BatchGetRequest, MoveValueRequest, PutRequest};
 use common_telemetry::warn;
 use snafu::{OptionExt, ResultExt};
 use table::engine::TableReference;
 
 use crate::error::{
-    ConvertProtoDataSnafu, DecodeTableRouteSnafu, InvalidCatalogValueSnafu, Result,
+    DecodeTableRouteSnafu, InvalidCatalogValueSnafu, Result, TableNotFoundSnafu,
-    TableNotFoundSnafu, TableRouteNotFoundSnafu,
+    TableRouteNotFoundSnafu,
 };
-use crate::service::store::ext::KvStoreExt;
 use crate::service::store::kv::KvStoreRef;
 
 pub async fn get_table_global_value(
     kv_store: &KvStoreRef,
     key: &TableGlobalKey,
 ) -> Result<Option<TableGlobalValue>> {
-    let kv = kv_store.get(key.to_raw_key()).await?;
+    let kv = kv_store.get(&key.to_raw_key()).await?;
     kv.map(|kv| TableGlobalValue::from_bytes(kv.value).context(InvalidCatalogValueSnafu))
         .transpose()
 }
@@ -45,13 +44,8 @@ pub(crate) async fn batch_get_table_global_value(
     let req = BatchGetRequest {
         keys: keys.iter().map(|x| x.to_raw_key()).collect::<Vec<_>>(),
     };
-    let mut resp: BatchGetResponse = kv_store
+    let kvs = kv_store.batch_get(req).await?.kvs;
-        .batch_get(req.into())
-        .await?
-        .try_into()
-        .context(ConvertProtoDataSnafu)?;
 
-    let kvs = resp.take_kvs();
     let mut result = HashMap::with_capacity(kvs.len());
     for kv in kvs {
         let key = TableGlobalKey::try_from_raw_key(kv.key()).context(InvalidCatalogValueSnafu)?;
@@ -73,7 +67,6 @@ pub(crate) async fn put_table_global_value(
     value: &TableGlobalValue,
 ) -> Result<()> {
     let req = PutRequest {
-        header: None,
         key: key.to_raw_key(),
         value: value.as_bytes().context(InvalidCatalogValueSnafu)?,
         prev_kv: false,
@@ -101,15 +94,12 @@ pub(crate) async fn get_table_route_value(
     key: &TableRouteKey<'_>,
 ) -> Result<TableRouteValue> {
     let kv = kv_store
-        .get(key.to_string().into_bytes())
+        .get(key.to_string().as_bytes())
         .await?
         .with_context(|| TableRouteNotFoundSnafu {
             key: key.to_string(),
         })?;
-    kv.value
+    kv.value().try_into().context(DecodeTableRouteSnafu)
-        .as_slice()
-        .try_into()
-        .context(DecodeTableRouteSnafu)
 }
 
 pub(crate) async fn put_table_route_value(
@@ -118,7 +108,6 @@ pub(crate) async fn put_table_route_value(
     value: TableRouteValue,
 ) -> Result<()> {
     let req = PutRequest {
-        header: None,
         key: key.to_string().into_bytes(),
         value: value.into(),
         prev_kv: false,
@@ -150,14 +139,10 @@ async fn move_value(
 ) -> Result<Option<(Vec<u8>, Vec<u8>)>> {
     let from_key = from_key.into();
     let to_key = to_key.into();
-    let move_req = MoveValueRequest {
+    let move_req = MoveValueRequest { from_key, to_key };
-        from_key,
-        to_key,
-        ..Default::default()
-    };
     let res = kv_store.move_value(move_req).await?;
 
-    Ok(res.kv.map(|kv| (kv.key, kv.value)))
+    Ok(res.0.map(Into::into))
 }
 
 pub(crate) fn table_route_key(table_id: u32, t: &TableGlobalKey) -> TableRouteKey<'_> {

tests-integration/tests/region_failover.rs

@@ -21,8 +21,8 @@ use catalog::helper::TableGlobalKey;
 use catalog::remote::CachedMetaKvBackend;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
 use common_meta::ident::TableIdent;
-use common_meta::kv_backend::Kv;
 use common_meta::rpc::router::TableRoute;
+use common_meta::rpc::KeyValue;
 use common_meta::table_name::TableName;
 use common_meta::RegionIdent;
 use common_procedure::{watcher, ProcedureWithId};
@@ -175,7 +175,7 @@ pub async fn test_region_failover(store_type: StorageType) {
     assert!(success)
 }
 
-fn get_table_cache(instance: &Arc<Instance>, key: &str) -> Option<Option<Kv>> {
+fn get_table_cache(instance: &Arc<Instance>, key: &str) -> Option<Option<KeyValue>> {
     let catalog_manager = instance
         .catalog_manager()
         .as_any()