test(storage): wait task before checking scheduled task num (#1811)

* fix: prevent filter pushdown in distributed planner

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix metadata

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

.github/workflows/release.yml

@@ -7,25 +7,35 @@ on:
     - cron: '0 0 * * 1'
   # Mannually trigger only builds binaries.
   workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Skip docker push and release steps'
+        type: boolean
+        default: true
+      skip_test:
+        description: 'Do not run tests during build'
+        type: boolean
+        default: false
 
 name: Release
 
 env:
   RUST_TOOLCHAIN: nightly-2023-05-03
 
-  SCHEDULED_BUILD_VERSION_PREFIX: v0.3.0
+  SCHEDULED_BUILD_VERSION_PREFIX: v0.4.0
 
   SCHEDULED_PERIOD: nightly
 
   CARGO_PROFILE: nightly
 
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
-  DISABLE_RUN_TESTS: false
+  DISABLE_RUN_TESTS: ${{ inputs.skip_test || false }}
 
 jobs:
   build-macos:
     name: Build macOS binary
     strategy:
+      fail-fast: false
       matrix:
         # The file format is greptime-<os>-<arch>
         include:
@@ -120,6 +130,7 @@ jobs:
   build-linux:
     name: Build linux binary
     strategy:
+      fail-fast: false
       matrix:
         # The file format is greptime-<os>-<arch>
         include:
@@ -281,7 +292,7 @@ jobs:
     name: Build docker image
     needs: [build-linux, build-macos]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
+    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -294,7 +305,7 @@ jobs:
 
       - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
         shell: bash
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         run: |
           buildTime=`date "+%Y%m%d"`
           SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
@@ -302,7 +313,7 @@ jobs:
 
       - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
         shell: bash
-        if: github.event_name != 'schedule'
+        if: github.event_name == 'push'
         run: |
           VERSION=${{ github.ref_name }}
           echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
@@ -367,7 +378,7 @@ jobs:
     # Release artifacts only when all the artifacts are built successfully.
     needs: [build-linux, build-macos, docker]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
+    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -377,7 +388,7 @@ jobs:
 
       - name: Configure scheduled build version # the version would be ${SCHEDULED_BUILD_VERSION_PREFIX}-${SCHEDULED_PERIOD}-YYYYMMDD, like v0.2.0-nigthly-20230313.
         shell: bash
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         run: |
           buildTime=`date "+%Y%m%d"`
           SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-${{ env.SCHEDULED_PERIOD }}-$buildTime
@@ -395,13 +406,13 @@ jobs:
           fi
 
       - name: Create scheduled build git tag
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         run: |
           git tag ${{ env.SCHEDULED_BUILD_VERSION }}
 
       - name: Publish scheduled release # configure the different release title and tags.
         uses: ncipollo/release-action@v1
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         with:
           name: "Release ${{ env.SCHEDULED_BUILD_VERSION }}"
           prerelease: ${{ env.prerelease }}
@@ -413,7 +424,7 @@ jobs:
 
       - name: Publish release
         uses: ncipollo/release-action@v1
-        if: github.event_name != 'schedule'
+        if: github.event_name == 'push'
         with:
           name: "${{ github.ref_name }}"
           prerelease: ${{ env.prerelease }}
@@ -426,7 +437,7 @@ jobs:
     name: Push docker image to alibaba cloud container registry
     needs: [docker]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
+    if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false)
     continue-on-error: true
     steps:
       - name: Checkout sources
@@ -447,7 +458,7 @@ jobs:
 
       - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
         shell: bash
-        if: github.event_name == 'schedule'
+        if: github.event_name != 'push'
         run: |
           buildTime=`date "+%Y%m%d"`
           SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
@@ -455,7 +466,7 @@ jobs:
 
       - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
         shell: bash
-        if: github.event_name != 'schedule'
+        if: github.event_name == 'push'
         run: |
           VERSION=${{ github.ref_name }}
           echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV

Cargo.toml

@@ -17,6 +17,7 @@ members = [
     "src/common/meta",
     "src/common/procedure",
     "src/common/procedure-test",
+    "src/common/pprof",
     "src/common/query",
     "src/common/recordbatch",
     "src/common/runtime",
@@ -49,7 +50,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.2.0"
+version = "0.4.0"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -71,7 +72,7 @@ datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev
 datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "4398d20c56d5f7939cc2960789cb1fa7dd18e6fe" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "7aeaeaba1e0ca6a5c736b6ab2eb63144ae3d284b" }
 itertools = "0.10"
 parquet = "40.0"
 paste = "1.0"

Cross.toml

@@ -0,0 +1,7 @@
+[build]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt update && apt install -y unzip zlib1g-dev:$CROSS_DEB_ARCH",
+    "curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip && unzip protoc-3.15.8-linux-x86_64.zip -d /usr/",
+    "chmod a+x /usr/bin/protoc && chmod -R a+rx /usr/include/google",
+]

Makefile

@@ -34,7 +34,7 @@ docker-image: ## Build docker image.
 ##@ Test
 
 test: nextest ## Run unit and integration tests.
-	cargo nextest run
+	cargo nextest run --retries 3
 
 .PHONY: nextest ## Install nextest tools.
 nextest:

README.md

@@ -106,7 +106,7 @@ Please see [the online document site](https://docs.greptime.com/getting-started/
 
 Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview#connect) on our [official document site](https://docs.greptime.com/).
 
-To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients).
+To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/client/overview).
 
 ## Resources
 
@@ -123,7 +123,7 @@ To write and query data, GreptimeDB is compatible with multiple [protocols and c
 
 ### Documentation
 
-- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts.html)
+- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts/overview)
 - GreptimeDB [Developer
   Guide](https://docs.greptime.com/developer-guide/overview.html)
 - GreptimeDB [internal code document](https://greptimedb.rs)

config/datanode.example.toml

@@ -26,8 +26,8 @@ tcp_nodelay = true
 [wal]
 # WAL data directory
 # dir = "/tmp/greptimedb/wal"
-file_size = "1GB"
-purge_threshold = "50GB"
+file_size = "256MB"
+purge_threshold = "4GB"
 purge_interval = "10m"
 read_batch_size = 128
 sync_write = false
@@ -51,7 +51,7 @@ max_purge_tasks = 32
 # Create a checkpoint every <checkpoint_margin> actions.
 checkpoint_margin = 10
 # Region manifest logs and checkpoints gc execution duration
-gc_duration = '30s'
+gc_duration = '10m'
 # Whether to try creating a manifest checkpoint on region opening
 checkpoint_on_startup = false
 

config/standalone.example.toml

@@ -81,9 +81,9 @@ addr = "127.0.0.1:4004"
 # WAL data directory
 # dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
-file_size = "1GB"
-# WAL purge threshold in bytes.
-purge_threshold = "50GB"
+file_size = "256MB"
+# WAL purge threshold.
+purge_threshold = "4GB"
 # WAL purge interval in seconds.
 purge_interval = "10m"
 # WAL read batch size.
@@ -115,7 +115,7 @@ max_purge_tasks = 32
 # Create a checkpoint every <checkpoint_margin> actions.
 checkpoint_margin = 10
 # Region manifest logs and checkpoints gc execution duration
-gc_duration = '30s'
+gc_duration = '10m'
 # Whether to try creating a manifest checkpoint on region opening
 checkpoint_on_startup = false
 

src/api/src/error.rs

@@ -41,7 +41,7 @@ pub enum Error {
     ))]
     ConvertColumnDefaultConstraint {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -52,7 +52,7 @@ pub enum Error {
     ))]
     InvalidColumnDefaultConstraint {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 }

src/catalog/src/error.rs

@@ -32,18 +32,18 @@ pub enum Error {
         source
     ))]
     CompileScriptInternal {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
     #[snafu(display("Failed to open system catalog table, source: {}", source))]
     OpenSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("Failed to create system catalog table, source: {}", source))]
     CreateSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -54,7 +54,7 @@ pub enum Error {
     ))]
     CreateTable {
         table_info: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -94,7 +94,7 @@ pub enum Error {
     #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
     TableEngineNotFound {
         engine_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -132,7 +132,7 @@ pub enum Error {
     #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
     OpenTable {
         table_info: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -147,13 +147,13 @@ pub enum Error {
 
     #[snafu(display("Failed to read system catalog table records"))]
     ReadSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
     #[snafu(display("Failed to create recordbatch, source: {}", source))]
     CreateRecordBatch {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
@@ -162,7 +162,7 @@ pub enum Error {
         source
     ))]
     InsertCatalogRecord {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -173,7 +173,7 @@ pub enum Error {
     ))]
     DeregisterTable {
         request: DeregisterTableRequest,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -182,36 +182,36 @@ pub enum Error {
 
     #[snafu(display("Failed to scan system catalog table, source: {}", source))]
     SystemCatalogTableScan {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("{source}"))]
     Internal {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
     #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
     SystemCatalogTableScanExec {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_query::error::Error,
     },
     #[snafu(display("Cannot parse catalog value, source: {}", source))]
     InvalidCatalogValue {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_catalog::error::Error,
     },
 
     #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
     MetaSrv {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_client::error::Error,
     },
 
     #[snafu(display("Invalid table info in catalog, source: {}", source))]
     InvalidTableInfoInCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -230,7 +230,7 @@ pub enum Error {
 
     #[snafu(display("Table schema mismatch, source: {}", source))]
     TableSchemaMismatch {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -258,7 +258,7 @@ impl ErrorExt for Error {
 
             Error::Generic { .. } | Error::SystemCatalogTypeMismatch { .. } => StatusCode::Internal,
 
-            Error::ReadSystemCatalog { source, .. } | Error::CreateRecordBatch { source } => {
+            Error::ReadSystemCatalog { source, .. } | Error::CreateRecordBatch { source, .. } => {
                 source.status_code()
             }
             Error::InvalidCatalogValue { source, .. } => source.status_code(),
@@ -275,14 +275,14 @@ impl ErrorExt for Error {
             | Error::OpenTable { source, .. }
             | Error::CreateTable { source, .. }
             | Error::DeregisterTable { source, .. }
-            | Error::TableSchemaMismatch { source } => source.status_code(),
+            | Error::TableSchemaMismatch { source, .. } => source.status_code(),
 
             Error::MetaSrv { source, .. } => source.status_code(),
-            Error::SystemCatalogTableScan { source } => source.status_code(),
-            Error::SystemCatalogTableScanExec { source } => source.status_code(),
-            Error::InvalidTableInfoInCatalog { source } => source.status_code(),
+            Error::SystemCatalogTableScan { source, .. } => source.status_code(),
+            Error::SystemCatalogTableScanExec { source, .. } => source.status_code(),
+            Error::InvalidTableInfoInCatalog { source, .. } => source.status_code(),
 
-            Error::CompileScriptInternal { source } | Error::Internal { source } => {
+            Error::CompileScriptInternal { source, .. } | Error::Internal { source, .. } => {
                 source.status_code()
             }
 

src/catalog/src/helper.rs

@@ -91,7 +91,7 @@ pub fn build_table_regional_prefix(
 }
 
 /// Table global info has only one key across all datanodes so it does not have `node_id` field.
-#[derive(Clone)]
+#[derive(Clone, Hash, Eq, PartialEq)]
 pub struct TableGlobalKey {
     pub catalog_name: String,
     pub schema_name: String,
@@ -124,6 +124,14 @@ impl TableGlobalKey {
             table_name: captures[3].to_string(),
         })
     }
+
+    pub fn to_raw_key(&self) -> Vec<u8> {
+        self.to_string().into_bytes()
+    }
+
+    pub fn try_from_raw_key(key: &[u8]) -> Result<Self, Error> {
+        Self::parse(String::from_utf8_lossy(key))
+    }
 }
 
 /// Table global info contains necessary info for a datanode to create table regions, including
@@ -141,6 +149,10 @@ impl TableGlobalValue {
     pub fn table_id(&self) -> TableId {
         self.table_info.ident.table_id
     }
+
+    pub fn engine(&self) -> &str {
+        &self.table_info.meta.engine
+    }
 }
 
 /// Table regional info that varies between datanode, so it contains a `node_id` field.

src/catalog/src/local/manager.rs

@@ -467,10 +467,7 @@ impl CatalogManager for LocalCatalogManager {
                 .ident
                 .table_id;
 
-            if !self.system.deregister_table(&request, table_id).await? {
-                return Ok(false);
-            }
-
+            self.system.deregister_table(&request, table_id).await?;
             self.catalogs.deregister_table(request).await
         }
     }

src/catalog/src/remote.rs

@@ -17,7 +17,7 @@ use std::fmt::Debug;
 use std::pin::Pin;
 use std::sync::Arc;
 
-pub use client::CachedMetaKvBackend;
+pub use client::{CachedMetaKvBackend, MetaKvBackend};
 use futures::Stream;
 use futures_util::StreamExt;
 pub use manager::{RemoteCatalogManager, RemoteCatalogProvider, RemoteSchemaProvider};
@@ -29,6 +29,7 @@ mod manager;
 
 #[cfg(feature = "testing")]
 pub mod mock;
+pub mod region_alive_keeper;
 
 #[derive(Debug, Clone)]
 pub struct Kv(pub Vec<u8>, pub Vec<u8>);

src/catalog/src/remote/manager.rs

@@ -20,13 +20,14 @@ use std::sync::Arc;
 use async_stream::stream;
 use async_trait::async_trait;
 use common_catalog::consts::{MAX_SYS_TABLE_ID, MITO_ENGINE};
+use common_meta::ident::TableIdent;
 use common_telemetry::{debug, error, info, warn};
 use dashmap::DashMap;
 use futures::Stream;
 use futures_util::{StreamExt, TryStreamExt};
 use metrics::{decrement_gauge, increment_gauge};
 use parking_lot::RwLock;
-use snafu::{OptionExt, ResultExt};
+use snafu::{ensure, OptionExt, ResultExt};
 use table::engine::manager::TableEngineManagerRef;
 use table::engine::{EngineContext, TableReference};
 use table::requests::{CreateTableRequest, OpenTableRequest};
@@ -43,6 +44,7 @@ use crate::helper::{
     build_table_regional_prefix, CatalogKey, CatalogValue, SchemaKey, SchemaValue, TableGlobalKey,
     TableGlobalValue, TableRegionalKey, TableRegionalValue, CATALOG_KEY_PREFIX,
 };
+use crate::remote::region_alive_keeper::RegionAliveKeepers;
 use crate::remote::{Kv, KvBackendRef};
 use crate::{
     handle_system_table_request, CatalogManager, CatalogProvider, CatalogProviderRef,
@@ -57,16 +59,23 @@ pub struct RemoteCatalogManager {
     catalogs: Arc<RwLock<DashMap<String, CatalogProviderRef>>>,
     engine_manager: TableEngineManagerRef,
     system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
 }
 
 impl RemoteCatalogManager {
-    pub fn new(engine_manager: TableEngineManagerRef, node_id: u64, backend: KvBackendRef) -> Self {
+    pub fn new(
+        engine_manager: TableEngineManagerRef,
+        node_id: u64,
+        backend: KvBackendRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
+    ) -> Self {
         Self {
             engine_manager,
             node_id,
             backend,
             catalogs: Default::default(),
             system_table_requests: Default::default(),
+            region_alive_keepers,
         }
     }
 
@@ -76,6 +85,7 @@ impl RemoteCatalogManager {
             catalog_name: catalog_name.to_string(),
             backend: self.backend.clone(),
             engine_manager: self.engine_manager.clone(),
+            region_alive_keepers: self.region_alive_keepers.clone(),
         }) as _
     }
 
@@ -123,10 +133,17 @@ impl RemoteCatalogManager {
 
             increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
 
+            let region_alive_keepers = self.region_alive_keepers.clone();
             joins.push(common_runtime::spawn_bg(async move {
-                let max_table_id =
-                    initiate_schemas(node_id, backend, engine_manager, &catalog_name, catalog)
-                        .await?;
+                let max_table_id = initiate_schemas(
+                    node_id,
+                    backend,
+                    engine_manager,
+                    &catalog_name,
+                    catalog,
+                    region_alive_keepers,
+                )
+                .await?;
                 info!(
                     "Catalog name: {}, max table id allocated: {}",
                     &catalog_name, max_table_id
@@ -155,6 +172,7 @@ impl RemoteCatalogManager {
             self.engine_manager.clone(),
             catalog_name,
             schema_name,
+            self.region_alive_keepers.clone(),
         );
 
         let catalog_provider = self.new_catalog_provider(catalog_name);
@@ -200,6 +218,7 @@ fn new_schema_provider(
     engine_manager: TableEngineManagerRef,
     catalog_name: &str,
     schema_name: &str,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
 ) -> SchemaProviderRef {
     Arc::new(RemoteSchemaProvider {
         catalog_name: catalog_name.to_string(),
@@ -207,6 +226,7 @@ fn new_schema_provider(
         node_id,
         backend,
         engine_manager,
+        region_alive_keepers,
     }) as _
 }
 
@@ -240,6 +260,7 @@ async fn initiate_schemas(
     engine_manager: TableEngineManagerRef,
     catalog_name: &str,
     catalog: CatalogProviderRef,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
 ) -> Result<u32> {
     let mut schemas = iter_remote_schemas(&backend, catalog_name).await;
     let mut joins = Vec::new();
@@ -259,6 +280,7 @@ async fn initiate_schemas(
                     engine_manager.clone(),
                     &catalog_name,
                     &schema_name,
+                    region_alive_keepers.clone(),
                 );
                 catalog
                     .register_schema(schema_name.clone(), schema.clone())
@@ -576,34 +598,33 @@ impl CatalogManager for RemoteCatalogManager {
     }
 
     async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        let catalog_name = request.catalog;
-        let schema_name = request.schema;
+        let catalog = &request.catalog;
+        let schema = &request.schema;
+        let table_name = &request.table_name;
+
         let schema_provider = self
-            .catalog(&catalog_name)
+            .catalog(catalog)
             .await?
             .context(CatalogNotFoundSnafu {
-                catalog_name: &catalog_name,
+                catalog_name: catalog,
             })?
-            .schema(&schema_name)
+            .schema(schema)
             .await?
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &catalog_name,
-                schema: &schema_name,
-            })?;
-        if schema_provider.table_exist(&request.table_name).await? {
-            return TableExistsSnafu {
-                table: format!("{}.{}.{}", &catalog_name, &schema_name, &request.table_name),
+            .context(SchemaNotFoundSnafu { catalog, schema })?;
+        ensure!(
+            !schema_provider.table_exist(table_name).await?,
+            TableExistsSnafu {
+                table: common_catalog::format_full_table_name(catalog, schema, table_name),
             }
-            .fail();
-        }
+        );
 
         increment_gauge!(
             crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
             1.0,
-            &[crate::metrics::db_label(&catalog_name, &schema_name)],
+            &[crate::metrics::db_label(catalog, schema)],
         );
         schema_provider
-            .register_table(request.table_name, request.table)
+            .register_table(table_name.to_string(), request.table)
             .await?;
 
         Ok(true)
@@ -626,7 +647,22 @@ impl CatalogManager for RemoteCatalogManager {
             1.0,
             &[crate::metrics::db_label(catalog_name, schema_name)],
         );
-        Ok(result.is_none())
+
+        if let Some(table) = result.as_ref() {
+            let table_info = table.table_info();
+            let table_ident = TableIdent {
+                catalog: request.catalog,
+                schema: request.schema,
+                table: request.table_name,
+                table_id: table_info.ident.table_id,
+                engine: table_info.meta.engine.clone(),
+            };
+            self.region_alive_keepers
+                .deregister_table(&table_ident)
+                .await;
+        }
+
+        Ok(true)
     }
 
     async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
@@ -644,6 +680,7 @@ impl CatalogManager for RemoteCatalogManager {
             self.engine_manager.clone(),
             &catalog_name,
             &schema_name,
+            self.region_alive_keepers.clone(),
         );
         catalog_provider
             .register_schema(schema_name, schema_provider)
@@ -779,6 +816,7 @@ pub struct RemoteCatalogProvider {
     catalog_name: String,
     backend: KvBackendRef,
     engine_manager: TableEngineManagerRef,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
 }
 
 impl RemoteCatalogProvider {
@@ -787,12 +825,14 @@ impl RemoteCatalogProvider {
         backend: KvBackendRef,
         engine_manager: TableEngineManagerRef,
         node_id: u64,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
     ) -> Self {
         Self {
             node_id,
             catalog_name,
             backend,
             engine_manager,
+            region_alive_keepers,
         }
     }
 
@@ -810,6 +850,7 @@ impl RemoteCatalogProvider {
             node_id: self.node_id,
             backend: self.backend.clone(),
             engine_manager: self.engine_manager.clone(),
+            region_alive_keepers: self.region_alive_keepers.clone(),
         };
         Arc::new(provider) as Arc<_>
     }
@@ -872,6 +913,7 @@ pub struct RemoteSchemaProvider {
     node_id: u64,
     backend: KvBackendRef,
     engine_manager: TableEngineManagerRef,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
 }
 
 impl RemoteSchemaProvider {
@@ -881,6 +923,7 @@ impl RemoteSchemaProvider {
         node_id: u64,
         engine_manager: TableEngineManagerRef,
         backend: KvBackendRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
     ) -> Self {
         Self {
             catalog_name,
@@ -888,6 +931,7 @@ impl RemoteSchemaProvider {
             node_id,
             backend,
             engine_manager,
+            region_alive_keepers,
         }
     }
 
@@ -910,15 +954,26 @@ impl SchemaProvider for RemoteSchemaProvider {
     async fn table_names(&self) -> Result<Vec<String>> {
         let key_prefix = build_table_regional_prefix(&self.catalog_name, &self.schema_name);
         let iter = self.backend.range(key_prefix.as_bytes());
-        let table_names = iter
+        let regional_keys = iter
             .map(|kv| {
                 let Kv(key, _) = kv?;
                 let regional_key = TableRegionalKey::parse(String::from_utf8_lossy(&key))
                     .context(InvalidCatalogValueSnafu)?;
-                Ok(regional_key.table_name)
+                Ok(regional_key)
             })
-            .try_collect()
+            .try_collect::<Vec<_>>()
             .await?;
+
+        let table_names = regional_keys
+            .into_iter()
+            .filter_map(|x| {
+                if x.node_id == self.node_id {
+                    Some(x.table_name)
+                } else {
+                    None
+                }
+            })
+            .collect();
         Ok(table_names)
     }
 
@@ -970,6 +1025,18 @@ impl SchemaProvider for RemoteSchemaProvider {
                 &table_value.as_bytes().context(InvalidCatalogValueSnafu)?,
             )
             .await?;
+
+        let table_ident = TableIdent {
+            catalog: table_info.catalog_name.clone(),
+            schema: table_info.schema_name.clone(),
+            table: table_info.name.clone(),
+            table_id: table_info.ident.table_id,
+            engine: table_info.meta.engine.clone(),
+        };
+        self.region_alive_keepers
+            .register_table(table_ident, table)
+            .await?;
+
         debug!(
             "Successfully set catalog table entry, key: {}, table value: {:?}",
             table_key, table_value

src/catalog/src/remote/mock.rs

@@ -27,9 +27,11 @@ use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, Schema};
 use datatypes::vectors::StringVector;
 use serde::Serializer;
-use table::engine::{EngineContext, TableEngine, TableReference};
+use table::engine::{CloseTableResult, EngineContext, TableEngine, TableReference};
 use table::metadata::TableId;
-use table::requests::{AlterTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest};
+use table::requests::{
+    AlterTableRequest, CloseTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest,
+};
 use table::test_util::MemTable;
 use table::TableRef;
 use tokio::sync::RwLock;
@@ -183,6 +185,8 @@ impl TableEngine for MockTableEngine {
         let table_name = request.table_name.clone();
         let catalog_name = request.catalog_name.clone();
         let schema_name = request.schema_name.clone();
+        let table_full_name =
+            TableReference::full(&catalog_name, &schema_name, &table_name).to_string();
 
         let default_table_id = "0".to_owned();
         let table_id = TableId::from_str(
@@ -211,7 +215,7 @@ impl TableEngine for MockTableEngine {
         )) as Arc<_>;
 
         let mut tables = self.tables.write().await;
-        tables.insert(table_name, table.clone() as TableRef);
+        tables.insert(table_full_name, table.clone() as TableRef);
         Ok(table)
     }
 
@@ -263,6 +267,19 @@ impl TableEngine for MockTableEngine {
         unimplemented!()
     }
 
+    async fn close_table(
+        &self,
+        _ctx: &EngineContext,
+        request: CloseTableRequest,
+    ) -> table::Result<CloseTableResult> {
+        let _ = self
+            .tables
+            .write()
+            .await
+            .remove(&request.table_ref().to_string());
+        Ok(CloseTableResult::Released(vec![]))
+    }
+
     async fn close(&self) -> table::Result<()> {
         Ok(())
     }

src/catalog/src/remote/region_alive_keeper.rs

@@ -0,0 +1,821 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_meta::error::InvalidProtoMsgSnafu;
+use common_meta::heartbeat::handler::{
+    HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
+};
+use common_meta::ident::TableIdent;
+use common_meta::RegionIdent;
+use common_telemetry::{debug, error, info, warn};
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::RegionNumber;
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::{CloseTableResult, EngineContext, TableEngineRef};
+use table::requests::CloseTableRequest;
+use table::TableRef;
+use tokio::sync::{mpsc, oneshot, Mutex};
+use tokio::task::JoinHandle;
+use tokio::time::{Duration, Instant};
+
+use crate::error::{Result, TableEngineNotFoundSnafu};
+
+/// [RegionAliveKeepers] manages all [RegionAliveKeeper] in a scope of tables.
+pub struct RegionAliveKeepers {
+    table_engine_manager: TableEngineManagerRef,
+    keepers: Arc<Mutex<HashMap<TableIdent, Arc<RegionAliveKeeper>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+
+    /// The epoch when [RegionAliveKeepers] is created. It's used to get a monotonically non-decreasing
+    /// elapsed time when submitting heartbeats to Metasrv (because [Instant] is monotonically
+    /// non-decreasing). The heartbeat request will carry the duration since this epoch, and the
+    /// duration acts like an "invariant point" for region's keep alive lease.
+    epoch: Instant,
+}
+
+impl RegionAliveKeepers {
+    pub fn new(
+        table_engine_manager: TableEngineManagerRef,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            table_engine_manager,
+            keepers: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+            epoch: Instant::now(),
+        }
+    }
+
+    pub async fn find_keeper(&self, table_ident: &TableIdent) -> Option<Arc<RegionAliveKeeper>> {
+        self.keepers.lock().await.get(table_ident).cloned()
+    }
+
+    pub async fn register_table(&self, table_ident: TableIdent, table: TableRef) -> Result<()> {
+        let keeper = self.find_keeper(&table_ident).await;
+        if keeper.is_some() {
+            return Ok(());
+        }
+
+        let table_engine = self
+            .table_engine_manager
+            .engine(&table_ident.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &table_ident.engine,
+            })?;
+
+        let keeper = Arc::new(RegionAliveKeeper::new(
+            table_engine,
+            table_ident.clone(),
+            self.heartbeat_interval_millis,
+        ));
+        for r in table.table_info().meta.region_numbers.iter() {
+            keeper.register_region(*r).await;
+        }
+
+        let mut keepers = self.keepers.lock().await;
+        keepers.insert(table_ident.clone(), keeper.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            keeper.start().await;
+
+            info!("RegionAliveKeeper for table {table_ident} is started!");
+        } else {
+            info!("RegionAliveKeeper for table {table_ident} is registered but not started yet!");
+        }
+        Ok(())
+    }
+
+    pub async fn deregister_table(
+        &self,
+        table_ident: &TableIdent,
+    ) -> Option<Arc<RegionAliveKeeper>> {
+        self.keepers.lock().await.remove(table_ident).map(|x| {
+            info!("Deregister RegionAliveKeeper for table {table_ident}");
+            x
+        })
+    }
+
+    pub async fn register_region(&self, region_ident: &RegionIdent) {
+        let table_ident = &region_ident.table_ident;
+        let Some(keeper) = self.find_keeper(table_ident).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        keeper.register_region(region_ident.region_number).await
+    }
+
+    pub async fn deregister_region(&self, region_ident: &RegionIdent) {
+        let table_ident = &region_ident.table_ident;
+        let Some(keeper) = self.find_keeper(table_ident).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        let _ = keeper.deregister_region(region_ident.region_number).await;
+    }
+
+    pub async fn start(&self) {
+        let keepers = self.keepers.lock().await;
+        for keeper in keepers.values() {
+            keeper.start().await;
+        }
+        self.started.store(true, Ordering::Relaxed);
+
+        info!(
+            "RegionAliveKeepers for tables {:?} are started!",
+            keepers.keys().map(|x| x.to_string()).collect::<Vec<_>>(),
+        );
+    }
+
+    pub fn epoch(&self) -> Instant {
+        self.epoch
+    }
+}
+
+#[async_trait]
+impl HeartbeatResponseHandler for RegionAliveKeepers {
+    fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
+        !ctx.response.region_leases.is_empty()
+    }
+
+    async fn handle(
+        &self,
+        ctx: &mut HeartbeatResponseHandlerContext,
+    ) -> common_meta::error::Result<HandleControl> {
+        let leases = ctx.response.region_leases.drain(..).collect::<Vec<_>>();
+        for lease in leases {
+            let table_ident: TableIdent = match lease
+                .table_ident
+                .context(InvalidProtoMsgSnafu {
+                    err_msg: "'table_ident' is missing in RegionLease",
+                })
+                .and_then(|x| x.try_into())
+            {
+                Ok(x) => x,
+                Err(e) => {
+                    error!(e; "");
+                    continue;
+                }
+            };
+
+            let Some(keeper) = self.keepers.lock().await.get(&table_ident).cloned() else {
+                // Alive keeper could be affected by lagging msg, just warn and ignore.
+                warn!("Alive keeper for table {table_ident} is not found!");
+                continue;
+            };
+
+            let start_instant = self.epoch + Duration::from_millis(lease.duration_since_epoch);
+            let deadline = start_instant + Duration::from_secs(lease.lease_seconds);
+            keeper.keep_lived(lease.regions, deadline).await;
+        }
+        Ok(HandleControl::Continue)
+    }
+}
+
+/// [RegionAliveKeeper] starts a countdown for each region in a table. When deadline is reached,
+/// the region will be closed.
+/// The deadline is controlled by Metasrv. It works like "lease" for regions: a Datanode submits its
+/// opened regions to Metasrv, in heartbeats. If Metasrv decides some region could be resided in this
+/// Datanode, it will "extend" the region's "lease", with a deadline for [RegionAliveKeeper] to
+/// countdown.
+pub struct RegionAliveKeeper {
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    countdown_task_handles: Arc<Mutex<HashMap<RegionNumber, Arc<CountdownTaskHandle>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+}
+
+impl RegionAliveKeeper {
+    fn new(
+        table_engine: TableEngineRef,
+        table_ident: TableIdent,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            table_engine,
+            table_ident,
+            countdown_task_handles: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+        }
+    }
+
+    async fn find_handle(&self, region: &RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .get(region)
+            .cloned()
+    }
+
+    async fn register_region(&self, region: RegionNumber) {
+        if self.find_handle(&region).await.is_some() {
+            return;
+        }
+
+        let countdown_task_handles = Arc::downgrade(&self.countdown_task_handles);
+        let on_task_finished = async move {
+            if let Some(x) = countdown_task_handles.upgrade() {
+                x.lock().await.remove(&region);
+            } // Else the countdown task handles map could be dropped because the keeper is dropped.
+        };
+        let handle = Arc::new(CountdownTaskHandle::new(
+            self.table_engine.clone(),
+            self.table_ident.clone(),
+            region,
+            || on_task_finished,
+        ));
+
+        let mut handles = self.countdown_task_handles.lock().await;
+        handles.insert(region, handle.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            handle.start(self.heartbeat_interval_millis).await;
+
+            info!(
+                "Region alive countdown for region {region} in table {} is started!",
+                self.table_ident
+            );
+        } else {
+            info!(
+                "Region alive countdown for region {region} in table {} is registered but not started yet!",
+                self.table_ident
+            );
+        }
+    }
+
+    async fn deregister_region(&self, region: RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .remove(&region)
+            .map(|x| {
+                info!(
+                    "Deregister alive countdown for region {region} in table {}",
+                    self.table_ident
+                );
+                x
+            })
+    }
+
+    async fn start(&self) {
+        let handles = self.countdown_task_handles.lock().await;
+        for handle in handles.values() {
+            handle.start(self.heartbeat_interval_millis).await;
+        }
+
+        self.started.store(true, Ordering::Relaxed);
+        info!(
+            "Region alive countdowns for regions {:?} in table {} are started!",
+            handles.keys().copied().collect::<Vec<_>>(),
+            self.table_ident
+        );
+    }
+
+    async fn keep_lived(&self, designated_regions: Vec<RegionNumber>, deadline: Instant) {
+        for region in designated_regions {
+            if let Some(handle) = self.find_handle(&region).await {
+                handle.reset_deadline(deadline).await;
+            }
+            // Else the region alive keeper might be triggered by lagging messages, we can safely ignore it.
+        }
+    }
+
+    pub async fn deadline(&self, region: RegionNumber) -> Option<Instant> {
+        let mut deadline = None;
+        if let Some(handle) = self.find_handle(&region).await {
+            let (s, r) = oneshot::channel();
+            if handle.tx.send(CountdownCommand::Deadline(s)).await.is_ok() {
+                deadline = r.await.ok()
+            }
+        }
+        deadline
+    }
+}
+
+#[derive(Debug)]
+enum CountdownCommand {
+    Start(u64),
+    Reset(Instant),
+    Deadline(oneshot::Sender<Instant>),
+}
+
+struct CountdownTaskHandle {
+    tx: mpsc::Sender<CountdownCommand>,
+    handler: JoinHandle<()>,
+    table_ident: TableIdent,
+    region: RegionNumber,
+}
+
+impl CountdownTaskHandle {
+    /// Creates a new [CountdownTaskHandle] and starts the countdown task.
+    /// # Params
+    /// - `on_task_finished`: a callback to be invoked when the task is finished. Note that it will not
+    ///   be invoked if the task is cancelled (by dropping the handle). This is because we want something
+    ///   meaningful to be done when the task is finished, e.g. deregister the handle from the map.
+    ///   While dropping the handle does not necessarily mean the task is finished.
+    fn new<Fut>(
+        table_engine: TableEngineRef,
+        table_ident: TableIdent,
+        region: RegionNumber,
+        on_task_finished: impl FnOnce() -> Fut + Send + 'static,
+    ) -> Self
+    where
+        Fut: Future<Output = ()> + Send,
+    {
+        let (tx, rx) = mpsc::channel(1024);
+
+        let mut countdown_task = CountdownTask {
+            table_engine,
+            table_ident: table_ident.clone(),
+            region,
+            rx,
+        };
+        let handler = common_runtime::spawn_bg(async move {
+            countdown_task.run().await;
+            on_task_finished().await;
+        });
+
+        Self {
+            tx,
+            handler,
+            table_ident,
+            region,
+        }
+    }
+
+    async fn start(&self, heartbeat_interval_millis: u64) {
+        if let Err(e) = self
+            .tx
+            .send(CountdownCommand::Start(heartbeat_interval_millis))
+            .await
+        {
+            warn!(
+                "Failed to start region alive keeper countdown: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+
+    async fn reset_deadline(&self, deadline: Instant) {
+        if let Err(e) = self.tx.send(CountdownCommand::Reset(deadline)).await {
+            warn!(
+                "Failed to reset region alive keeper deadline: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+}
+
+impl Drop for CountdownTaskHandle {
+    fn drop(&mut self) {
+        debug!(
+            "Aborting region alive countdown task for region {} in table {}",
+            self.region, self.table_ident,
+        );
+        self.handler.abort();
+    }
+}
+
+struct CountdownTask {
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    region: RegionNumber,
+    rx: mpsc::Receiver<CountdownCommand>,
+}
+
+impl CountdownTask {
+    async fn run(&mut self) {
+        // 30 years. See `Instant::far_future`.
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
+
+        // Make sure the alive countdown is not gonna happen before heartbeat task is started (the
+        // "start countdown" command will be sent from heartbeat task).
+        let countdown = tokio::time::sleep_until(far_future);
+        tokio::pin!(countdown);
+
+        let region = &self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            tokio::select! {
+                command = self.rx.recv() => {
+                    match command {
+                        Some(CountdownCommand::Start(heartbeat_interval_millis)) => {
+                            // Set first deadline in 4 heartbeats (roughly after 20 seconds from now if heartbeat
+                            // interval is set to default 5 seconds), to make Datanode and Metasrv more tolerable to
+                            // network or other jitters during startup.
+                            let first_deadline = Instant::now() + Duration::from_millis(heartbeat_interval_millis) * 4;
+                            countdown.set(tokio::time::sleep_until(first_deadline));
+                        },
+                        Some(CountdownCommand::Reset(deadline)) => {
+                            if countdown.deadline() < deadline {
+                                debug!(
+                                    "Reset deadline of region {region} of table {table_ident} to approximately {} seconds later",
+                                    (deadline - Instant::now()).as_secs_f32(),
+                                );
+                                countdown.set(tokio::time::sleep_until(deadline));
+                            }
+                            // Else the countdown could be either:
+                            // - not started yet;
+                            // - during startup protection;
+                            // - received a lagging heartbeat message.
+                            // All can be safely ignored.
+                        },
+                        None => {
+                            info!(
+                                "The handle of countdown task for region {region} of table {table_ident} \
+                                is dropped, RegionAliveKeeper out."
+                            );
+                            break;
+                        },
+                        Some(CountdownCommand::Deadline(tx)) => {
+                            let _ = tx.send(countdown.deadline());
+                        }
+                    }
+                }
+                () = &mut countdown => {
+                    let result = self.close_region().await;
+                    warn!(
+                        "Region {region} of table {table_ident} is closed, result: {result:?}. \
+                        RegionAliveKeeper out.",
+                    );
+                    break;
+                }
+            }
+        }
+    }
+
+    async fn close_region(&self) -> CloseTableResult {
+        let ctx = EngineContext::default();
+        let region = self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            let request = CloseTableRequest {
+                catalog_name: table_ident.catalog.clone(),
+                schema_name: table_ident.schema.clone(),
+                table_name: table_ident.table.clone(),
+                region_numbers: vec![region],
+                flush: true,
+            };
+            match self.table_engine.close_table(&ctx, request).await {
+                Ok(result) => return result,
+                // If region is failed to close, immediately retry. Maybe we should panic instead?
+                Err(e) => error!(e;
+                    "Failed to close region {region} of table {table_ident}. \
+                    For the integrity of data, retry closing and retry without wait.",
+                ),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+
+    use api::v1::meta::{HeartbeatResponse, RegionLease};
+    use common_meta::heartbeat::mailbox::HeartbeatMailbox;
+    use datatypes::schema::RawSchema;
+    use table::engine::manager::MemoryTableEngineManager;
+    use table::engine::{TableEngine, TableReference};
+    use table::requests::{CreateTableRequest, TableOptions};
+    use table::test_util::EmptyTable;
+
+    use super::*;
+    use crate::remote::mock::MockTableEngine;
+
+    async fn prepare_keepers() -> (TableIdent, RegionAliveKeepers) {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_engine_manager = Arc::new(MemoryTableEngineManager::new(table_engine));
+        let keepers = RegionAliveKeepers::new(table_engine_manager, 5000);
+
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id: 1,
+            engine: "MockTableEngine".to_string(),
+        };
+        let table = Arc::new(EmptyTable::new(CreateTableRequest {
+            id: 1,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![1, 2, 3],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "MockTableEngine".to_string(),
+        }));
+        keepers
+            .register_table(table_ident.clone(), table)
+            .await
+            .unwrap();
+        assert!(keepers.keepers.lock().await.contains_key(&table_ident));
+
+        (table_ident, keepers)
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_handle_heartbeat_response() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers.start().await;
+        let startup_protection_until = Instant::now() + Duration::from_secs(21);
+
+        let duration_since_epoch = (Instant::now() - keepers.epoch).as_millis() as _;
+        let lease_seconds = 100;
+        let response = HeartbeatResponse {
+            region_leases: vec![RegionLease {
+                table_ident: Some(table_ident.clone().into()),
+                regions: vec![1, 3], // Not extending region 2's lease time.
+                duration_since_epoch,
+                lease_seconds,
+            }],
+            ..Default::default()
+        };
+        let keep_alive_until = keepers.epoch
+            + Duration::from_millis(duration_since_epoch)
+            + Duration::from_secs(lease_seconds);
+
+        let (tx, _) = mpsc::channel(8);
+        let mailbox = Arc::new(HeartbeatMailbox::new(tx));
+        let mut ctx = HeartbeatResponseHandlerContext::new(mailbox, response);
+
+        assert!(keepers.handle(&mut ctx).await.unwrap() == HandleControl::Continue);
+
+        // sleep to wait for background task spawned in `handle`
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        async fn test(
+            keeper: &Arc<RegionAliveKeeper>,
+            region_number: RegionNumber,
+            startup_protection_until: Instant,
+            keep_alive_until: Instant,
+            is_kept_live: bool,
+        ) {
+            let deadline = keeper.deadline(region_number).await.unwrap();
+            if is_kept_live {
+                assert!(deadline > startup_protection_until && deadline == keep_alive_until);
+            } else {
+                assert!(deadline <= startup_protection_until);
+            }
+        }
+
+        let keeper = &keepers
+            .keepers
+            .lock()
+            .await
+            .get(&table_ident)
+            .cloned()
+            .unwrap();
+
+        // Test region 1 and 3 is kept lived. Their deadlines are updated to desired instant.
+        test(keeper, 1, startup_protection_until, keep_alive_until, true).await;
+        test(keeper, 3, startup_protection_until, keep_alive_until, true).await;
+
+        // Test region 2 is not kept lived. It's deadline is not updated: still during startup protection period.
+        test(keeper, 2, startup_protection_until, keep_alive_until, false).await;
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keepers() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers
+            .register_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 4,
+            })
+            .await;
+
+        keepers.start().await;
+        for keeper in keepers.keepers.lock().await.values() {
+            let regions = {
+                let handles = keeper.countdown_task_handles.lock().await;
+                handles.keys().copied().collect::<Vec<_>>()
+            };
+            for region in regions {
+                // assert countdown tasks are started
+                let deadline = keeper.deadline(region).await.unwrap();
+                assert!(deadline <= Instant::now() + Duration::from_secs(20));
+            }
+        }
+
+        keepers
+            .deregister_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 1,
+            })
+            .await;
+        let mut regions = keepers
+            .find_keeper(&table_ident)
+            .await
+            .unwrap()
+            .countdown_task_handles
+            .lock()
+            .await
+            .keys()
+            .copied()
+            .collect::<Vec<_>>();
+        regions.sort();
+        assert_eq!(regions, vec![2, 3, 4]);
+
+        let keeper = keepers.deregister_table(&table_ident).await.unwrap();
+        assert!(Arc::try_unwrap(keeper).is_ok(), "keeper is not dropped");
+        assert!(keepers.keepers.lock().await.is_empty());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keeper() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let keeper = RegionAliveKeeper::new(table_engine, table_ident, 1000);
+
+        let region = 1;
+        assert!(keeper.find_handle(&region).await.is_none());
+        keeper.register_region(region).await;
+        assert!(keeper.find_handle(&region).await.is_some());
+
+        let ten_seconds_later = || Instant::now() + Duration::from_secs(10);
+
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        assert!(keeper.find_handle(&2).await.is_none());
+        assert!(keeper.find_handle(&3).await.is_none());
+
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert if keeper is not started, keep_lived is of no use
+        assert!(keeper.deadline(region).await.unwrap() > far_future);
+
+        keeper.start().await;
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        // assert keep_lived works if keeper is started
+        assert!(keeper.deadline(region).await.unwrap() <= ten_seconds_later());
+
+        let handle = keeper.deregister_region(region).await.unwrap();
+        assert!(Arc::try_unwrap(handle).is_ok(), "handle is not dropped");
+        assert!(keeper.find_handle(&region).await.is_none());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_handle() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(
+            table_engine.clone(),
+            table_ident.clone(),
+            1,
+            || async move { finished_clone.store(true, Ordering::Relaxed) },
+        );
+        let tx = handle.tx.clone();
+
+        // assert countdown task is running
+        assert!(tx.send(CountdownCommand::Start(5000)).await.is_ok());
+        assert!(!finished.load(Ordering::Relaxed));
+
+        drop(handle);
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        // assert countdown task is stopped
+        assert!(tx
+            .try_send(CountdownCommand::Reset(
+                Instant::now() + Duration::from_secs(10)
+            ))
+            .is_err());
+        // assert `on_task_finished` is not called (because the task is aborted by the handle's drop)
+        assert!(!finished.load(Ordering::Relaxed));
+
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(table_engine, table_ident, 1, || async move {
+            finished_clone.store(true, Ordering::Relaxed)
+        });
+        handle.tx.send(CountdownCommand::Start(100)).await.unwrap();
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        // assert `on_task_finished` is called when task is finished normally
+        assert!(finished.load(Ordering::Relaxed));
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_run() {
+        let ctx = &EngineContext::default();
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let request = CreateTableRequest {
+            id: 1,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "mito".to_string(),
+        };
+        let table_ref = TableReference::full(catalog, schema, table);
+
+        let table_engine = Arc::new(MockTableEngine::default());
+        table_engine.create_table(ctx, request).await.unwrap();
+
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let (tx, rx) = mpsc::channel(10);
+        let mut task = CountdownTask {
+            table_engine: table_engine.clone(),
+            table_ident,
+            region: 1,
+            rx,
+        };
+        common_runtime::spawn_bg(async move {
+            task.run().await;
+        });
+
+        async fn deadline(tx: &mpsc::Sender<CountdownCommand>) -> Instant {
+            let (s, r) = oneshot::channel();
+            tx.send(CountdownCommand::Deadline(s)).await.unwrap();
+            r.await.unwrap()
+        }
+
+        // if countdown task is not started, its deadline is set to far future
+        assert!(deadline(&tx).await > Instant::now() + Duration::from_secs(86400 * 365 * 29));
+
+        // start countdown in 250ms * 4 = 1s
+        tx.send(CountdownCommand::Start(250)).await.unwrap();
+        // assert deadline is correctly set
+        assert!(deadline(&tx).await <= Instant::now() + Duration::from_secs(1));
+
+        // reset countdown in 1.5s
+        tx.send(CountdownCommand::Reset(
+            Instant::now() + Duration::from_millis(1500),
+        ))
+        .await
+        .unwrap();
+
+        // assert the table is closed after deadline is reached
+        assert!(table_engine.table_exists(ctx, &table_ref));
+        // spare 500ms for the task to close the table
+        tokio::time::sleep(Duration::from_millis(2000)).await;
+        assert!(!table_engine.table_exists(ctx, &table_ref));
+    }
+}

src/catalog/src/tables.rs

@@ -19,6 +19,7 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use common_catalog::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_TABLE_NAME};
+use common_telemetry::logging;
 use snafu::ResultExt;
 use table::metadata::TableId;
 use table::{Table, TableRef};
@@ -91,12 +92,21 @@ impl SystemCatalog {
         &self,
         request: &DeregisterTableRequest,
         table_id: TableId,
-    ) -> CatalogResult<bool> {
+    ) -> CatalogResult<()> {
         self.information_schema
             .system
             .delete(build_table_deletion_request(request, table_id))
             .await
-            .map(|x| x == 1)
+            .map(|x| {
+                if x != 1 {
+                    let table = common_catalog::format_full_table_name(
+                        &request.catalog,
+                        &request.schema,
+                        &request.table_name
+                    );
+                    logging::warn!("Failed to delete table record from information_schema, unexpected returned result: {x}, table: {table}");
+                }
+            })
             .with_context(|_| error::DeregisterTableSnafu {
                 request: request.clone(),
             })

src/catalog/tests/remote_catalog_tests.rs

@@ -19,20 +19,38 @@ mod tests {
     use std::assert_matches::assert_matches;
     use std::collections::HashSet;
     use std::sync::Arc;
+    use std::time::Duration;
 
     use catalog::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
     use catalog::remote::mock::{MockKvBackend, MockTableEngine};
+    use catalog::remote::region_alive_keeper::RegionAliveKeepers;
     use catalog::remote::{
         CachedMetaKvBackend, KvBackend, KvBackendRef, RemoteCatalogManager, RemoteCatalogProvider,
         RemoteSchemaProvider,
     };
     use catalog::{CatalogManager, RegisterTableRequest};
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
+    use common_meta::ident::TableIdent;
     use datatypes::schema::RawSchema;
     use futures_util::StreamExt;
     use table::engine::manager::{MemoryTableEngineManager, TableEngineManagerRef};
     use table::engine::{EngineContext, TableEngineRef};
     use table::requests::CreateTableRequest;
+    use table::test_util::EmptyTable;
+    use tokio::time::Instant;
+
+    struct TestingComponents {
+        kv_backend: KvBackendRef,
+        catalog_manager: Arc<RemoteCatalogManager>,
+        table_engine_manager: TableEngineManagerRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
+    }
+
+    impl TestingComponents {
+        fn table_engine(&self) -> TableEngineRef {
+            self.table_engine_manager.engine(MITO_ENGINE).unwrap()
+        }
+    }
 
     #[tokio::test]
     async fn test_backend() {
@@ -120,14 +138,7 @@ mod tests {
         assert!(ret.is_none());
     }
 
-    async fn prepare_components(
-        node_id: u64,
-    ) -> (
-        KvBackendRef,
-        TableEngineRef,
-        Arc<RemoteCatalogManager>,
-        TableEngineManagerRef,
-    ) {
+    async fn prepare_components(node_id: u64) -> TestingComponents {
         let cached_backend = Arc::new(CachedMetaKvBackend::wrap(
             Arc::new(MockKvBackend::default()),
         ));
@@ -135,26 +146,34 @@ mod tests {
         let table_engine = Arc::new(MockTableEngine::default());
         let engine_manager = Arc::new(MemoryTableEngineManager::alias(
             MITO_ENGINE.to_string(),
-            table_engine.clone(),
+            table_engine,
         ));
 
-        let catalog_manager =
-            RemoteCatalogManager::new(engine_manager.clone(), node_id, cached_backend.clone());
+        let region_alive_keepers = Arc::new(RegionAliveKeepers::new(engine_manager.clone(), 5000));
+
+        let catalog_manager = RemoteCatalogManager::new(
+            engine_manager.clone(),
+            node_id,
+            cached_backend.clone(),
+            region_alive_keepers.clone(),
+        );
         catalog_manager.start().await.unwrap();
 
-        (
-            cached_backend,
-            table_engine,
-            Arc::new(catalog_manager),
-            engine_manager as Arc<_>,
-        )
+        TestingComponents {
+            kv_backend: cached_backend,
+            catalog_manager: Arc::new(catalog_manager),
+            table_engine_manager: engine_manager,
+            region_alive_keepers,
+        }
     }
 
     #[tokio::test]
     async fn test_remote_catalog_default() {
         common_telemetry::init_default_ut_logging();
         let node_id = 42;
-        let (_, _, catalog_manager, _) = prepare_components(node_id).await;
+        let TestingComponents {
+            catalog_manager, ..
+        } = prepare_components(node_id).await;
         assert_eq!(
             vec![DEFAULT_CATALOG_NAME.to_string()],
             catalog_manager.catalog_names().await.unwrap()
@@ -175,14 +194,16 @@ mod tests {
     async fn test_remote_catalog_register_nonexistent() {
         common_telemetry::init_default_ut_logging();
         let node_id = 42;
-        let (_, table_engine, catalog_manager, _) = prepare_components(node_id).await;
+        let components = prepare_components(node_id).await;
+
         // register a new table with an nonexistent catalog
         let catalog_name = "nonexistent_catalog".to_string();
         let schema_name = "nonexistent_schema".to_string();
         let table_name = "fail_table".to_string();
         // this schema has no effect
         let table_schema = RawSchema::new(vec![]);
-        let table = table_engine
+        let table = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -208,7 +229,7 @@ mod tests {
             table_id: 1,
             table,
         };
-        let res = catalog_manager.register_table(reg_req).await;
+        let res = components.catalog_manager.register_table(reg_req).await;
 
         // because nonexistent_catalog does not exist yet.
         assert_matches!(
@@ -220,7 +241,8 @@ mod tests {
     #[tokio::test]
     async fn test_register_table() {
         let node_id = 42;
-        let (_, table_engine, catalog_manager, _) = prepare_components(node_id).await;
+        let components = prepare_components(node_id).await;
+        let catalog_manager = &components.catalog_manager;
         let default_catalog = catalog_manager
             .catalog(DEFAULT_CATALOG_NAME)
             .await
@@ -244,7 +266,8 @@ mod tests {
         let table_id = 1;
         // this schema has no effect
         let table_schema = RawSchema::new(vec![]);
-        let table = table_engine
+        let table = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -280,8 +303,10 @@ mod tests {
     #[tokio::test]
     async fn test_register_catalog_schema_table() {
         let node_id = 42;
-        let (backend, table_engine, catalog_manager, engine_manager) =
-            prepare_components(node_id).await;
+        let components = prepare_components(node_id).await;
+        let backend = &components.kv_backend;
+        let catalog_manager = components.catalog_manager.clone();
+        let engine_manager = components.table_engine_manager.clone();
 
         let catalog_name = "test_catalog".to_string();
         let schema_name = "nonexistent_schema".to_string();
@@ -290,6 +315,7 @@ mod tests {
             backend.clone(),
             engine_manager.clone(),
             node_id,
+            components.region_alive_keepers.clone(),
         ));
 
         // register catalog to catalog manager
@@ -303,7 +329,8 @@ mod tests {
             HashSet::from_iter(catalog_manager.catalog_names().await.unwrap().into_iter())
         );
 
-        let table_to_register = table_engine
+        let table_to_register = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -350,6 +377,7 @@ mod tests {
             node_id,
             engine_manager,
             backend.clone(),
+            components.region_alive_keepers.clone(),
         ));
 
         let prev = new_catalog
@@ -369,4 +397,94 @@ mod tests {
                 .collect()
         )
     }
+
+    #[tokio::test]
+    async fn test_register_table_before_and_after_region_alive_keeper_started() {
+        let components = prepare_components(42).await;
+        let catalog_manager = &components.catalog_manager;
+        let region_alive_keepers = &components.region_alive_keepers;
+
+        let table_before = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_before".to_string(),
+            table_id: 1,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_before.catalog.clone(),
+            schema: table_before.schema.clone(),
+            table_name: table_before.table.clone(),
+            table_id: table_before.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_before.table_id,
+                catalog_name: table_before.catalog.clone(),
+                schema_name: table_before.schema.clone(),
+                table_name: table_before.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(&table_before)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert region alive countdown is not started
+        assert!(deadline > far_future);
+
+        region_alive_keepers.start().await;
+
+        let table_after = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_after".to_string(),
+            table_id: 2,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_after.catalog.clone(),
+            schema: table_after.schema.clone(),
+            table_name: table_after.table.clone(),
+            table_id: table_after.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_after.table_id,
+                catalog_name: table_after.catalog.clone(),
+                schema_name: table_after.schema.clone(),
+                table_name: table_after.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(&table_after)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered after [RegionAliveKeepers] started
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+
+        let keeper = region_alive_keepers
+            .find_keeper(&table_before)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered before [RegionAliveKeepers] started, too
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+    }
 }

src/client/Cargo.toml

@@ -30,12 +30,13 @@ parking_lot = "0.12"
 prost.workspace = true
 rand.workspace = true
 snafu.workspace = true
+tokio-stream = { version = "0.1", features = ["net"] }
+tokio.workspace = true
 tonic.workspace = true
 
 [dev-dependencies]
 datanode = { path = "../datanode" }
 substrait = { path = "../common/substrait" }
-tokio.workspace = true
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 prost.workspace = true

src/client/src/client_manager.rs

@@ -31,7 +31,6 @@ pub struct DatanodeClients {
 
 impl Default for DatanodeClients {
     fn default() -> Self {
-        // TODO(LFC): Make this channel config configurable.
         let config = ChannelConfig::new().timeout(Duration::from_secs(8));
 
         Self {

src/client/src/database.rs

@@ -29,6 +29,9 @@ use common_telemetry::{logging, timer};
 use futures_util::{TryFutureExt, TryStreamExt};
 use prost::Message;
 use snafu::{ensure, ResultExt};
+use tokio::sync::mpsc::Sender;
+use tokio::sync::{mpsc, OnceCell};
+use tokio_stream::wrappers::ReceiverStream;
 
 use crate::error::{
     ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
@@ -47,6 +50,7 @@ pub struct Database {
     dbname: String,
 
     client: Client,
+    streaming_client: OnceCell<Sender<GreptimeRequest>>,
     ctx: FlightContext,
 }
 
@@ -58,6 +62,7 @@ impl Database {
             schema: schema.into(),
             dbname: "".to_string(),
             client,
+            streaming_client: OnceCell::new(),
             ctx: FlightContext::default(),
         }
     }
@@ -75,6 +80,7 @@ impl Database {
             schema: "".to_string(),
             dbname: dbname.into(),
             client,
+            streaming_client: OnceCell::new(),
             ctx: FlightContext::default(),
         }
     }
@@ -114,6 +120,22 @@ impl Database {
         self.handle(Request::Inserts(requests)).await
     }
 
+    pub async fn insert_to_stream(&self, requests: InsertRequests) -> Result<()> {
+        let streaming_client = self
+            .streaming_client
+            .get_or_try_init(|| self.client_stream())
+            .await?;
+
+        let request = self.to_rpc_request(Request::Inserts(requests));
+
+        streaming_client.send(request).await.map_err(|e| {
+            error::ClientStreamingSnafu {
+                err_msg: e.to_string(),
+            }
+            .build()
+        })
+    }
+
     pub async fn delete(&self, request: DeleteRequest) -> Result<u32> {
         let _timer = timer!(metrics::METRIC_GRPC_DELETE);
         self.handle(Request::Delete(request)).await
@@ -121,15 +143,7 @@ impl Database {
 
     async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = self.client.make_database_client()?.inner;
-        let request = GreptimeRequest {
-            header: Some(RequestHeader {
-                catalog: self.catalog.clone(),
-                schema: self.schema.clone(),
-                authorization: self.ctx.auth_header.clone(),
-                dbname: self.dbname.clone(),
-            }),
-            request: Some(request),
-        };
+        let request = self.to_rpc_request(request);
         let response = client
             .handle(request)
             .await?
@@ -142,6 +156,27 @@ impl Database {
         Ok(value)
     }
 
+    #[inline]
+    fn to_rpc_request(&self, request: Request) -> GreptimeRequest {
+        GreptimeRequest {
+            header: Some(RequestHeader {
+                catalog: self.catalog.clone(),
+                schema: self.schema.clone(),
+                authorization: self.ctx.auth_header.clone(),
+                dbname: self.dbname.clone(),
+            }),
+            request: Some(request),
+        }
+    }
+
+    async fn client_stream(&self) -> Result<Sender<GreptimeRequest>> {
+        let mut client = self.client.make_database_client()?.inner;
+        let (sender, receiver) = mpsc::channel::<GreptimeRequest>(65536);
+        let receiver = ReceiverStream::new(receiver);
+        client.handle_requests(receiver).await?;
+        Ok(sender)
+    }
+
     pub async fn sql(&self, sql: &str) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_SQL);
         self.do_get(Request::Query(QueryRequest {
@@ -212,22 +247,13 @@ impl Database {
     async fn do_get(&self, request: Request) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
         let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
-        let request = GreptimeRequest {
-            header: Some(RequestHeader {
-                catalog: self.catalog.clone(),
-                schema: self.schema.clone(),
-                authorization: self.ctx.auth_header.clone(),
-                dbname: self.dbname.clone(),
-            }),
-            request: Some(request),
-        };
+        let request = self.to_rpc_request(request);
         let request = Ticket {
             ticket: request.encode_to_vec().into(),
         };
 
         let mut client = self.client.make_flight_client()?;
 
-        // TODO(LFC): Streaming get flight data.
         let flight_data: Vec<FlightData> = client
             .mut_inner()
             .do_get(request)

src/client/src/error.rs

@@ -34,13 +34,13 @@ pub enum Error {
 
     #[snafu(display("Failed to convert FlightData, source: {}", source))]
     ConvertFlightData {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc::Error,
     },
 
     #[snafu(display("Column datatype error, source: {}", source))]
     ColumnDataType {
-        #[snafu(backtrace)]
+        location: Location,
         source: api::error::Error,
     },
 
@@ -57,7 +57,7 @@ pub enum Error {
     ))]
     CreateChannel {
         addr: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc::error::Error,
     },
 
@@ -67,6 +67,9 @@ pub enum Error {
 
     #[snafu(display("Illegal Database response: {err_msg}"))]
     IllegalDatabaseResponse { err_msg: String },
+
+    #[snafu(display("Failed to send request with streaming: {}", err_msg))]
+    ClientStreaming { err_msg: String, location: Location },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -77,11 +80,12 @@ impl ErrorExt for Error {
             Error::IllegalFlightMessages { .. }
             | Error::ColumnDataType { .. }
             | Error::MissingField { .. }
-            | Error::IllegalDatabaseResponse { .. } => StatusCode::Internal,
+            | Error::IllegalDatabaseResponse { .. }
+            | Error::ClientStreaming { .. } => StatusCode::Internal,
 
             Error::Server { code, .. } => *code,
             Error::FlightGet { source, .. } => source.status_code(),
-            Error::CreateChannel { source, .. } | Error::ConvertFlightData { source } => {
+            Error::CreateChannel { source, .. } | Error::ConvertFlightData { source, .. } => {
                 source.status_code()
             }
             Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected,

src/cmd/Cargo.toml

@@ -10,7 +10,6 @@ name = "greptime"
 path = "src/bin/greptime.rs"
 
 [features]
-mem-prof = ["tikv-jemallocator", "tikv-jemalloc-ctl"]
 tokio-console = ["common-telemetry/tokio-console"]
 
 [dependencies]
@@ -42,8 +41,7 @@ servers = { path = "../servers" }
 session = { path = "../session" }
 snafu.workspace = true
 substrait = { path = "../common/substrait" }
-tikv-jemalloc-ctl = { version = "0.5", optional = true }
-tikv-jemallocator = { version = "0.5", optional = true }
+tikv-jemallocator = "0.5"
 tokio.workspace = true
 
 [dev-dependencies]
@@ -54,4 +52,4 @@ serde.workspace = true
 toml = "0.5"
 
 [build-dependencies]
-build-data = "0.1.3"
+build-data = "0.1.4"

src/cmd/src/bin/greptime.rs

@@ -180,15 +180,19 @@ fn full_version() -> &'static str {
     )
 }
 
-#[cfg(feature = "mem-prof")]
+fn log_env_flags() {
+    info!("command line arguments");
+    for argument in std::env::args() {
+        info!("argument: {}", argument);
+    }
+}
+
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
 #[tokio::main]
 async fn main() -> Result<()> {
     let cmd = Command::parse();
-    // TODO(dennis):
-    // 1. adds ip/port to app
     let app_name = &cmd.subcmd.to_string();
 
     let opts = cmd.load_options()?;
@@ -205,6 +209,14 @@ async fn main() -> Result<()> {
     // Report app version as gauge.
     gauge!("app_version", 1.0, "short_version" => short_version(), "version" => full_version());
 
+    // Log version and argument flags.
+    info!(
+        "short_version: {}, full_version: {}",
+        short_version(),
+        full_version()
+    );
+    log_env_flags();
+
     let mut app = cmd.build(opts).await?;
 
     tokio::select! {

src/cmd/src/datanode.rs

@@ -84,8 +84,6 @@ struct StartCommand {
     rpc_addr: Option<String>,
     #[clap(long)]
     rpc_hostname: Option<String>,
-    #[clap(long)]
-    mysql_addr: Option<String>,
     #[clap(long, multiple = true, value_delimiter = ',')]
     metasrv_addr: Option<Vec<String>>,
     #[clap(short, long)]
@@ -126,10 +124,6 @@ impl StartCommand {
             opts.rpc_hostname = self.rpc_hostname.clone();
         }
 
-        if let Some(addr) = &self.mysql_addr {
-            opts.mysql_addr = addr.clone();
-        }
-
         if let Some(node_id) = self.node_id {
             opts.node_id = Some(node_id);
         }
@@ -205,8 +199,6 @@ mod tests {
             rpc_addr = "127.0.0.1:3001"
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
-            mysql_addr = "127.0.0.1:4406"
-            mysql_runtime_size = 2
 
             [meta_client_options]
             metasrv_addrs = ["127.0.0.1:3002"]
@@ -252,8 +244,6 @@ mod tests {
             cmd.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
 
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
-        assert_eq!("127.0.0.1:4406".to_string(), options.mysql_addr);
-        assert_eq!(2, options.mysql_runtime_size);
         assert_eq!(Some(42), options.node_id);
 
         assert_eq!("/other/wal", options.wal.dir.unwrap());
@@ -368,8 +358,6 @@ mod tests {
             rpc_addr = "127.0.0.1:3001"
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
-            mysql_addr = "127.0.0.1:4406"
-            mysql_runtime_size = 2
 
             [meta_client_options]
             timeout_millis = 3000

src/cmd/src/error.rs

@@ -24,43 +24,43 @@ use snafu::Location;
 pub enum Error {
     #[snafu(display("Failed to start datanode, source: {}", source))]
     StartDatanode {
-        #[snafu(backtrace)]
+        location: Location,
         source: datanode::error::Error,
     },
 
     #[snafu(display("Failed to shutdown datanode, source: {}", source))]
     ShutdownDatanode {
-        #[snafu(backtrace)]
+        location: Location,
         source: datanode::error::Error,
     },
 
     #[snafu(display("Failed to start frontend, source: {}", source))]
     StartFrontend {
-        #[snafu(backtrace)]
+        location: Location,
         source: frontend::error::Error,
     },
 
     #[snafu(display("Failed to shutdown frontend, source: {}", source))]
     ShutdownFrontend {
-        #[snafu(backtrace)]
+        location: Location,
         source: frontend::error::Error,
     },
 
     #[snafu(display("Failed to build meta server, source: {}", source))]
     BuildMetaServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_srv::error::Error,
     },
 
     #[snafu(display("Failed to start meta server, source: {}", source))]
     StartMetaServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_srv::error::Error,
     },
 
     #[snafu(display("Failed to shutdown meta server, source: {}", source))]
     ShutdownMetaServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_srv::error::Error,
     },
 
@@ -72,14 +72,14 @@ pub enum Error {
 
     #[snafu(display("Illegal auth config: {}", source))]
     IllegalAuthConfig {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::auth::Error,
     },
 
     #[snafu(display("Unsupported selector type, {} source: {}", selector_type, source))]
     UnsupportedSelectorType {
         selector_type: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_srv::error::Error,
     },
 
@@ -101,44 +101,44 @@ pub enum Error {
     #[snafu(display("Failed to request database, sql: {sql}, source: {source}"))]
     RequestDatabase {
         sql: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: client::Error,
     },
 
     #[snafu(display("Failed to collect RecordBatches, source: {source}"))]
     CollectRecordBatches {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
     #[snafu(display("Failed to pretty print Recordbatches, source: {source}"))]
     PrettyPrintRecordBatches {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
     #[snafu(display("Failed to start Meta client, source: {}", source))]
     StartMetaClient {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_client::error::Error,
     },
 
     #[snafu(display("Failed to parse SQL: {}, source: {}", sql, source))]
     ParseSql {
         sql: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to plan statement, source: {}", source))]
     PlanStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to encode logical plan in substrait, source: {}", source))]
     SubstraitEncodeLogicalPlan {
-        #[snafu(backtrace)]
+        location: Location,
         source: substrait::error::Error,
     },
 
@@ -150,7 +150,7 @@ pub enum Error {
 
     #[snafu(display("Failed to start catalog manager, source: {}", source))]
     StartCatalogManager {
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 }
@@ -160,13 +160,13 @@ pub type Result<T> = std::result::Result<T, Error>;
 impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
-            Error::StartDatanode { source } => source.status_code(),
-            Error::StartFrontend { source } => source.status_code(),
-            Error::ShutdownDatanode { source } => source.status_code(),
-            Error::ShutdownFrontend { source } => source.status_code(),
-            Error::StartMetaServer { source } => source.status_code(),
-            Error::ShutdownMetaServer { source } => source.status_code(),
-            Error::BuildMetaServer { source } => source.status_code(),
+            Error::StartDatanode { source, .. } => source.status_code(),
+            Error::StartFrontend { source, .. } => source.status_code(),
+            Error::ShutdownDatanode { source, .. } => source.status_code(),
+            Error::ShutdownFrontend { source, .. } => source.status_code(),
+            Error::StartMetaServer { source, .. } => source.status_code(),
+            Error::ShutdownMetaServer { source, .. } => source.status_code(),
+            Error::BuildMetaServer { source, .. } => source.status_code(),
             Error::UnsupportedSelectorType { source, .. } => source.status_code(),
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
@@ -175,15 +175,14 @@ impl ErrorExt for Error {
             | Error::IllegalAuthConfig { .. } => StatusCode::InvalidArguments,
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
             Error::RequestDatabase { source, .. } => source.status_code(),
-            Error::CollectRecordBatches { source } | Error::PrettyPrintRecordBatches { source } => {
+            Error::CollectRecordBatches { source, .. }
+            | Error::PrettyPrintRecordBatches { source, .. } => source.status_code(),
+            Error::StartMetaClient { source, .. } => source.status_code(),
+            Error::ParseSql { source, .. } | Error::PlanStatement { source, .. } => {
                 source.status_code()
             }
-            Error::StartMetaClient { source } => source.status_code(),
-            Error::ParseSql { source, .. } | Error::PlanStatement { source } => {
-                source.status_code()
-            }
-            Error::SubstraitEncodeLogicalPlan { source } => source.status_code(),
-            Error::StartCatalogManager { source } => source.status_code(),
+            Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
+            Error::StartCatalogManager { source, .. } => source.status_code(),
         }
     }
 

src/cmd/src/metasrv.rs

@@ -93,6 +93,8 @@ struct StartCommand {
     #[clap(long)]
     use_memory_store: bool,
     #[clap(long)]
+    disable_region_failover: bool,
+    #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
     http_timeout: Option<u64>,
@@ -134,9 +136,9 @@ impl StartCommand {
                 .context(error::UnsupportedSelectorTypeSnafu { selector_type })?;
         }
 
-        if self.use_memory_store {
-            opts.use_memory_store = true;
-        }
+        opts.use_memory_store = self.use_memory_store;
+
+        opts.disable_region_failover = self.disable_region_failover;
 
         if let Some(http_addr) = &self.http_addr {
             opts.http_opts.addr = http_addr.clone();

src/common/datasource/src/buffered_writer.rs

@@ -12,24 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::future::Future;
+
 use arrow::record_batch::RecordBatch;
 use async_trait::async_trait;
 use datafusion::parquet::format::FileMetaData;
-use object_store::Writer;
 use snafu::{OptionExt, ResultExt};
 use tokio::io::{AsyncWrite, AsyncWriteExt};
-use tokio_util::compat::Compat;
 
 use crate::error::{self, Result};
 use crate::share_buffer::SharedBuffer;
 
-pub struct BufferedWriter<T, U> {
-    writer: T,
-    /// None stands for [`BufferedWriter`] closed.
+pub struct LazyBufferedWriter<T, U, F> {
+    path: String,
+    writer_factory: F,
+    writer: Option<T>,
+    /// None stands for [`LazyBufferedWriter`] closed.
     encoder: Option<U>,
     buffer: SharedBuffer,
+    rows_written: usize,
     bytes_written: u64,
-    flushed: bool,
     threshold: usize,
 }
 
@@ -42,58 +44,79 @@ pub trait ArrowWriterCloser {
     async fn close(mut self) -> Result<FileMetaData>;
 }
 
-pub type DefaultBufferedWriter<E> = BufferedWriter<Compat<Writer>, E>;
-
-impl<T: AsyncWrite + Send + Unpin, U: DfRecordBatchEncoder + ArrowWriterCloser>
-    BufferedWriter<T, U>
+impl<
+        T: AsyncWrite + Send + Unpin,
+        U: DfRecordBatchEncoder + ArrowWriterCloser,
+        F: FnMut(String) -> Fut,
+        Fut: Future<Output = Result<T>>,
+    > LazyBufferedWriter<T, U, F>
 {
+    /// Closes `LazyBufferedWriter` and optionally flushes all data to underlying storage
+    /// if any row's been written.
     pub async fn close_with_arrow_writer(mut self) -> Result<(FileMetaData, u64)> {
         let encoder = self
             .encoder
             .take()
             .context(error::BufferedWriterClosedSnafu)?;
         let metadata = encoder.close().await?;
-        let written = self.try_flush(true).await?;
+
+        // Use `rows_written` to keep a track of if any rows have been written.
+        // If no row's been written, then we can simply close the underlying
+        // writer without flush so that no file will be actually created.
+        if self.rows_written != 0 {
+            self.bytes_written += self.try_flush(true).await?;
+        }
         // It's important to shut down! flushes all pending writes
-        self.close().await?;
-        Ok((metadata, written))
+        self.close_inner_writer().await?;
+        Ok((metadata, self.bytes_written))
     }
 }
 
-impl<T: AsyncWrite + Send + Unpin, U: DfRecordBatchEncoder> BufferedWriter<T, U> {
-    pub async fn close(&mut self) -> Result<()> {
-        self.writer.shutdown().await.context(error::AsyncWriteSnafu)
+impl<
+        T: AsyncWrite + Send + Unpin,
+        U: DfRecordBatchEncoder,
+        F: FnMut(String) -> Fut,
+        Fut: Future<Output = Result<T>>,
+    > LazyBufferedWriter<T, U, F>
+{
+    /// Closes the writer without flushing the buffer data.
+    pub async fn close_inner_writer(&mut self) -> Result<()> {
+        if let Some(writer) = &mut self.writer {
+            writer.shutdown().await.context(error::AsyncWriteSnafu)?;
+        }
+        Ok(())
     }
 
-    pub fn new(threshold: usize, buffer: SharedBuffer, encoder: U, writer: T) -> Self {
+    pub fn new(
+        threshold: usize,
+        buffer: SharedBuffer,
+        encoder: U,
+        path: impl AsRef<str>,
+        writer_factory: F,
+    ) -> Self {
         Self {
+            path: path.as_ref().to_string(),
             threshold,
-            writer,
             encoder: Some(encoder),
             buffer,
+            rows_written: 0,
             bytes_written: 0,
-            flushed: false,
+            writer_factory,
+            writer: None,
         }
     }
 
-    pub fn bytes_written(&self) -> u64 {
-        self.bytes_written
-    }
-
     pub async fn write(&mut self, batch: &RecordBatch) -> Result<()> {
         let encoder = self
             .encoder
             .as_mut()
             .context(error::BufferedWriterClosedSnafu)?;
         encoder.write(batch)?;
-        self.try_flush(false).await?;
+        self.rows_written += batch.num_rows();
+        self.bytes_written += self.try_flush(false).await?;
         Ok(())
     }
 
-    pub fn flushed(&self) -> bool {
-        self.flushed
-    }
-
     pub async fn try_flush(&mut self, all: bool) -> Result<u64> {
         let mut bytes_written: u64 = 0;
 
@@ -106,7 +129,8 @@ impl<T: AsyncWrite + Send + Unpin, U: DfRecordBatchEncoder> BufferedWriter<T, U>
             };
             let size = chunk.len();
 
-            self.writer
+            self.maybe_init_writer()
+                .await?
                 .write_all(&chunk)
                 .await
                 .context(error::AsyncWriteSnafu)?;
@@ -117,22 +141,27 @@ impl<T: AsyncWrite + Send + Unpin, U: DfRecordBatchEncoder> BufferedWriter<T, U>
         if all {
             bytes_written += self.try_flush_all().await?;
         }
-
-        self.flushed = bytes_written > 0;
-        self.bytes_written += bytes_written;
-
         Ok(bytes_written)
     }
 
+    /// Only initiates underlying file writer when rows have been written.
+    async fn maybe_init_writer(&mut self) -> Result<&mut T> {
+        if let Some(ref mut writer) = self.writer {
+            Ok(writer)
+        } else {
+            let writer = (self.writer_factory)(self.path.clone()).await?;
+            Ok(self.writer.insert(writer))
+        }
+    }
+
     async fn try_flush_all(&mut self) -> Result<u64> {
         let remain = self.buffer.buffer.lock().unwrap().split();
         let size = remain.len();
-
-        self.writer
+        self.maybe_init_writer()
+            .await?
             .write_all(&remain)
             .await
             .context(error::AsyncWriteSnafu)?;
-
         Ok(size as u64)
     }
 }

src/common/datasource/src/file_format.rs

@@ -35,12 +35,11 @@ use datafusion::physical_plan::SendableRecordBatchStream;
 use futures::StreamExt;
 use object_store::ObjectStore;
 use snafu::ResultExt;
-use tokio_util::compat::FuturesAsyncWriteCompatExt;
 
 use self::csv::CsvFormat;
 use self::json::JsonFormat;
 use self::parquet::ParquetFormat;
-use crate::buffered_writer::{BufferedWriter, DfRecordBatchEncoder};
+use crate::buffered_writer::{DfRecordBatchEncoder, LazyBufferedWriter};
 use crate::compression::CompressionType;
 use crate::error::{self, Result};
 use crate::share_buffer::SharedBuffer;
@@ -59,6 +58,16 @@ pub enum Format {
     Parquet(ParquetFormat),
 }
 
+impl Format {
+    pub fn suffix(&self) -> &'static str {
+        match self {
+            Format::Csv(_) => ".csv",
+            Format::Json(_) => ".json",
+            Format::Parquet(_) => ".parquet",
+        }
+    }
+}
+
 impl TryFrom<&HashMap<String, String>> for Format {
     type Error = error::Error;
 
@@ -181,15 +190,14 @@ pub async fn stream_to_file<T: DfRecordBatchEncoder, U: Fn(SharedBuffer) -> T>(
     threshold: usize,
     encoder_factory: U,
 ) -> Result<usize> {
-    let writer = store
-        .writer(path)
-        .await
-        .context(error::WriteObjectSnafu { path })?
-        .compat_write();
-
     let buffer = SharedBuffer::with_capacity(threshold);
     let encoder = encoder_factory(buffer.clone());
-    let mut writer = BufferedWriter::new(threshold, buffer, encoder, writer);
+    let mut writer = LazyBufferedWriter::new(threshold, buffer, encoder, path, |path| async {
+        store
+            .writer(&path)
+            .await
+            .context(error::WriteObjectSnafu { path })
+    });
 
     let mut rows = 0;
 
@@ -201,8 +209,7 @@ pub async fn stream_to_file<T: DfRecordBatchEncoder, U: Fn(SharedBuffer) -> T>(
 
     // Flushes all pending writes
     writer.try_flush(true).await?;
-
-    writer.close().await?;
+    writer.close_inner_writer().await?;
 
     Ok(rows)
 }

src/common/function/src/scalars/timestamp/to_unixtime.rs

@@ -16,13 +16,16 @@ use std::fmt;
 use std::str::FromStr;
 use std::sync::Arc;
 
-use common_query::error::{self, Result, UnsupportedInputDataTypeSnafu};
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
 use common_query::prelude::{Signature, Volatility};
 use common_time::timestamp::TimeUnit;
 use common_time::Timestamp;
 use datatypes::prelude::ConcreteDataType;
-use datatypes::types::StringType;
-use datatypes::vectors::{Int64Vector, StringVector, Vector, VectorRef};
+use datatypes::types::TimestampType;
+use datatypes::vectors::{
+    Int64Vector, StringVector, TimestampMicrosecondVector, TimestampMillisecondVector,
+    TimestampNanosecondVector, TimestampSecondVector, Vector, VectorRef,
+};
 use snafu::ensure;
 
 use crate::scalars::function::{Function, FunctionContext};
@@ -42,18 +45,33 @@ fn convert_to_seconds(arg: &str) -> Option<i64> {
     }
 }
 
+fn process_vector(vector: &dyn Vector) -> Vec<Option<i64>> {
+    (0..vector.len())
+        .map(|i| paste::expr!((vector.get(i)).as_timestamp().map(|ts| ts.value())))
+        .collect::<Vec<Option<i64>>>()
+}
+
 impl Function for ToUnixtimeFunction {
     fn name(&self) -> &str {
         NAME
     }
 
     fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::timestamp_second_datatype())
+        Ok(ConcreteDataType::int64_datatype())
     }
 
     fn signature(&self) -> Signature {
-        Signature::exact(
-            vec![ConcreteDataType::String(StringType)],
+        Signature::uniform(
+            1,
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::int32_datatype(),
+                ConcreteDataType::int64_datatype(),
+                ConcreteDataType::timestamp_second_datatype(),
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                ConcreteDataType::timestamp_microsecond_datatype(),
+                ConcreteDataType::timestamp_nanosecond_datatype(),
+            ],
             Volatility::Immutable,
         )
     }
@@ -61,7 +79,7 @@ impl Function for ToUnixtimeFunction {
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
         ensure!(
             columns.len() == 1,
-            error::InvalidFuncArgsSnafu {
+            InvalidFuncArgsSnafu {
                 err_msg: format!(
                     "The length of the args is not correct, expect exactly one, have: {}",
                     columns.len()
@@ -79,6 +97,42 @@ impl Function for ToUnixtimeFunction {
                         .collect::<Vec<_>>(),
                 )))
             }
+            ConcreteDataType::Int64(_) | ConcreteDataType::Int32(_) => {
+                let array = columns[0].to_arrow_array();
+                Ok(Arc::new(Int64Vector::try_from_arrow_array(&array).unwrap()))
+            }
+            ConcreteDataType::Timestamp(ts) => {
+                let array = columns[0].to_arrow_array();
+                let value = match ts {
+                    TimestampType::Second(_) => {
+                        let vector = paste::expr!(TimestampSecondVector::try_from_arrow_array(
+                            array
+                        )
+                        .unwrap());
+                        process_vector(&vector)
+                    }
+                    TimestampType::Millisecond(_) => {
+                        let vector = paste::expr!(
+                            TimestampMillisecondVector::try_from_arrow_array(array).unwrap()
+                        );
+                        process_vector(&vector)
+                    }
+                    TimestampType::Microsecond(_) => {
+                        let vector = paste::expr!(
+                            TimestampMicrosecondVector::try_from_arrow_array(array).unwrap()
+                        );
+                        process_vector(&vector)
+                    }
+                    TimestampType::Nanosecond(_) => {
+                        let vector = paste::expr!(TimestampNanosecondVector::try_from_arrow_array(
+                            array
+                        )
+                        .unwrap());
+                        process_vector(&vector)
+                    }
+                };
+                Ok(Arc::new(Int64Vector::from(value)))
+            }
             _ => UnsupportedInputDataTypeSnafu {
                 function: NAME,
                 datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
@@ -97,28 +151,37 @@ impl fmt::Display for ToUnixtimeFunction {
 #[cfg(test)]
 mod tests {
     use common_query::prelude::TypeSignature;
-    use datatypes::prelude::ConcreteDataType;
-    use datatypes::types::StringType;
+    use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder};
+    use datatypes::scalars::ScalarVector;
+    use datatypes::timestamp::TimestampSecond;
     use datatypes::value::Value;
-    use datatypes::vectors::StringVector;
+    use datatypes::vectors::{StringVector, TimestampSecondVector};
 
     use super::{ToUnixtimeFunction, *};
     use crate::scalars::Function;
 
     #[test]
-    fn test_to_unixtime() {
+    fn test_string_to_unixtime() {
         let f = ToUnixtimeFunction::default();
         assert_eq!("to_unixtime", f.name());
         assert_eq!(
-            ConcreteDataType::timestamp_second_datatype(),
+            ConcreteDataType::int64_datatype(),
             f.return_type(&[]).unwrap()
         );
 
         assert!(matches!(f.signature(),
-                         Signature {
-                            type_signature: TypeSignature::Exact(valid_types),
-                            volatility: Volatility::Immutable
-                         } if  valid_types == vec![ConcreteDataType::String(StringType)]
+        Signature {
+                type_signature: TypeSignature::Uniform(1, valid_types),
+                volatility: Volatility::Immutable
+            } if  valid_types == vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::int32_datatype(),
+                ConcreteDataType::int64_datatype(),
+                ConcreteDataType::timestamp_second_datatype(),
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                ConcreteDataType::timestamp_microsecond_datatype(),
+                ConcreteDataType::timestamp_nanosecond_datatype(),
+            ]
         ));
 
         let times = vec![
@@ -145,4 +208,106 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_int_to_unixtime() {
+        let f = ToUnixtimeFunction::default();
+        assert_eq!("to_unixtime", f.name());
+        assert_eq!(
+            ConcreteDataType::int64_datatype(),
+            f.return_type(&[]).unwrap()
+        );
+
+        assert!(matches!(f.signature(),
+        Signature {
+                type_signature: TypeSignature::Uniform(1, valid_types),
+                volatility: Volatility::Immutable
+            } if  valid_types == vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::int32_datatype(),
+                ConcreteDataType::int64_datatype(),
+                ConcreteDataType::timestamp_second_datatype(),
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                ConcreteDataType::timestamp_microsecond_datatype(),
+                ConcreteDataType::timestamp_nanosecond_datatype(),
+            ]
+        ));
+
+        let times = vec![Some(3_i64), None, Some(5_i64), None];
+        let results = vec![Some(3), None, Some(5), None];
+        let args: Vec<VectorRef> = vec![Arc::new(Int64Vector::from(times.clone()))];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+        assert_eq!(4, vector.len());
+        for (i, _t) in times.iter().enumerate() {
+            let v = vector.get(i);
+            if i == 1 || i == 3 {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Int64(ts) => {
+                    assert_eq!(ts, (*results.get(i).unwrap()).unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_timestamp_to_unixtime() {
+        let f = ToUnixtimeFunction::default();
+        assert_eq!("to_unixtime", f.name());
+        assert_eq!(
+            ConcreteDataType::int64_datatype(),
+            f.return_type(&[]).unwrap()
+        );
+
+        assert!(matches!(f.signature(),
+        Signature {
+                type_signature: TypeSignature::Uniform(1, valid_types),
+                volatility: Volatility::Immutable
+            } if  valid_types == vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::int32_datatype(),
+                ConcreteDataType::int64_datatype(),
+                ConcreteDataType::timestamp_second_datatype(),
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                ConcreteDataType::timestamp_microsecond_datatype(),
+                ConcreteDataType::timestamp_nanosecond_datatype(),
+            ]
+        ));
+
+        let times: Vec<Option<TimestampSecond>> = vec![
+            Some(TimestampSecond::new(123)),
+            None,
+            Some(TimestampSecond::new(42)),
+            None,
+        ];
+        let results = vec![Some(123), None, Some(42), None];
+        let ts_vector: TimestampSecondVector = build_vector_from_slice(&times);
+        let args: Vec<VectorRef> = vec![Arc::new(ts_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+        assert_eq!(4, vector.len());
+        for (i, _t) in times.iter().enumerate() {
+            let v = vector.get(i);
+            if i == 1 || i == 3 {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Int64(ts) => {
+                    assert_eq!(ts, (*results.get(i).unwrap()).unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    fn build_vector_from_slice<T: ScalarVector>(items: &[Option<T::RefItem<'_>>]) -> T {
+        let mut builder = T::Builder::with_capacity(items.len());
+        for item in items {
+            builder.push(*item);
+        }
+        builder.finish()
+    }
 }

src/common/grpc-expr/src/delete.rs

@@ -16,7 +16,6 @@ use std::collections::HashMap;
 
 use api::helper::ColumnDataTypeWrapper;
 use api::v1::{Column, DeleteRequest as GrpcDeleteRequest};
-use datatypes::data_type::DataType;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{ensure, ResultExt};
 use table::requests::DeleteRequest;
@@ -41,14 +40,11 @@ pub fn to_table_delete_request(request: GrpcDeleteRequest) -> Result<DeleteReque
         let datatype: ConcreteDataType = ColumnDataTypeWrapper::try_new(datatype)
             .context(ColumnDataTypeSnafu)?
             .into();
-
-        let vector_builder = &mut datatype.create_mutable_vector(row_count);
-
-        add_values_to_builder(vector_builder, values, row_count, null_mask)?;
+        let vector = add_values_to_builder(datatype, values, row_count, null_mask)?;
 
         ensure!(
             key_column_values
-                .insert(column_name.clone(), vector_builder.to_vector())
+                .insert(column_name.clone(), vector)
                 .is_none(),
             IllegalDeleteRequestSnafu {
                 reason: format!("Duplicated column '{column_name}' in delete request.")

src/common/grpc-expr/src/error.rs

@@ -32,7 +32,7 @@ pub enum Error {
 
     #[snafu(display("Column datatype error, source: {}", source))]
     ColumnDataType {
-        #[snafu(backtrace)]
+        location: Location,
         source: api::error::Error,
     },
 
@@ -54,7 +54,7 @@ pub enum Error {
     InvalidColumnProto { err_msg: String, location: Location },
     #[snafu(display("Failed to create vector, source: {}", source))]
     CreateVector {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -68,13 +68,13 @@ pub enum Error {
     ))]
     InvalidColumnDef {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: api::error::Error,
     },
 
     #[snafu(display("Unrecognized table option: {}", source))]
     UnrecognizedTableOption {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 

src/common/grpc-expr/src/insert.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
 
 use api::helper::ColumnDataTypeWrapper;
 use api::v1::column::{SemanticType, Values};
@@ -25,10 +26,16 @@ use common_time::timestamp::Timestamp;
 use common_time::{Date, DateTime};
 use datatypes::data_type::{ConcreteDataType, DataType};
 use datatypes::prelude::{ValueRef, VectorRef};
+use datatypes::scalars::ScalarVector;
 use datatypes::schema::SchemaRef;
-use datatypes::types::TimestampType;
+use datatypes::types::{Int16Type, Int8Type, TimestampType, UInt16Type, UInt8Type};
 use datatypes::value::Value;
-use datatypes::vectors::MutableVector;
+use datatypes::vectors::{
+    BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
+    Int32Vector, Int64Vector, PrimitiveVector, StringVector, TimestampMicrosecondVector,
+    TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, UInt32Vector,
+    UInt64Vector,
+};
 use snafu::{ensure, OptionExt, ResultExt};
 use table::metadata::TableId;
 use table::requests::InsertRequest;
@@ -287,15 +294,10 @@ pub fn to_table_insert_request(
         let datatype: ConcreteDataType = ColumnDataTypeWrapper::try_new(datatype)
             .context(ColumnDataTypeSnafu)?
             .into();
-
-        let vector_builder = &mut datatype.create_mutable_vector(row_count);
-
-        add_values_to_builder(vector_builder, values, row_count, null_mask)?;
+        let vector = add_values_to_builder(datatype, values, row_count, null_mask)?;
 
         ensure!(
-            columns_values
-                .insert(column_name.clone(), vector_builder.to_vector())
-                .is_none(),
+            columns_values.insert(column_name.clone(), vector).is_none(),
             ColumnAlreadyExistsSnafu {
                 column: column_name
             }
@@ -312,28 +314,16 @@ pub fn to_table_insert_request(
 }
 
 pub(crate) fn add_values_to_builder(
-    builder: &mut Box<dyn MutableVector>,
+    data_type: ConcreteDataType,
     values: Values,
     row_count: usize,
     null_mask: Vec<u8>,
-) -> Result<()> {
-    let data_type = builder.data_type();
-    let values = convert_values(&data_type, values);
-
+) -> Result<VectorRef> {
     if null_mask.is_empty() {
-        ensure!(
-            values.len() == row_count,
-            UnexpectedValuesLengthSnafu {
-                reason: "If null_mask is empty, the length of values must be equal to row_count."
-            }
-        );
-
-        values.iter().try_for_each(|value| {
-            builder
-                .try_push_value_ref(value.as_value_ref())
-                .context(CreateVectorSnafu)
-        })?;
+        Ok(values_to_vector(&data_type, values))
     } else {
+        let builder = &mut data_type.create_mutable_vector(row_count);
+        let values = convert_values(&data_type, values);
         let null_mask = BitVec::from_vec(null_mask);
         ensure!(
             null_mask.count_ones() + values.len() == row_count,
@@ -354,8 +344,53 @@ pub(crate) fn add_values_to_builder(
                 }
             }
         }
+        Ok(builder.to_vector())
+    }
+}
+
+fn values_to_vector(data_type: &ConcreteDataType, values: Values) -> VectorRef {
+    match data_type {
+        ConcreteDataType::Boolean(_) => Arc::new(BooleanVector::from(values.bool_values)),
+        ConcreteDataType::Int8(_) => Arc::new(PrimitiveVector::<Int8Type>::from_iter_values(
+            values.i8_values.into_iter().map(|x| x as i8),
+        )),
+        ConcreteDataType::Int16(_) => Arc::new(PrimitiveVector::<Int16Type>::from_iter_values(
+            values.i16_values.into_iter().map(|x| x as i16),
+        )),
+        ConcreteDataType::Int32(_) => Arc::new(Int32Vector::from_vec(values.i32_values)),
+        ConcreteDataType::Int64(_) => Arc::new(Int64Vector::from_vec(values.i64_values)),
+        ConcreteDataType::UInt8(_) => Arc::new(PrimitiveVector::<UInt8Type>::from_iter_values(
+            values.u8_values.into_iter().map(|x| x as u8),
+        )),
+        ConcreteDataType::UInt16(_) => Arc::new(PrimitiveVector::<UInt16Type>::from_iter_values(
+            values.u16_values.into_iter().map(|x| x as u16),
+        )),
+        ConcreteDataType::UInt32(_) => Arc::new(UInt32Vector::from_vec(values.u32_values)),
+        ConcreteDataType::UInt64(_) => Arc::new(UInt64Vector::from_vec(values.u64_values)),
+        ConcreteDataType::Float32(_) => Arc::new(Float32Vector::from_vec(values.f32_values)),
+        ConcreteDataType::Float64(_) => Arc::new(Float64Vector::from_vec(values.f64_values)),
+        ConcreteDataType::Binary(_) => Arc::new(BinaryVector::from(values.binary_values)),
+        ConcreteDataType::String(_) => Arc::new(StringVector::from_vec(values.string_values)),
+        ConcreteDataType::Date(_) => Arc::new(DateVector::from_vec(values.date_values)),
+        ConcreteDataType::DateTime(_) => Arc::new(DateTimeVector::from_vec(values.datetime_values)),
+        ConcreteDataType::Timestamp(unit) => match unit {
+            TimestampType::Second(_) => {
+                Arc::new(TimestampSecondVector::from_vec(values.ts_second_values))
+            }
+            TimestampType::Millisecond(_) => Arc::new(TimestampMillisecondVector::from_vec(
+                values.ts_millisecond_values,
+            )),
+            TimestampType::Microsecond(_) => Arc::new(TimestampMicrosecondVector::from_vec(
+                values.ts_microsecond_values,
+            )),
+            TimestampType::Nanosecond(_) => Arc::new(TimestampNanosecondVector::from_vec(
+                values.ts_nanosecond_values,
+            )),
+        },
+        ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
+            unreachable!()
+        }
     }
-    Ok(())
 }
 
 fn convert_values(data_type: &ConcreteDataType, values: Values) -> Vec<Value> {

src/common/grpc/src/error.rs

@@ -53,7 +53,7 @@ pub enum Error {
 
     #[snafu(display("Failed to create RecordBatch, source: {}", source))]
     CreateRecordBatch {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
@@ -71,7 +71,7 @@ pub enum Error {
 
     #[snafu(display("Failed to convert Arrow Schema, source: {}", source))]
     ConvertArrowSchema {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 }
@@ -88,8 +88,8 @@ impl ErrorExt for Error {
             | Error::Conversion { .. }
             | Error::DecodeFlightData { .. } => StatusCode::Internal,
 
-            Error::CreateRecordBatch { source } => source.status_code(),
-            Error::ConvertArrowSchema { source } => source.status_code(),
+            Error::CreateRecordBatch { source, .. } => source.status_code(),
+            Error::ConvertArrowSchema { source, .. } => source.status_code(),
         }
     }
 

src/common/mem-prof/src/error.rs

@@ -23,7 +23,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
-    #[snafu(display("Failed to read OPT_PROF"))]
+    #[snafu(display("Failed to read OPT_PROF, source: {}", source))]
     ReadOptProf { source: tikv_jemalloc_ctl::Error },
 
     #[snafu(display("Memory profiling is not enabled"))]
@@ -32,13 +32,17 @@ pub enum Error {
     #[snafu(display("Failed to build temp file from given path: {:?}", path))]
     BuildTempPath { path: PathBuf, location: Location },
 
-    #[snafu(display("Failed to open temp file: {}", path))]
+    #[snafu(display("Failed to open temp file: {}, source: {}", path, source))]
     OpenTempFile {
         path: String,
         source: std::io::Error,
     },
 
-    #[snafu(display("Failed to dump profiling data to temp file: {:?}", path))]
+    #[snafu(display(
+        "Failed to dump profiling data to temp file: {:?}, source: {}",
+        path,
+        source
+    ))]
     DumpProfileData {
         path: PathBuf,
         source: tikv_jemalloc_ctl::Error,

src/common/meta/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 api = { path = "../../api" }
+async-trait.workspace = true
 common-catalog = { path = "../catalog" }
 common-error = { path = "../error" }
 common-runtime = { path = "../runtime" }

src/common/meta/src/error.rs

@@ -52,6 +52,9 @@ pub enum Error {
         err_msg: String,
         location: Location,
     },
+
+    #[snafu(display("Invalid protobuf message, err: {}", err_msg))]
+    InvalidProtoMsg { err_msg: String, location: Location },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -61,7 +64,10 @@ impl ErrorExt for Error {
         use Error::*;
         match self {
             IllegalServerState { .. } => StatusCode::Internal,
-            SerdeJson { .. } | RouteInfoCorrupted { .. } => StatusCode::Unexpected,
+
+            SerdeJson { .. } | RouteInfoCorrupted { .. } | InvalidProtoMsg { .. } => {
+                StatusCode::Unexpected
+            }
 
             SendMessage { .. } => StatusCode::Internal,
 

src/common/meta/src/heartbeat/handler.rs

@@ -15,6 +15,7 @@
 use std::sync::Arc;
 
 use api::v1::meta::HeartbeatResponse;
+use async_trait::async_trait;
 use common_telemetry::error;
 
 use crate::error::Result;
@@ -57,14 +58,16 @@ impl HeartbeatResponseHandlerContext {
 /// [`HeartbeatResponseHandler::is_acceptable`] returns true if handler can handle incoming [`HeartbeatResponseHandlerContext`].
 ///
 /// [`HeartbeatResponseHandler::handle`] handles all or part of incoming [`HeartbeatResponseHandlerContext`].
+#[async_trait]
 pub trait HeartbeatResponseHandler: Send + Sync {
     fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool;
 
-    fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> Result<HandleControl>;
+    async fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> Result<HandleControl>;
 }
 
+#[async_trait]
 pub trait HeartbeatResponseHandlerExecutor: Send + Sync {
-    fn handle(&self, ctx: HeartbeatResponseHandlerContext) -> Result<()>;
+    async fn handle(&self, ctx: HeartbeatResponseHandlerContext) -> Result<()>;
 }
 
 pub struct HandlerGroupExecutor {
@@ -77,14 +80,15 @@ impl HandlerGroupExecutor {
     }
 }
 
+#[async_trait]
 impl HeartbeatResponseHandlerExecutor for HandlerGroupExecutor {
-    fn handle(&self, mut ctx: HeartbeatResponseHandlerContext) -> Result<()> {
+    async fn handle(&self, mut ctx: HeartbeatResponseHandlerContext) -> Result<()> {
         for handler in &self.handlers {
             if !handler.is_acceptable(&ctx) {
                 continue;
             }
 
-            match handler.handle(&mut ctx) {
+            match handler.handle(&mut ctx).await {
                 Ok(HandleControl::Done) => break,
                 Ok(HandleControl::Continue) => {}
                 Err(e) => {

src/common/meta/src/heartbeat/handler/parse_mailbox_message.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use async_trait::async_trait;
+
 use crate::error::Result;
 use crate::heartbeat::handler::{
     HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
@@ -21,12 +23,13 @@ use crate::heartbeat::utils::mailbox_message_to_incoming_message;
 #[derive(Default)]
 pub struct ParseMailboxMessageHandler;
 
+#[async_trait]
 impl HeartbeatResponseHandler for ParseMailboxMessageHandler {
     fn is_acceptable(&self, _ctx: &HeartbeatResponseHandlerContext) -> bool {
         true
     }
 
-    fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> Result<HandleControl> {
+    async fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> Result<HandleControl> {
         if let Some(message) = &ctx.response.mailbox_message {
             if message.payload.is_some() {
                 // mailbox_message_to_incoming_message will raise an error if payload is none

src/common/meta/src/ident.rs

@@ -0,0 +1,71 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{Display, Formatter};
+
+use api::v1::meta::{TableIdent as RawTableIdent, TableName};
+use serde::{Deserialize, Serialize};
+use snafu::OptionExt;
+
+use crate::error::{Error, InvalidProtoMsgSnafu};
+
+#[derive(Eq, Hash, PartialEq, Clone, Debug, Serialize, Deserialize)]
+pub struct TableIdent {
+    pub catalog: String,
+    pub schema: String,
+    pub table: String,
+    pub table_id: u32,
+    pub engine: String,
+}
+
+impl Display for TableIdent {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Table(id={}, name='{}.{}.{}', engine='{}')",
+            self.table_id, self.catalog, self.schema, self.table, self.engine,
+        )
+    }
+}
+
+impl TryFrom<RawTableIdent> for TableIdent {
+    type Error = Error;
+
+    fn try_from(value: RawTableIdent) -> Result<Self, Self::Error> {
+        let table_name = value.table_name.context(InvalidProtoMsgSnafu {
+            err_msg: "'table_name' is missing in TableIdent",
+        })?;
+        Ok(Self {
+            catalog: table_name.catalog_name,
+            schema: table_name.schema_name,
+            table: table_name.table_name,
+            table_id: value.table_id,
+            engine: value.engine,
+        })
+    }
+}
+
+impl From<TableIdent> for RawTableIdent {
+    fn from(table_ident: TableIdent) -> Self {
+        Self {
+            table_id: table_ident.table_id,
+            engine: table_ident.engine,
+            table_name: Some(TableName {
+                catalog_name: table_ident.catalog,
+                schema_name: table_ident.schema,
+                table_name: table_ident.table,
+            }),
+        }
+    }
+}

src/common/meta/src/instruction.rs

@@ -16,6 +16,7 @@ use std::fmt::{Display, Formatter};
 
 use serde::{Deserialize, Serialize};
 
+use crate::ident::TableIdent;
 use crate::{ClusterId, DatanodeId};
 
 #[derive(Eq, Hash, PartialEq, Clone, Debug, Serialize, Deserialize)]
@@ -49,25 +50,6 @@ impl From<RegionIdent> for TableIdent {
     }
 }
 
-#[derive(Eq, Hash, PartialEq, Clone, Debug, Serialize, Deserialize)]
-pub struct TableIdent {
-    pub catalog: String,
-    pub schema: String,
-    pub table: String,
-    pub table_id: u32,
-    pub engine: String,
-}
-
-impl Display for TableIdent {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "TableIdent(table_id='{}', table_name='{}.{}.{}', table_engine='{}')",
-            self.table_id, self.catalog, self.schema, self.table, self.engine,
-        )
-    }
-}
-
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
 pub struct SimpleReply {
     pub result: bool,

src/common/meta/src/lib.rs

@@ -14,6 +14,7 @@
 
 pub mod error;
 pub mod heartbeat;
+pub mod ident;
 pub mod instruction;
 pub mod key;
 pub mod peer;

src/common/pprof/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "common-pprof"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+common-error = { path = "../error" }
+pprof = { version = "0.11", features = [
+    "flamegraph",
+    "prost-codec",
+    "protobuf",
+] }
+prost.workspace = true
+snafu.workspace = true
+tokio.workspace = true

src/common/pprof/README.md

@@ -0,0 +1,28 @@
+# Profiling CPU
+
+## Build GreptimeDB with `pprof` feature
+
+```bash
+cargo build --features=pprof
+```
+
+## HTTP API
+Sample at 99 Hertz, for 5 seconds, output report in [protobuf format](https://github.com/google/pprof/blob/master/proto/profile.proto).
+```bash
+curl -s '0:4000/v1/prof/cpu' > /tmp/pprof.out
+```
+
+Then you can use `pprof` command with the protobuf file.
+```bash
+go tool pprof -top /tmp/pprof.out
+```
+
+Sample at 99 Hertz, for 60 seconds, output report in flamegraph format.
+```bash
+curl -s '0:4000/v1/prof/cpu?seconds=60&output=flamegraph' > /tmp/pprof.svg
+```
+
+Sample at 49 Hertz, for 10 seconds, output report in text format.
+```bash
+curl -s '0:4000/v1/prof/cpu?seconds=10&frequency=49&output=text' > /tmp/pprof.txt
+```

src/common/pprof/src/lib.rs

@@ -0,0 +1,124 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::time::Duration;
+
+use common_error::prelude::{ErrorExt, StatusCode};
+use prost::Message;
+use snafu::{Location, ResultExt, Snafu};
+
+#[derive(Debug, Snafu)]
+pub enum Error {
+    #[snafu(display(
+        "Failed to create profiler guard, source: {}, location: {}",
+        source,
+        location
+    ))]
+    CreateGuard {
+        source: pprof::Error,
+        location: Location,
+    },
+
+    #[snafu(display("Failed to create report, source: {}, location: {}", source, location))]
+    CreateReport {
+        source: pprof::Error,
+        location: Location,
+    },
+
+    #[snafu(display(
+        "Failed to create flamegraph, source: {}, location: {}",
+        source,
+        location
+    ))]
+    CreateFlamegraph {
+        source: pprof::Error,
+        location: Location,
+    },
+
+    #[snafu(display(
+        "Failed to create pprof report, source: {}, location: {}",
+        source,
+        location
+    ))]
+    ReportPprof {
+        source: pprof::Error,
+        location: Location,
+    },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        StatusCode::Unexpected
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// CPU profiler utility.
+// Inspired by https://github.com/datafuselabs/databend/blob/67f445e83cd4eceda98f6c1c114858929d564029/src/common/base/src/base/profiling.rs
+#[derive(Debug)]
+pub struct Profiling {
+    /// Sample duration.
+    duration: Duration,
+    /// Sample frequency.
+    frequency: i32,
+}
+
+impl Profiling {
+    /// Creates a new profiler.
+    pub fn new(duration: Duration, frequency: i32) -> Profiling {
+        Profiling {
+            duration,
+            frequency,
+        }
+    }
+
+    /// Profiles and returns a generated pprof report.
+    pub async fn report(&self) -> Result<pprof::Report> {
+        let guard = pprof::ProfilerGuardBuilder::default()
+            .frequency(self.frequency)
+            .blocklist(&["libc", "libgcc", "pthread", "vdso"])
+            .build()
+            .context(CreateGuardSnafu)?;
+        tokio::time::sleep(self.duration).await;
+        guard.report().build().context(CreateReportSnafu)
+    }
+
+    /// Profiles and returns a generated flamegraph.
+    pub async fn dump_flamegraph(&self) -> Result<Vec<u8>> {
+        let mut body: Vec<u8> = Vec::new();
+
+        let report = self.report().await?;
+        report
+            .flamegraph(&mut body)
+            .context(CreateFlamegraphSnafu)?;
+
+        Ok(body)
+    }
+
+    /// Profiles and returns a generated proto.
+    pub async fn dump_proto(&self) -> Result<Vec<u8>> {
+        let report = self.report().await?;
+        // Generate google’s pprof format report.
+        let profile = report.pprof().context(ReportPprofSnafu)?;
+        let body = profile.encode_to_vec();
+
+        Ok(body)
+    }
+}

src/common/procedure/src/error.rs

@@ -29,10 +29,7 @@ pub enum Error {
         "Failed to execute procedure due to external error, source: {}",
         source
     ))]
-    External {
-        #[snafu(backtrace)]
-        source: BoxedError,
-    },
+    External { source: BoxedError },
 
     #[snafu(display("Loader {} is already registered", name))]
     LoaderConflict { name: String, location: Location },
@@ -52,7 +49,7 @@ pub enum Error {
     #[snafu(display("Failed to put state, key: '{key}', source: {source}"))]
     PutState {
         key: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
@@ -65,14 +62,14 @@ pub enum Error {
     #[snafu(display("Failed to delete keys: '{keys}', source: {source}"))]
     DeleteStates {
         keys: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
     #[snafu(display("Failed to list state, path: '{path}', source: {source}"))]
     ListState {
         path: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
@@ -83,10 +80,7 @@ pub enum Error {
     },
 
     #[snafu(display("Procedure exec failed, source: {}", source))]
-    RetryLater {
-        #[snafu(backtrace)]
-        source: BoxedError,
-    },
+    RetryLater { source: BoxedError },
 
     #[snafu(display("Procedure panics, procedure_id: {}", procedure_id))]
     ProcedurePanic { procedure_id: ProcedureId },

src/common/query/src/error.rs

@@ -40,7 +40,7 @@ pub enum Error {
         source
     ))]
     UdfTempRecordBatch {
-        #[snafu(backtrace)]
+        location: Location,
         source: RecordbatchError,
     },
 
@@ -65,19 +65,19 @@ pub enum Error {
 
     #[snafu(display("Fail to cast scalar value into vector: {}", source))]
     FromScalarValue {
-        #[snafu(backtrace)]
+        location: Location,
         source: DataTypeError,
     },
 
     #[snafu(display("Fail to cast arrow array into vector: {}", source))]
     FromArrowArray {
-        #[snafu(backtrace)]
+        location: Location,
         source: DataTypeError,
     },
 
     #[snafu(display("Fail to cast arrow array into vector: {:?}, {}", data_type, source))]
     IntoVector {
-        #[snafu(backtrace)]
+        location: Location,
         source: DataTypeError,
         data_type: ArrowDatatype,
     },
@@ -93,7 +93,7 @@ pub enum Error {
 
     #[snafu(display("Invalid input type: {}", err_msg))]
     InvalidInputType {
-        #[snafu(backtrace)]
+        location: Location,
         source: DataTypeError,
         err_msg: String,
     },
@@ -120,19 +120,19 @@ pub enum Error {
         source
     ))]
     ConvertDfRecordBatchStream {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
     #[snafu(display("Failed to convert arrow schema, source: {}", source))]
     ConvertArrowSchema {
-        #[snafu(backtrace)]
+        location: Location,
         source: DataTypeError,
     },
 
     #[snafu(display("Failed to execute physical plan, source: {}", source))]
     ExecutePhysicalPlan {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
@@ -154,13 +154,13 @@ pub enum Error {
 
     #[snafu(display("Query engine fail to cast value: {}", source))]
     ToScalarValue {
-        #[snafu(backtrace)]
+        location: Location,
         source: DataTypeError,
     },
 
     #[snafu(display("Failed to get scalar vector, {}", source))]
     GetScalarVector {
-        #[snafu(backtrace)]
+        location: Location,
         source: DataTypeError,
     },
 
@@ -188,9 +188,9 @@ impl ErrorExt for Error {
 
             Error::InvalidInputType { source, .. }
             | Error::IntoVector { source, .. }
-            | Error::FromScalarValue { source }
-            | Error::ConvertArrowSchema { source }
-            | Error::FromArrowArray { source } => source.status_code(),
+            | Error::FromScalarValue { source, .. }
+            | Error::ConvertArrowSchema { source, .. }
+            | Error::FromArrowArray { source, .. } => source.status_code(),
 
             Error::ExecuteRepeatedly { .. } | Error::GeneralDataFusion { .. } => {
                 StatusCode::Unexpected
@@ -201,7 +201,7 @@ impl ErrorExt for Error {
             | Error::InvalidFuncArgs { .. } => StatusCode::InvalidArguments,
 
             Error::ConvertDfRecordBatchStream { source, .. } => source.status_code(),
-            Error::ExecutePhysicalPlan { source } => source.status_code(),
+            Error::ExecutePhysicalPlan { source, .. } => source.status_code(),
         }
     }
 
@@ -215,9 +215,3 @@ impl From<Error> for DataFusionError {
         DataFusionError::External(Box::new(e))
     }
 }
-
-impl From<BoxedError> for Error {
-    fn from(source: BoxedError) -> Self {
-        Error::ExecutePhysicalPlan { source }
-    }
-}

src/common/query/src/logical_plan.rs

@@ -20,6 +20,7 @@ mod udf;
 use std::sync::Arc;
 
 use datatypes::prelude::ConcreteDataType;
+pub use expr::build_filter_from_timestamp;
 
 pub use self::accumulator::{Accumulator, AggregateFunctionCreator, AggregateFunctionCreatorRef};
 pub use self::expr::{DfExpr, Expr};
@@ -28,7 +29,6 @@ pub use self::udf::ScalarUdf;
 use crate::function::{ReturnTypeFunction, ScalarFunctionImplementation};
 use crate::logical_plan::accumulator::*;
 use crate::signature::{Signature, Volatility};
-
 /// Creates a new UDF with a specific signature and specific return type.
 /// This is a helper function to create a new UDF.
 /// The function `create_udf` returns a subset of all possible `ScalarFunction`:

src/common/query/src/logical_plan/accumulator.rs

@@ -172,7 +172,6 @@ impl DfAccumulator for DfAccumulatorAdaptor {
     }
 
     fn size(&self) -> usize {
-        // TODO(LFC): Implement new "size" method for Accumulator.
         0
     }
 }

src/common/query/src/logical_plan/expr.rs

@@ -12,7 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_time::range::TimestampRange;
+use common_time::timestamp::TimeUnit;
+use common_time::Timestamp;
+use datafusion_common::{Column, ScalarValue};
 pub use datafusion_expr::expr::Expr as DfExpr;
+use datafusion_expr::{and, binary_expr, Operator};
 
 /// Central struct of query API.
 /// Represent logical expressions such as `A + 1`, or `CAST(c1 AS int)`.
@@ -33,6 +38,54 @@ impl From<DfExpr> for Expr {
     }
 }
 
+/// Builds an `Expr` that filters timestamp column from given timestamp range.
+/// Returns [None] if time range is [None] or full time range.
+pub fn build_filter_from_timestamp(
+    ts_col_name: &str,
+    time_range: Option<&TimestampRange>,
+) -> Option<Expr> {
+    let Some(time_range) = time_range else { return None; };
+    let ts_col_expr = DfExpr::Column(Column {
+        relation: None,
+        name: ts_col_name.to_string(),
+    });
+
+    let df_expr = match (time_range.start(), time_range.end()) {
+        (None, None) => None,
+        (Some(start), None) => Some(binary_expr(
+            ts_col_expr,
+            Operator::GtEq,
+            timestamp_to_literal(start),
+        )),
+        (None, Some(end)) => Some(binary_expr(
+            ts_col_expr,
+            Operator::Lt,
+            timestamp_to_literal(end),
+        )),
+        (Some(start), Some(end)) => Some(and(
+            binary_expr(
+                ts_col_expr.clone(),
+                Operator::GtEq,
+                timestamp_to_literal(start),
+            ),
+            binary_expr(ts_col_expr, Operator::Lt, timestamp_to_literal(end)),
+        )),
+    };
+
+    df_expr.map(Expr::from)
+}
+
+/// Converts a [Timestamp] to datafusion literal value.
+fn timestamp_to_literal(timestamp: &Timestamp) -> DfExpr {
+    let scalar_value = match timestamp.unit() {
+        TimeUnit::Second => ScalarValue::TimestampSecond(Some(timestamp.value()), None),
+        TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(Some(timestamp.value()), None),
+        TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(Some(timestamp.value()), None),
+        TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(Some(timestamp.value()), None),
+    };
+    DfExpr::Literal(scalar_value)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/common/query/src/physical_plan.rs

@@ -194,7 +194,6 @@ impl DfPhysicalPlan for DfPhysicalPlanAdapter {
     }
 
     fn statistics(&self) -> Statistics {
-        // TODO(LFC): impl statistics
         Statistics::default()
     }
 }

src/common/recordbatch/src/adapter.rs

@@ -225,6 +225,7 @@ mod test {
     use datatypes::prelude::ConcreteDataType;
     use datatypes::schema::ColumnSchema;
     use datatypes::vectors::Int32Vector;
+    use snafu::IntoError;
 
     use super::*;
     use crate::RecordBatches;
@@ -296,9 +297,8 @@ mod test {
 
         let poll_err_stream = new_future_stream(Ok(vec![
             Ok(batch1.clone()),
-            Err(error::Error::External {
-                source: BoxedError::new(MockError::new(StatusCode::Unknown)),
-            }),
+            Err(error::ExternalSnafu
+                .into_error(BoxedError::new(MockError::new(StatusCode::Unknown)))),
         ]));
         let adapter = AsyncRecordBatchStreamAdapter::new(schema.clone(), poll_err_stream);
         let result = RecordBatches::try_collect(Box::pin(adapter)).await;
@@ -307,9 +307,9 @@ mod test {
             "Failed to poll stream, source: External error: External error, source: Unknown"
         );
 
-        let failed_to_init_stream = new_future_stream(Err(error::Error::External {
-            source: BoxedError::new(MockError::new(StatusCode::Internal)),
-        }));
+        let failed_to_init_stream =
+            new_future_stream(Err(error::ExternalSnafu
+                .into_error(BoxedError::new(MockError::new(StatusCode::Internal)))));
         let adapter = AsyncRecordBatchStreamAdapter::new(schema.clone(), failed_to_init_stream);
         let result = RecordBatches::try_collect(Box::pin(adapter)).await;
         assert_eq!(

src/common/recordbatch/src/error.rs

@@ -33,13 +33,13 @@ pub enum Error {
 
     #[snafu(display("Data types error, source: {}", source))]
     DataTypes {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
     #[snafu(display("External error, source: {}", source))]
     External {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
@@ -99,7 +99,7 @@ pub enum Error {
     CastVector {
         from_type: ConcreteDataType,
         to_type: ConcreteDataType,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 }
@@ -117,7 +117,7 @@ impl ErrorExt for Error {
             | Error::ColumnNotExists { .. }
             | Error::ProjectArrowRecordBatch { .. } => StatusCode::Internal,
 
-            Error::External { source } => source.status_code(),
+            Error::External { source, .. } => source.status_code(),
 
             Error::SchemaConversion { source, .. } | Error::CastVector { source, .. } => {
                 source.status_code()

src/common/substrait/src/error.rs

@@ -74,7 +74,7 @@ pub enum Error {
 
     #[snafu(display("Internal error: {}", source))]
     Internal {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
@@ -96,14 +96,14 @@ pub enum Error {
 
     #[snafu(display("Failed to convert DataFusion schema, source: {}", source))]
     ConvertDfSchema {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
     #[snafu(display("Unable to resolve table: {table_name}, error: {source}"))]
     ResolveTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
@@ -141,7 +141,7 @@ impl ErrorExt for Error {
             | Error::Internal { .. }
             | Error::EncodeDfPlan { .. }
             | Error::DecodeDfPlan { .. } => StatusCode::Internal,
-            Error::ConvertDfSchema { source } => source.status_code(),
+            Error::ConvertDfSchema { source, .. } => source.status_code(),
             Error::ResolveTable { source, .. } => source.status_code(),
         }
     }

src/common/time/src/date.rs

@@ -52,6 +52,12 @@ impl From<i32> for Date {
     }
 }
 
+impl From<NaiveDate> for Date {
+    fn from(date: NaiveDate) -> Self {
+        Self(date.num_days_from_ce() - UNIX_EPOCH_FROM_CE)
+    }
+}
+
 impl Display for Date {
     /// [Date] is formatted according to ISO-8601 standard.
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {

src/common/time/src/range.rs

@@ -12,6 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::{Debug, Display, Formatter};
+
+use serde::{Deserialize, Serialize};
+
 use crate::timestamp::TimeUnit;
 use crate::timestamp_millis::TimestampMillis;
 use crate::Timestamp;
@@ -21,7 +25,7 @@ use crate::Timestamp;
 /// The range contains values that `value >= start` and `val < end`.
 ///
 /// The range is empty iff `start == end == "the default value of T"`
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct GenericRange<T> {
     start: Option<T>,
     end: Option<T>,
@@ -193,6 +197,38 @@ impl<T: PartialOrd> GenericRange<T> {
 
 pub type TimestampRange = GenericRange<Timestamp>;
 
+impl Display for TimestampRange {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let s = match (&self.start, &self.end) {
+            (Some(start), Some(end)) => {
+                format!(
+                    "TimestampRange{{[{}{},{}{})}}",
+                    start.value(),
+                    start.unit().short_name(),
+                    end.value(),
+                    end.unit().short_name()
+                )
+            }
+            (Some(start), None) => {
+                format!(
+                    "TimestampRange{{[{}{},#)}}",
+                    start.value(),
+                    start.unit().short_name()
+                )
+            }
+            (None, Some(end)) => {
+                format!(
+                    "TimestampRange{{[#,{}{})}}",
+                    end.value(),
+                    end.unit().short_name()
+                )
+            }
+            (None, None) => "TimestampRange{{[#,#)}}".to_string(),
+        };
+        f.write_str(&s)
+    }
+}
+
 impl TimestampRange {
     /// Create a TimestampRange with optional inclusive end timestamp.
     /// If end timestamp is present and is less than start timestamp, this method will return
@@ -488,4 +524,25 @@ mod tests {
         );
         assert!(range.is_empty());
     }
+
+    #[test]
+    fn test_serialize_timestamp_range() {
+        macro_rules! test_serde_for_unit {
+            ($($unit: expr),*) => {
+                $(
+                let original_range = TimestampRange::with_unit(0, 10, $unit).unwrap();
+                let string = serde_json::to_string(&original_range).unwrap();
+                let deserialized: TimestampRange = serde_json::from_str(&string).unwrap();
+                assert_eq!(original_range, deserialized);
+                )*
+            };
+        }
+
+        test_serde_for_unit!(
+            TimeUnit::Second,
+            TimeUnit::Millisecond,
+            TimeUnit::Microsecond,
+            TimeUnit::Nanosecond
+        );
+    }
 }

src/common/time/src/timestamp.rs

@@ -336,6 +336,15 @@ impl TimeUnit {
             TimeUnit::Nanosecond => 1,
         }
     }
+
+    pub(crate) fn short_name(&self) -> &'static str {
+        match self {
+            TimeUnit::Second => "s",
+            TimeUnit::Millisecond => "ms",
+            TimeUnit::Microsecond => "us",
+            TimeUnit::Nanosecond => "ns",
+        }
+    }
 }
 
 impl PartialOrd for Timestamp {

src/datanode/src/datanode.rs

@@ -192,8 +192,8 @@ impl Default for WalConfig {
     fn default() -> Self {
         Self {
             dir: None,
-            file_size: ReadableSize::gb(1),        // log file size 1G
-            purge_threshold: ReadableSize::gb(50), // purge threshold 50G
+            file_size: ReadableSize::mb(256), // log file size 256MB
+            purge_threshold: ReadableSize::gb(4), // purge threshold 4GB
             purge_interval: Duration::from_secs(600),
             read_batch_size: 128,
             sync_write: false,
@@ -221,7 +221,7 @@ impl Default for RegionManifestConfig {
     fn default() -> Self {
         Self {
             checkpoint_margin: Some(10u16),
-            gc_duration: Some(Duration::from_secs(30)),
+            gc_duration: Some(Duration::from_secs(600)),
             checkpoint_on_startup: false,
             compress: false,
         }
@@ -340,8 +340,6 @@ pub struct DatanodeOptions {
     pub rpc_addr: String,
     pub rpc_hostname: Option<String>,
     pub rpc_runtime_size: usize,
-    pub mysql_addr: String,
-    pub mysql_runtime_size: usize,
     pub http_opts: HttpOptions,
     pub meta_client_options: Option<MetaClientOptions>,
     pub wal: WalConfig,
@@ -359,8 +357,6 @@ impl Default for DatanodeOptions {
             rpc_addr: "127.0.0.1:3001".to_string(),
             rpc_hostname: None,
             rpc_runtime_size: 8,
-            mysql_addr: "127.0.0.1:4406".to_string(),
-            mysql_runtime_size: 2,
             http_opts: HttpOptions::default(),
             meta_client_options: None,
             wal: WalConfig::default(),

src/datanode/src/error.rs

@@ -27,14 +27,14 @@ use table::error::Error as TableError;
 pub enum Error {
     #[snafu(display("Failed to access catalog, source: {}", source))]
     AccessCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
     #[snafu(display("Failed to deregister table: {}, source: {}", table_name, source))]
     DeregisterTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
@@ -48,7 +48,7 @@ pub enum Error {
     #[snafu(display("Failed to open table: {}, source: {}", table_name, source))]
     OpenTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: TableError,
     },
 
@@ -68,7 +68,7 @@ pub enum Error {
     CloseTable {
         table_name: String,
         region_numbers: Vec<RegionNumber>,
-        #[snafu(backtrace)]
+        location: Location,
         source: TableError,
     },
 
@@ -93,31 +93,31 @@ pub enum Error {
 
     #[snafu(display("Failed to execute sql, source: {}", source))]
     ExecuteSql {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to plan statement, source: {}", source))]
     PlanStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to execute statement, source: {}", source))]
     ExecuteStatement {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to execute logical plan, source: {}", source))]
     ExecuteLogicalPlan {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to decode logical plan, source: {}", source))]
     DecodeLogicalPlan {
-        #[snafu(backtrace)]
+        location: Location,
         source: substrait::error::Error,
     },
 
@@ -126,7 +126,7 @@ pub enum Error {
 
     #[snafu(display("Failed to create catalog list, source: {}", source))]
     NewCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
@@ -139,21 +139,21 @@ pub enum Error {
     #[snafu(display("Failed to create table: {}, source: {}", table_name, source))]
     CreateTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: TableError,
     },
 
     #[snafu(display("Failed to drop table {}, source: {}", table_name, source))]
     DropTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
     #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
     TableEngineNotFound {
         engine_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -164,7 +164,7 @@ pub enum Error {
     ))]
     EngineProcedureNotFound {
         engine_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -192,7 +192,7 @@ pub enum Error {
 
     #[snafu(display("Failed to parse sql value, source: {}", source))]
     ParseSqlValue {
-        #[snafu(backtrace)]
+        location: Location,
         source: sql::error::Error,
     },
 
@@ -202,7 +202,7 @@ pub enum Error {
     #[snafu(display("Failed to insert value to table: {}, source: {}", table_name, source))]
     Insert {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: TableError,
     },
 
@@ -213,20 +213,20 @@ pub enum Error {
     ))]
     Delete {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: TableError,
     },
 
     #[snafu(display("Failed to flush table: {}, source: {}", table_name, source))]
     FlushTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: TableError,
     },
 
     #[snafu(display("Failed to start server, source: {}", source))]
     StartServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
     },
 
@@ -250,8 +250,8 @@ pub enum Error {
 
     #[snafu(display("Failed to open log store, source: {}", source))]
     OpenLogStore {
-        #[snafu(backtrace)]
-        source: log_store::error::Error,
+        location: Location,
+        source: Box<log_store::error::Error>,
     },
 
     #[snafu(display("Failed to init backend, source: {}", source))]
@@ -262,7 +262,7 @@ pub enum Error {
 
     #[snafu(display("Runtime resource error, source: {}", source))]
     RuntimeResource {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_runtime::error::Error,
     },
 
@@ -289,7 +289,7 @@ pub enum Error {
 
     #[snafu(display("Failed to register a new schema, source: {}", source))]
     RegisterSchema {
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
@@ -298,25 +298,25 @@ pub enum Error {
 
     #[snafu(display("Failed to convert alter expr to request: {}", source))]
     AlterExprToRequest {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc_expr::error::Error,
     },
 
     #[snafu(display("Failed to convert create expr to request: {}", source))]
     CreateExprToRequest {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc_expr::error::Error,
     },
 
     #[snafu(display("Failed to convert delete expr to request: {}", source))]
     DeleteExprToRequest {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc_expr::error::Error,
     },
 
     #[snafu(display("Failed to parse SQL, source: {}", source))]
     ParseSql {
-        #[snafu(backtrace)]
+        location: Location,
         source: sql::error::Error,
     },
 
@@ -327,38 +327,38 @@ pub enum Error {
     ))]
     ParseTimestamp {
         raw: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_time::error::Error,
     },
 
     #[snafu(display("Failed to prepare immutable table: {}", source))]
     PrepareImmutableTable {
-        #[snafu(backtrace)]
+        location: Location,
         source: query::error::Error,
     },
 
     #[snafu(display("Failed to access catalog, source: {}", source))]
     Catalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
     #[snafu(display("Failed to find table {} from catalog, source: {}", table_name, source))]
     FindTable {
         table_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: catalog::error::Error,
     },
 
     #[snafu(display("Failed to initialize meta client, source: {}", source))]
     MetaClientInit {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_client::error::Error,
     },
 
     #[snafu(display("Failed to insert data, source: {}", source))]
     InsertData {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc_expr::error::Error,
     },
 
@@ -369,7 +369,7 @@ pub enum Error {
 
     #[snafu(display("Failed to bump table id, source: {}", source))]
     BumpTableId {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -392,7 +392,7 @@ pub enum Error {
     ))]
     ColumnDefaultValue {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -404,45 +404,45 @@ pub enum Error {
 
     #[snafu(display("Unrecognized table option: {}", source))]
     UnrecognizedTableOption {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("Failed to recover procedure, source: {}", source))]
     RecoverProcedure {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_procedure::error::Error,
     },
 
     #[snafu(display("Failed to submit procedure {}, source: {}", procedure_id, source))]
     SubmitProcedure {
         procedure_id: ProcedureId,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_procedure::error::Error,
     },
 
     #[snafu(display("Failed to wait procedure {} done, source: {}", procedure_id, source))]
     WaitProcedure {
         procedure_id: ProcedureId,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_procedure::error::Error,
     },
 
     #[snafu(display("Failed to close table engine, source: {}", source))]
     CloseTableEngine {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
     #[snafu(display("Failed to shutdown server, source: {}", source))]
     ShutdownServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
     },
 
     #[snafu(display("Failed to shutdown instance, source: {}", source))]
     ShutdownInstance {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
@@ -487,15 +487,15 @@ impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         use Error::*;
         match self {
-            ExecuteSql { source }
-            | PlanStatement { source }
-            | ExecuteStatement { source }
-            | ExecuteLogicalPlan { source } => source.status_code(),
+            ExecuteSql { source, .. }
+            | PlanStatement { source, .. }
+            | ExecuteStatement { source, .. }
+            | ExecuteLogicalPlan { source, .. } => source.status_code(),
 
             HandleHeartbeatResponse { source, .. } => source.status_code(),
 
-            DecodeLogicalPlan { source } => source.status_code(),
-            NewCatalog { source } | RegisterSchema { source } => source.status_code(),
+            DecodeLogicalPlan { source, .. } => source.status_code(),
+            NewCatalog { source, .. } | RegisterSchema { source, .. } => source.status_code(),
             FindTable { source, .. } => source.status_code(),
             CreateTable { source, .. } => source.status_code(),
             DropTable { source, .. } => source.status_code(),
@@ -512,9 +512,9 @@ impl ErrorExt for Error {
             ParseSqlValue { source, .. } | ParseSql { source, .. } => source.status_code(),
 
             AlterExprToRequest { source, .. }
-            | CreateExprToRequest { source }
-            | DeleteExprToRequest { source }
-            | InsertData { source } => source.status_code(),
+            | CreateExprToRequest { source, .. }
+            | DeleteExprToRequest { source, .. }
+            | InsertData { source, .. } => source.status_code(),
 
             ColumnValuesNumberMismatch { .. }
             | InvalidSql { .. }
@@ -559,13 +559,13 @@ impl ErrorExt for Error {
             | CloseTableEngine { .. }
             | JoinTask { .. } => StatusCode::Internal,
 
-            StartServer { source }
-            | ShutdownServer { source }
+            StartServer { source, .. }
+            | ShutdownServer { source, .. }
             | WaitForGrpcServing { source, .. } => source.status_code(),
 
             InitBackend { .. } => StatusCode::StorageUnavailable,
 
-            OpenLogStore { source } => source.status_code(),
+            OpenLogStore { source, .. } => source.status_code(),
             RuntimeResource { .. } => StatusCode::RuntimeResourcesExhausted,
             MetaClientInit { source, .. } => source.status_code(),
             TableIdProviderNotFound { .. } => StatusCode::Unsupported,

src/datanode/src/heartbeat.rs

@@ -17,24 +17,27 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use api::v1::meta::{HeartbeatRequest, NodeStat, Peer};
+use catalog::remote::region_alive_keeper::RegionAliveKeepers;
 use catalog::{datanode_stat, CatalogManagerRef};
 use common_meta::heartbeat::handler::{
     HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutorRef,
 };
 use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MailboxRef};
 use common_meta::heartbeat::utils::outgoing_message_to_mailbox_message;
-use common_telemetry::{error, info, trace, warn};
+use common_telemetry::{debug, error, info, trace, warn};
 use meta_client::client::{HeartbeatSender, MetaClient};
 use snafu::ResultExt;
 use tokio::sync::mpsc;
 use tokio::time::Instant;
 
+use crate::datanode::DatanodeOptions;
 use crate::error::{self, MetaClientInitSnafu, Result};
 
 pub(crate) mod handler;
 
 pub struct HeartbeatTask {
     node_id: u64,
+    node_epoch: u64,
     server_addr: String,
     server_hostname: Option<String>,
     running: Arc<AtomicBool>,
@@ -42,6 +45,7 @@ pub struct HeartbeatTask {
     catalog_manager: CatalogManagerRef,
     interval: u64,
     resp_handler_executor: HeartbeatResponseHandlerExecutorRef,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
 }
 
 impl Drop for HeartbeatTask {
@@ -54,21 +58,25 @@ impl HeartbeatTask {
     /// Create a new heartbeat task instance.
     pub fn new(
         node_id: u64,
-        server_addr: String,
-        server_hostname: Option<String>,
+        opts: &DatanodeOptions,
         meta_client: Arc<MetaClient>,
         catalog_manager: CatalogManagerRef,
         resp_handler_executor: HeartbeatResponseHandlerExecutorRef,
+        heartbeat_interval_millis: u64,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
     ) -> Self {
         Self {
             node_id,
-            server_addr,
-            server_hostname,
+            // We use datanode's start time millis as the node's epoch.
+            node_epoch: common_time::util::current_time_millis() as u64,
+            server_addr: opts.rpc_addr.clone(),
+            server_hostname: opts.rpc_hostname.clone(),
             running: Arc::new(AtomicBool::new(false)),
             meta_client,
             catalog_manager,
-            interval: 5_000, // default interval is set to 5 secs
+            interval: heartbeat_interval_millis,
             resp_handler_executor,
+            region_alive_keepers,
         }
     }
 
@@ -94,7 +102,7 @@ impl HeartbeatTask {
                 }
 
                 let ctx = HeartbeatResponseHandlerContext::new(mailbox.clone(), res);
-                if let Err(e) = Self::handle_response(ctx, handler_executor.clone()) {
+                if let Err(e) = Self::handle_response(ctx, handler_executor.clone()).await {
                     error!(e; "Error while handling heartbeat response");
                 }
                 if !running.load(Ordering::Acquire) {
@@ -106,13 +114,14 @@ impl HeartbeatTask {
         Ok(tx)
     }
 
-    fn handle_response(
+    async fn handle_response(
         ctx: HeartbeatResponseHandlerContext,
         handler_executor: HeartbeatResponseHandlerExecutorRef,
     ) -> Result<()> {
         trace!("heartbeat response: {:?}", ctx.response);
         handler_executor
             .handle(ctx)
+            .await
             .context(error::HandleHeartbeatResponseSnafu)
     }
 
@@ -128,9 +137,12 @@ impl HeartbeatTask {
         }
         let interval = self.interval;
         let node_id = self.node_id;
+        let node_epoch = self.node_epoch;
         let addr = resolve_addr(&self.server_addr, &self.server_hostname);
         info!("Starting heartbeat to Metasrv with interval {interval}. My node id is {node_id}, address is {addr}.");
 
+        self.region_alive_keepers.start().await;
+
         let meta_client = self.meta_client.clone();
         let catalog_manager_clone = self.catalog_manager.clone();
 
@@ -147,6 +159,7 @@ impl HeartbeatTask {
         )
         .await?;
 
+        let epoch = self.region_alive_keepers.epoch();
         common_runtime::spawn_bg(async move {
             let sleep = tokio::time::sleep(Duration::from_millis(0));
             tokio::pin!(sleep);
@@ -192,6 +205,8 @@ impl HeartbeatTask {
                                 ..Default::default()
                             }),
                             region_stats,
+                            duration_since_epoch: (Instant::now() - epoch).as_millis() as u64,
+                            node_epoch,
                             ..Default::default()
                         };
                         sleep.as_mut().reset(Instant::now() + Duration::from_millis(interval));
@@ -199,6 +214,7 @@ impl HeartbeatTask {
                     }
                 };
                 if let Some(req) = req {
+                    debug!("Sending heartbeat request: {:?}", req);
                     if let Err(e) = tx.send(req).await {
                         error!("Failed to send heartbeat to metasrv, error: {:?}", e);
                         match Self::create_streams(

src/datanode/src/heartbeat/handler/close_region.rs

@@ -14,15 +14,16 @@
 
 use std::sync::Arc;
 
+use async_trait::async_trait;
+use catalog::remote::region_alive_keeper::RegionAliveKeepers;
 use catalog::{CatalogManagerRef, DeregisterTableRequest};
 use common_catalog::format_full_table_name;
 use common_meta::error::Result as MetaResult;
 use common_meta::heartbeat::handler::{
     HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
 };
-use common_meta::instruction::{
-    Instruction, InstructionReply, RegionIdent, SimpleReply, TableIdent,
-};
+use common_meta::instruction::{Instruction, InstructionReply, SimpleReply};
+use common_meta::RegionIdent;
 use common_telemetry::{error, info, warn};
 use snafu::ResultExt;
 use store_api::storage::RegionNumber;
@@ -36,8 +37,10 @@ use crate::error::{self, Result};
 pub struct CloseRegionHandler {
     catalog_manager: CatalogManagerRef,
     table_engine_manager: TableEngineManagerRef,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
 }
 
+#[async_trait]
 impl HeartbeatResponseHandler for CloseRegionHandler {
     fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
         matches!(
@@ -46,35 +49,15 @@ impl HeartbeatResponseHandler for CloseRegionHandler {
         )
     }
 
-    fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> MetaResult<HandleControl> {
+    async fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> MetaResult<HandleControl> {
         let Some((meta, Instruction::CloseRegion(region_ident))) = ctx.incoming_message.take() else {
             unreachable!("CloseRegionHandler: should be guarded by 'is_acceptable'");
         };
 
         let mailbox = ctx.mailbox.clone();
         let self_ref = Arc::new(self.clone());
-
-        let RegionIdent {
-            table_ident:
-                TableIdent {
-                    engine,
-                    catalog,
-                    schema,
-                    table,
-                    ..
-                },
-            region_number,
-            ..
-        } = region_ident;
-
         common_runtime::spawn_bg(async move {
-            let result = self_ref
-                .close_region_inner(
-                    engine,
-                    &TableReference::full(&catalog, &schema, &table),
-                    vec![region_number],
-                )
-                .await;
+            let result = self_ref.close_region_inner(region_ident).await;
 
             if let Err(e) = mailbox
                 .send((meta, CloseRegionHandler::map_result(result)))
@@ -92,10 +75,12 @@ impl CloseRegionHandler {
     pub fn new(
         catalog_manager: CatalogManagerRef,
         table_engine_manager: TableEngineManagerRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
     ) -> Self {
         Self {
             catalog_manager,
             table_engine_manager,
+            region_alive_keepers,
         }
     }
 
@@ -151,20 +136,21 @@ impl CloseRegionHandler {
         Ok(true)
     }
 
-    async fn close_region_inner(
-        &self,
-        engine: String,
-        table_ref: &TableReference<'_>,
-        region_numbers: Vec<RegionNumber>,
-    ) -> Result<bool> {
-        let engine =
-            self.table_engine_manager
-                .engine(&engine)
-                .context(error::TableEngineNotFoundSnafu {
-                    engine_name: &engine,
-                })?;
+    async fn close_region_inner(&self, region_ident: RegionIdent) -> Result<bool> {
+        let table_ident = &region_ident.table_ident;
+        let engine_name = &table_ident.engine;
+        let engine = self
+            .table_engine_manager
+            .engine(engine_name)
+            .context(error::TableEngineNotFoundSnafu { engine_name })?;
         let ctx = EngineContext::default();
 
+        let table_ref = &TableReference::full(
+            &table_ident.catalog,
+            &table_ident.schema,
+            &table_ident.table,
+        );
+        let region_numbers = vec![region_ident.region_number];
         if self
             .regions_closed(
                 table_ref.catalog,
@@ -202,7 +188,15 @@ impl CloseRegionHandler {
                 })? {
                 CloseTableResult::NotFound | CloseTableResult::Released(_) => {
                     // Deregister table if The table released.
-                    self.deregister_table(table_ref).await
+                    let deregistered = self.deregister_table(table_ref).await?;
+
+                    if deregistered {
+                        self.region_alive_keepers
+                            .deregister_table(table_ident)
+                            .await;
+                    }
+
+                    Ok(deregistered)
                 }
                 CloseTableResult::PartialClosed(regions) => {
                     // Requires caller to update the region_numbers
@@ -210,6 +204,11 @@ impl CloseRegionHandler {
                         "Close partial regions: {:?} in table: {}",
                         regions, table_ref
                     );
+
+                    self.region_alive_keepers
+                        .deregister_region(&region_ident)
+                        .await;
+
                     Ok(true)
                 }
             };

src/datanode/src/heartbeat/handler/open_region.rs

@@ -14,16 +14,16 @@
 
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use catalog::error::Error as CatalogError;
+use catalog::remote::region_alive_keeper::RegionAliveKeepers;
 use catalog::{CatalogManagerRef, RegisterTableRequest};
 use common_catalog::format_full_table_name;
 use common_meta::error::Result as MetaResult;
 use common_meta::heartbeat::handler::{
     HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
 };
-use common_meta::instruction::{
-    Instruction, InstructionReply, RegionIdent, SimpleReply, TableIdent,
-};
+use common_meta::instruction::{Instruction, InstructionReply, SimpleReply};
 use common_telemetry::{error, warn};
 use snafu::ResultExt;
 use store_api::storage::RegionNumber;
@@ -37,8 +37,10 @@ use crate::error::{self, Result};
 pub struct OpenRegionHandler {
     catalog_manager: CatalogManagerRef,
     table_engine_manager: TableEngineManagerRef,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
 }
 
+#[async_trait]
 impl HeartbeatResponseHandler for OpenRegionHandler {
     fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
         matches!(
@@ -47,7 +49,7 @@ impl HeartbeatResponseHandler for OpenRegionHandler {
         )
     }
 
-    fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> MetaResult<HandleControl> {
+    async fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> MetaResult<HandleControl> {
         let Some((meta, Instruction::OpenRegion(region_ident))) = ctx.incoming_message.take() else {
             unreachable!("OpenRegionHandler: should be guarded by 'is_acceptable'");
         };
@@ -55,9 +57,24 @@ impl HeartbeatResponseHandler for OpenRegionHandler {
         let mailbox = ctx.mailbox.clone();
         let self_ref = Arc::new(self.clone());
 
+        let region_alive_keepers = self.region_alive_keepers.clone();
         common_runtime::spawn_bg(async move {
-            let (engine, request) = OpenRegionHandler::prepare_request(region_ident);
-            let result = self_ref.open_region_inner(engine, request).await;
+            let table_ident = &region_ident.table_ident;
+            let request = OpenTableRequest {
+                catalog_name: table_ident.catalog.clone(),
+                schema_name: table_ident.schema.clone(),
+                table_name: table_ident.table.clone(),
+                table_id: table_ident.table_id,
+                region_numbers: vec![region_ident.region_number],
+            };
+            let result = self_ref
+                .open_region_inner(table_ident.engine.clone(), request)
+                .await;
+
+            if matches!(result, Ok(true)) {
+                region_alive_keepers.register_region(&region_ident).await;
+            }
+
             if let Err(e) = mailbox
                 .send((meta, OpenRegionHandler::map_result(result)))
                 .await
@@ -73,10 +90,12 @@ impl OpenRegionHandler {
     pub fn new(
         catalog_manager: CatalogManagerRef,
         table_engine_manager: TableEngineManagerRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
     ) -> Self {
         Self {
             catalog_manager,
             table_engine_manager,
+            region_alive_keepers,
         }
     }
 
@@ -97,32 +116,6 @@ impl OpenRegionHandler {
         )
     }
 
-    fn prepare_request(ident: RegionIdent) -> (String, OpenTableRequest) {
-        let RegionIdent {
-            table_ident:
-                TableIdent {
-                    catalog,
-                    schema,
-                    table,
-                    table_id,
-                    engine,
-                },
-            region_number,
-            ..
-        } = ident;
-
-        (
-            engine,
-            OpenTableRequest {
-                catalog_name: catalog,
-                schema_name: schema,
-                table_name: table,
-                table_id,
-                region_numbers: vec![region_number],
-            },
-        )
-    }
-
     /// Returns true if a table or target regions have been opened.
     async fn regions_opened(
         &self,

src/datanode/src/instance.rs

@@ -18,7 +18,8 @@ use std::time::Duration;
 use std::{fs, path};
 
 use api::v1::meta::Role;
-use catalog::remote::CachedMetaKvBackend;
+use catalog::remote::region_alive_keeper::RegionAliveKeepers;
+use catalog::remote::{CachedMetaKvBackend, RemoteCatalogManager};
 use catalog::{CatalogManager, CatalogManagerRef, RegisterTableRequest};
 use common_base::paths::{CLUSTER_DIR, WAL_DIR};
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID};
@@ -56,9 +57,9 @@ use table::Table;
 
 use crate::datanode::{DatanodeOptions, ObjectStoreConfig, ProcedureConfig, WalConfig};
 use crate::error::{
-    self, CatalogSnafu, MetaClientInitSnafu, MissingMetasrvOptsSnafu, MissingNodeIdSnafu,
-    NewCatalogSnafu, OpenLogStoreSnafu, RecoverProcedureSnafu, Result, ShutdownInstanceSnafu,
-    StartProcedureManagerSnafu, StopProcedureManagerSnafu,
+    self, CatalogSnafu, IncorrectInternalStateSnafu, MetaClientInitSnafu, MissingMetasrvOptsSnafu,
+    MissingNodeIdSnafu, NewCatalogSnafu, OpenLogStoreSnafu, RecoverProcedureSnafu, Result,
+    ShutdownInstanceSnafu, StartProcedureManagerSnafu, StopProcedureManagerSnafu,
 };
 use crate::heartbeat::handler::close_region::CloseRegionHandler;
 use crate::heartbeat::handler::open_region::OpenRegionHandler;
@@ -150,7 +151,7 @@ impl Instance {
         );
 
         // create remote catalog manager
-        let (catalog_manager, table_id_provider) = match opts.mode {
+        let (catalog_manager, table_id_provider, heartbeat_task) = match opts.mode {
             Mode::Standalone => {
                 if opts.enable_memory_catalog {
                     let catalog = Arc::new(catalog::local::MemoryCatalogManager::default());
@@ -170,6 +171,7 @@ impl Instance {
                     (
                         catalog.clone() as CatalogManagerRef,
                         Some(catalog as TableIdProviderRef),
+                        None,
                     )
                 } else {
                     let catalog = Arc::new(
@@ -181,51 +183,64 @@ impl Instance {
                     (
                         catalog.clone() as CatalogManagerRef,
                         Some(catalog as TableIdProviderRef),
+                        None,
                     )
                 }
             }
 
             Mode::Distributed => {
-                let kv_backend = Arc::new(CachedMetaKvBackend::new(
-                    meta_client.as_ref().unwrap().clone(),
+                let meta_client = meta_client.context(IncorrectInternalStateSnafu {
+                    state: "meta client is not provided when creating distributed Datanode",
+                })?;
+
+                let kv_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
+
+                let heartbeat_interval_millis = 5000;
+
+                let region_alive_keepers = Arc::new(RegionAliveKeepers::new(
+                    engine_manager.clone(),
+                    heartbeat_interval_millis,
                 ));
 
-                let catalog = Arc::new(catalog::remote::RemoteCatalogManager::new(
+                let catalog_manager = Arc::new(RemoteCatalogManager::new(
                     engine_manager.clone(),
                     opts.node_id.context(MissingNodeIdSnafu)?,
                     kv_backend,
+                    region_alive_keepers.clone(),
                 ));
-                (catalog as CatalogManagerRef, None)
+
+                let handlers_executor = HandlerGroupExecutor::new(vec![
+                    Arc::new(ParseMailboxMessageHandler::default()),
+                    Arc::new(OpenRegionHandler::new(
+                        catalog_manager.clone(),
+                        engine_manager.clone(),
+                        region_alive_keepers.clone(),
+                    )),
+                    Arc::new(CloseRegionHandler::new(
+                        catalog_manager.clone(),
+                        engine_manager.clone(),
+                        region_alive_keepers.clone(),
+                    )),
+                    region_alive_keepers.clone(),
+                ]);
+
+                let heartbeat_task = Some(HeartbeatTask::new(
+                    opts.node_id.context(MissingNodeIdSnafu)?,
+                    opts,
+                    meta_client,
+                    catalog_manager.clone(),
+                    Arc::new(handlers_executor),
+                    heartbeat_interval_millis,
+                    region_alive_keepers,
+                ));
+
+                (catalog_manager as CatalogManagerRef, None, heartbeat_task)
             }
         };
 
         let factory = QueryEngineFactory::new(catalog_manager.clone(), false);
         let query_engine = factory.query_engine();
 
-        let handlers_executor = HandlerGroupExecutor::new(vec![
-            Arc::new(ParseMailboxMessageHandler::default()),
-            Arc::new(OpenRegionHandler::new(
-                catalog_manager.clone(),
-                engine_manager.clone(),
-            )),
-            Arc::new(CloseRegionHandler::new(
-                catalog_manager.clone(),
-                engine_manager.clone(),
-            )),
-        ]);
-
-        let heartbeat_task = match opts.mode {
-            Mode::Standalone => None,
-            Mode::Distributed => Some(HeartbeatTask::new(
-                opts.node_id.context(MissingNodeIdSnafu)?,
-                opts.rpc_addr.clone(),
-                opts.rpc_hostname.clone(),
-                meta_client.as_ref().unwrap().clone(),
-                catalog_manager.clone(),
-                Arc::new(handlers_executor),
-            )),
-        };
-
         let procedure_manager =
             create_procedure_manager(opts.node_id.unwrap_or(0), &opts.procedure, object_store)
                 .await?;
@@ -354,7 +369,7 @@ impl Instance {
 fn create_compaction_scheduler<S: LogStore>(opts: &DatanodeOptions) -> CompactionSchedulerRef<S> {
     let picker = SimplePicker::default();
     let config = SchedulerConfig::from(opts);
-    let handler = CompactionHandler::new(picker);
+    let handler = CompactionHandler { picker };
     let scheduler = LocalScheduler::new(config, handler);
     Arc::new(scheduler)
 }
@@ -421,6 +436,7 @@ pub(crate) async fn create_log_store(
 
     let logstore = RaftEngineLogStore::try_new(log_config)
         .await
+        .map_err(Box::new)
         .context(OpenLogStoreSnafu)?;
     Ok(logstore)
 }

src/datanode/src/instance/sql.rs

@@ -228,6 +228,22 @@ pub fn table_idents_to_full_name(
     }
 }
 
+pub fn idents_to_full_database_name(
+    obj_name: &ObjectName,
+    query_ctx: &QueryContextRef,
+) -> Result<(String, String)> {
+    match &obj_name.0[..] {
+        [database] => Ok((query_ctx.current_catalog(), database.value.clone())),
+        [catalog, database] => Ok((catalog.value.clone(), database.value.clone())),
+        _ => error::InvalidSqlSnafu {
+            msg: format!(
+                "expect database name to be <catalog>.<database>, <database>, found: {obj_name}",
+            ),
+        }
+        .fail(),
+    }
+}
+
 #[async_trait]
 impl SqlStatementExecutor for Instance {
     async fn execute_sql(

src/datanode/src/tests.rs

@@ -14,26 +14,29 @@
 
 use std::assert_matches::assert_matches;
 use std::sync::Arc;
+use std::time::Duration;
 
 use api::v1::greptime_request::Request as GrpcRequest;
 use api::v1::meta::HeartbeatResponse;
 use api::v1::query_request::Query;
 use api::v1::QueryRequest;
+use catalog::remote::region_alive_keeper::RegionAliveKeepers;
 use catalog::CatalogManagerRef;
 use common_meta::heartbeat::handler::{
     HandlerGroupExecutor, HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutor,
 };
 use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MessageMeta};
-use common_meta::instruction::{
-    Instruction, InstructionReply, RegionIdent, SimpleReply, TableIdent,
-};
+use common_meta::ident::TableIdent;
+use common_meta::instruction::{Instruction, InstructionReply, RegionIdent, SimpleReply};
 use common_query::Output;
 use datatypes::prelude::ConcreteDataType;
 use servers::query_handler::grpc::GrpcQueryHandler;
 use session::context::QueryContext;
 use table::engine::manager::TableEngineManagerRef;
+use table::TableRef;
 use test_util::MockInstance;
 use tokio::sync::mpsc::{self, Receiver};
+use tokio::time::Instant;
 
 use crate::heartbeat::handler::close_region::CloseRegionHandler;
 use crate::heartbeat::handler::open_region::OpenRegionHandler;
@@ -61,7 +64,11 @@ async fn test_close_region_handler() {
     } = prepare_handler_test("test_close_region_handler").await;
 
     let executor = Arc::new(HandlerGroupExecutor::new(vec![Arc::new(
-        CloseRegionHandler::new(catalog_manager_ref.clone(), engine_manager_ref.clone()),
+        CloseRegionHandler::new(
+            catalog_manager_ref.clone(),
+            engine_manager_ref.clone(),
+            Arc::new(RegionAliveKeepers::new(engine_manager_ref.clone(), 5000)),
+        ),
     )]));
 
     prepare_table(instance.inner()).await;
@@ -71,7 +78,8 @@ async fn test_close_region_handler() {
         executor.clone(),
         mailbox.clone(),
         close_region_instruction(),
-    );
+    )
+    .await;
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
         reply,
@@ -85,7 +93,8 @@ async fn test_close_region_handler() {
         executor.clone(),
         mailbox.clone(),
         close_region_instruction(),
-    );
+    )
+    .await;
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
         reply,
@@ -108,7 +117,8 @@ async fn test_close_region_handler() {
             cluster_id: 1,
             datanode_id: 2,
         }),
-    );
+    )
+    .await;
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
         reply,
@@ -127,56 +137,81 @@ async fn test_open_region_handler() {
         ..
     } = prepare_handler_test("test_open_region_handler").await;
 
+    let region_alive_keepers = Arc::new(RegionAliveKeepers::new(engine_manager_ref.clone(), 5000));
+    region_alive_keepers.start().await;
+
     let executor = Arc::new(HandlerGroupExecutor::new(vec![
         Arc::new(OpenRegionHandler::new(
             catalog_manager_ref.clone(),
             engine_manager_ref.clone(),
+            region_alive_keepers.clone(),
         )),
         Arc::new(CloseRegionHandler::new(
             catalog_manager_ref.clone(),
             engine_manager_ref.clone(),
+            region_alive_keepers.clone(),
         )),
     ]));
 
-    prepare_table(instance.inner()).await;
+    let instruction = open_region_instruction();
+    let Instruction::OpenRegion(region_ident) = instruction.clone() else { unreachable!() };
+    let table_ident = &region_ident.table_ident;
+
+    let table = prepare_table(instance.inner()).await;
+    region_alive_keepers
+        .register_table(table_ident.clone(), table)
+        .await
+        .unwrap();
 
     // Opens a opened table
-    handle_instruction(executor.clone(), mailbox.clone(), open_region_instruction());
+    handle_instruction(executor.clone(), mailbox.clone(), instruction.clone()).await;
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
         reply,
         InstructionReply::OpenRegion(SimpleReply { result: true, .. })
     );
 
+    let keeper = region_alive_keepers.find_keeper(table_ident).await.unwrap();
+    let deadline = keeper.deadline(0).await.unwrap();
+    assert!(deadline <= Instant::now() + Duration::from_secs(20));
+
     // Opens a non-exist table
+    let non_exist_table_ident = TableIdent {
+        catalog: "greptime".to_string(),
+        schema: "public".to_string(),
+        table: "non-exist".to_string(),
+        table_id: 2024,
+        engine: "mito".to_string(),
+    };
     handle_instruction(
         executor.clone(),
         mailbox.clone(),
         Instruction::OpenRegion(RegionIdent {
-            table_ident: TableIdent {
-                catalog: "greptime".to_string(),
-                schema: "public".to_string(),
-                table: "non-exist".to_string(),
-                table_id: 2024,
-                engine: "mito".to_string(),
-            },
+            table_ident: non_exist_table_ident.clone(),
             region_number: 0,
             cluster_id: 1,
             datanode_id: 2,
         }),
-    );
+    )
+    .await;
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
         reply,
         InstructionReply::OpenRegion(SimpleReply { result: false, .. })
     );
 
+    assert!(region_alive_keepers
+        .find_keeper(&non_exist_table_ident)
+        .await
+        .is_none());
+
     // Closes demo table
     handle_instruction(
         executor.clone(),
         mailbox.clone(),
         close_region_instruction(),
-    );
+    )
+    .await;
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
         reply,
@@ -184,8 +219,13 @@ async fn test_open_region_handler() {
     );
     assert_test_table_not_found(instance.inner()).await;
 
+    assert!(region_alive_keepers
+        .find_keeper(table_ident)
+        .await
+        .is_none());
+
     // Opens demo table
-    handle_instruction(executor.clone(), mailbox.clone(), open_region_instruction());
+    handle_instruction(executor.clone(), mailbox.clone(), instruction).await;
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
         reply,
@@ -220,7 +260,7 @@ pub fn test_message_meta(id: u64, subject: &str, to: &str, from: &str) -> Messag
     }
 }
 
-fn handle_instruction(
+async fn handle_instruction(
     executor: Arc<dyn HeartbeatResponseHandlerExecutor>,
     mailbox: Arc<HeartbeatMailbox>,
     instruction: Instruction,
@@ -229,7 +269,7 @@ fn handle_instruction(
     let mut ctx: HeartbeatResponseHandlerContext =
         HeartbeatResponseHandlerContext::new(mailbox, response);
     ctx.incoming_message = Some((test_message_meta(1, "hi", "foo", "bar"), instruction));
-    executor.handle(ctx).unwrap();
+    executor.handle(ctx).await.unwrap();
 }
 
 fn close_region_instruction() -> Instruction {
@@ -262,10 +302,10 @@ fn open_region_instruction() -> Instruction {
     })
 }
 
-async fn prepare_table(instance: &Instance) {
+async fn prepare_table(instance: &Instance) -> TableRef {
     test_util::create_test_table(instance, ConcreteDataType::timestamp_millisecond_datatype())
         .await
-        .unwrap();
+        .unwrap()
 }
 
 async fn assert_test_table_not_found(instance: &Instance) {

src/datanode/src/tests/test_util.rs

@@ -22,6 +22,7 @@ use servers::Mode;
 use snafu::ResultExt;
 use table::engine::{EngineContext, TableEngineRef};
 use table::requests::{CreateTableRequest, TableOptions};
+use table::TableRef;
 
 use crate::datanode::{
     DatanodeOptions, FileConfig, ObjectStoreConfig, ProcedureConfig, StorageConfig, WalConfig,
@@ -84,7 +85,7 @@ fn create_tmp_dir_and_datanode_opts(name: &str) -> (DatanodeOptions, TestGuard)
 pub(crate) async fn create_test_table(
     instance: &Instance,
     ts_type: ConcreteDataType,
-) -> Result<()> {
+) -> Result<TableRef> {
     let column_schemas = vec![
         ColumnSchema::new("host", ConcreteDataType::string_datatype(), true),
         ColumnSchema::new("cpu", ConcreteDataType::float64_datatype(), true),
@@ -125,8 +126,8 @@ pub(crate) async fn create_test_table(
         .unwrap()
         .unwrap();
     schema_provider
-        .register_table(table_name.to_string(), table)
+        .register_table(table_name.to_string(), table.clone())
         .await
         .unwrap();
-    Ok(())
+    Ok(table)
 }

src/datatypes/src/data_type.rs

@@ -183,6 +183,12 @@ impl ConcreteDataType {
     }
 }
 
+impl From<&ConcreteDataType> for ConcreteDataType {
+    fn from(t: &ConcreteDataType) -> Self {
+        t.clone()
+    }
+}
+
 impl TryFrom<&ArrowDataType> for ConcreteDataType {
     type Error = Error;
 

src/datatypes/src/value.rs

@@ -248,7 +248,7 @@ impl Value {
             Value::Binary(v) => ScalarValue::LargeBinary(Some(v.to_vec())),
             Value::Date(v) => ScalarValue::Date32(Some(v.val())),
             Value::DateTime(v) => ScalarValue::Date64(Some(v.val())),
-            Value::Null => to_null_value(output_type),
+            Value::Null => to_null_scalar_value(output_type),
             Value::List(list) => {
                 // Safety: The logical type of the value and output_type are the same.
                 let list_type = output_type.as_list().unwrap();
@@ -261,7 +261,7 @@ impl Value {
     }
 }
 
-fn to_null_value(output_type: &ConcreteDataType) -> ScalarValue {
+pub fn to_null_scalar_value(output_type: &ConcreteDataType) -> ScalarValue {
     match output_type {
         ConcreteDataType::Null(_) => ScalarValue::Null,
         ConcreteDataType::Boolean(_) => ScalarValue::Boolean(None),
@@ -285,7 +285,7 @@ fn to_null_value(output_type: &ConcreteDataType) -> ScalarValue {
         }
         ConcreteDataType::Dictionary(dict) => ScalarValue::Dictionary(
             Box::new(dict.key_type().as_arrow_type()),
-            Box::new(to_null_value(dict.value_type())),
+            Box::new(to_null_scalar_value(dict.value_type())),
         ),
     }
 }

src/datatypes/src/vectors/binary.rs

@@ -113,6 +113,14 @@ impl Vector for BinaryVector {
     }
 }
 
+impl From<Vec<Vec<u8>>> for BinaryVector {
+    fn from(data: Vec<Vec<u8>>) -> Self {
+        Self {
+            array: BinaryArray::from_iter_values(data),
+        }
+    }
+}
+
 impl ScalarVector for BinaryVector {
     type OwnedItem = Vec<u8>;
     type RefItem<'a> = &'a [u8];

src/datatypes/src/vectors/primitive.rs

@@ -130,6 +130,12 @@ impl<T: LogicalPrimitiveType> PrimitiveVector<T> {
         }
     }
 
+    pub fn from_iter_values<I: IntoIterator<Item = T::Native>>(iter: I) -> Self {
+        Self {
+            array: PrimitiveArray::from_iter_values(iter),
+        }
+    }
+
     pub fn from_values<I: IntoIterator<Item = T::Native>>(iter: I) -> Self {
         Self {
             array: PrimitiveArray::from_iter_values(iter),

src/file-table-engine/src/error.rs

@@ -115,13 +115,13 @@ pub enum Error {
         source
     ))]
     ConvertRaw {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::metadata::ConvertError,
     },
 
     #[snafu(display("Invalid schema, source: {}", source))]
     InvalidRawSchema {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -130,7 +130,7 @@ pub enum Error {
 
     #[snafu(display("Failed to build backend, source: {}", source))]
     BuildBackend {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_datasource::error::Error,
     },
 
@@ -154,13 +154,13 @@ pub enum Error {
 
     #[snafu(display("Failed to build stream adapter: {}", source))]
     BuildStreamAdapter {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
     #[snafu(display("Failed to parse file format: {}", source))]
     ParseFileFormat {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_datasource::error::Error,
     },
 

src/frontend/Cargo.toml

@@ -29,6 +29,7 @@ common-meta = { path = "../common/meta" }
 common-recordbatch = { path = "../common/recordbatch" }
 common-runtime = { path = "../common/runtime" }
 common-telemetry = { path = "../common/telemetry" }
+common-time = { path = "../common/time" }
 datafusion.workspace = true
 datafusion-common.workspace = true
 datafusion-expr.workspace = true

src/frontend/src/error.rs

@@ -279,6 +279,13 @@ pub enum Error {
         source: query::error::Error,
     },
 
+    #[snafu(display("Failed to read table: {table_name}, source: {source}"))]
+    ReadTable {
+        table_name: String,
+        #[snafu(backtrace)]
+        source: query::error::Error,
+    },
+
     #[snafu(display("Failed to execute logical plan, source: {}", source))]
     ExecLogicalPlan {
         #[snafu(backtrace)]
@@ -363,13 +370,22 @@ pub enum Error {
     },
 
     // TODO(ruihang): merge all query execution error kinds
-    #[snafu(display("failed to execute PromQL query {}, source: {}", query, source))]
+    #[snafu(display("Failed to execute PromQL query {}, source: {}", query, source))]
     ExecutePromql {
         query: String,
         #[snafu(backtrace)]
         source: servers::error::Error,
     },
 
+    #[snafu(display(
+        "Failed to create logical plan for prometheus query, source: {}",
+        source
+    ))]
+    PrometheusRemoteQueryPlan {
+        #[snafu(backtrace)]
+        source: servers::error::Error,
+    },
+
     #[snafu(display("Failed to describe schema for given statement, source: {}", source))]
     DescribeStatement {
         #[snafu(backtrace)]
@@ -532,6 +548,13 @@ pub enum Error {
         #[snafu(backtrace)]
         source: query::error::Error,
     },
+
+    #[snafu(display("Invalid COPY parameter, key: {}, value: {}", key, value))]
+    InvalidCopyParameter {
+        key: String,
+        value: String,
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -559,7 +582,8 @@ impl ErrorExt for Error {
             Error::HandleHeartbeatResponse { source, .. } => source.status_code(),
 
             Error::RuntimeResource { source, .. } => source.status_code(),
-            Error::ExecutePromql { source, .. } => source.status_code(),
+            Error::PrometheusRemoteQueryPlan { source, .. }
+            | Error::ExecutePromql { source, .. } => source.status_code(),
 
             Error::SqlExecIntercepted { source, .. } => source.status_code(),
             Error::StartServer { source, .. } => source.status_code(),
@@ -621,6 +645,7 @@ impl ErrorExt for Error {
             Error::ExecuteStatement { source, .. }
             | Error::PlanStatement { source }
             | Error::ParseQuery { source }
+            | Error::ReadTable { source, .. }
             | Error::ExecLogicalPlan { source }
             | Error::DescribeStatement { source } => source.status_code(),
 
@@ -649,6 +674,7 @@ impl ErrorExt for Error {
             | Error::BuildBackend { source } => source.status_code(),
 
             Error::WriteParquet { source, .. } => source.status_code(),
+            Error::InvalidCopyParameter { .. } => StatusCode::InvalidArguments,
         }
     }
 

src/frontend/src/heartbeat.rs

@@ -20,8 +20,7 @@ use common_meta::heartbeat::handler::{
 };
 use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MailboxRef, OutgoingMessage};
 use common_meta::heartbeat::utils::outgoing_message_to_mailbox_message;
-use common_telemetry::tracing::trace;
-use common_telemetry::{error, info};
+use common_telemetry::{debug, error, info};
 use meta_client::client::{HeartbeatSender, HeartbeatStream, MetaClient};
 use snafu::ResultExt;
 use tokio::sync::mpsc;
@@ -83,16 +82,15 @@ impl HeartbeatTask {
             loop {
                 match resp_stream.message().await {
                     Ok(Some(resp)) => {
-                        trace!("Received a heartbeat response: {:?}", resp);
+                        debug!("Receiving heartbeat response: {:?}", resp);
                         let ctx = HeartbeatResponseHandlerContext::new(mailbox.clone(), resp);
-                        if let Err(e) = capture_self.handle_response(ctx) {
+                        if let Err(e) = capture_self.handle_response(ctx).await {
                             error!(e; "Error while handling heartbeat response");
                         }
                     }
                     Ok(None) => break,
                     Err(e) => {
                         error!(e; "Occur error while reading heartbeat response");
-
                         capture_self
                             .start_with_retry(Duration::from_secs(retry_interval))
                             .await;
@@ -148,16 +146,17 @@ impl HeartbeatTask {
                         error!(e; "Failed to send heartbeat to metasrv");
                         break;
                     } else {
-                        trace!("Send a heartbeat request to metasrv, content: {:?}", req);
+                        debug!("Send a heartbeat request to metasrv, content: {:?}", req);
                     }
                 }
             }
         });
     }
 
-    fn handle_response(&self, ctx: HeartbeatResponseHandlerContext) -> Result<()> {
+    async fn handle_response(&self, ctx: HeartbeatResponseHandlerContext) -> Result<()> {
         self.resp_handler_executor
             .handle(ctx)
+            .await
             .context(error::HandleHeartbeatResponseSnafu)
     }
 

src/frontend/src/heartbeat/handler/invalidate_table_cache.rs

@@ -12,13 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use async_trait::async_trait;
 use catalog::helper::TableGlobalKey;
 use catalog::remote::KvCacheInvalidatorRef;
 use common_meta::error::Result as MetaResult;
 use common_meta::heartbeat::handler::{
     HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
 };
-use common_meta::instruction::{Instruction, InstructionReply, SimpleReply, TableIdent};
+use common_meta::ident::TableIdent;
+use common_meta::instruction::{Instruction, InstructionReply, SimpleReply};
 use common_meta::table_name::TableName;
 use common_telemetry::{error, info};
 use partition::manager::TableRouteCacheInvalidatorRef;
@@ -29,6 +31,7 @@ pub struct InvalidateTableCacheHandler {
     table_route_cache_invalidator: TableRouteCacheInvalidatorRef,
 }
 
+#[async_trait]
 impl HeartbeatResponseHandler for InvalidateTableCacheHandler {
     fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
         matches!(
@@ -37,7 +40,7 @@ impl HeartbeatResponseHandler for InvalidateTableCacheHandler {
         )
     }
 
-    fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> MetaResult<HandleControl> {
+    async fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> MetaResult<HandleControl> {
         // TODO(weny): considers introducing a macro
         let Some((meta, Instruction::InvalidateTableCache(table_ident))) = ctx.incoming_message.take() else {
             unreachable!("InvalidateTableCacheHandler: should be guarded by 'is_acceptable'");

src/frontend/src/heartbeat/handler/tests.rs

@@ -23,7 +23,8 @@ use common_meta::heartbeat::handler::{
     HandlerGroupExecutor, HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutor,
 };
 use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MessageMeta};
-use common_meta::instruction::{Instruction, InstructionReply, SimpleReply, TableIdent};
+use common_meta::ident::TableIdent;
+use common_meta::instruction::{Instruction, InstructionReply, SimpleReply};
 use common_meta::table_name::TableName;
 use partition::manager::TableRouteCacheInvalidator;
 use tokio::sync::mpsc;
@@ -89,7 +90,8 @@ async fn test_invalidate_table_cache_handler() {
             table_id: 0,
             engine: "mito".to_string(),
         }),
-    );
+    )
+    .await;
 
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
@@ -125,7 +127,8 @@ async fn test_invalidate_table_cache_handler() {
             table_id: 0,
             engine: "mito".to_string(),
         }),
-    );
+    )
+    .await;
 
     let (_, reply) = rx.recv().await.unwrap();
     assert_matches!(
@@ -143,7 +146,7 @@ pub fn test_message_meta(id: u64, subject: &str, to: &str, from: &str) -> Messag
     }
 }
 
-fn handle_instruction(
+async fn handle_instruction(
     executor: Arc<dyn HeartbeatResponseHandlerExecutor>,
     mailbox: Arc<HeartbeatMailbox>,
     instruction: Instruction,
@@ -152,5 +155,5 @@ fn handle_instruction(
     let mut ctx: HeartbeatResponseHandlerContext =
         HeartbeatResponseHandlerContext::new(mailbox, response);
     ctx.incoming_message = Some((test_message_meta(1, "hi", "foo", "bar"), instruction));
-    executor.handle(ctx).unwrap();
+    executor.handle(ctx).await.unwrap();
 }

src/frontend/src/instance.rs

@@ -53,7 +53,9 @@ use meta_client::MetaClientOptions;
 use partition::manager::PartitionRuleManager;
 use partition::route::TableRoutes;
 use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
+use query::plan::LogicalPlan;
 use query::query_engine::options::{validate_catalog_and_schema, QueryOptions};
+use query::query_engine::DescribeResult;
 use query::{QueryEngineFactory, QueryEngineRef};
 use servers::error as server_error;
 use servers::error::{ExecuteQuerySnafu, ParsePromQLSnafu};
@@ -73,8 +75,9 @@ use sql::statements::statement::Statement;
 
 use crate::catalog::FrontendCatalogManager;
 use crate::error::{
-    self, Error, ExecutePromqlSnafu, ExternalSnafu, InvalidInsertRequestSnafu,
-    MissingMetasrvOptsSnafu, ParseSqlSnafu, PlanStatementSnafu, Result, SqlExecInterceptedSnafu,
+    self, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExternalSnafu,
+    InvalidInsertRequestSnafu, MissingMetasrvOptsSnafu, ParseSqlSnafu, PlanStatementSnafu, Result,
+    SqlExecInterceptedSnafu,
 };
 use crate::expr_factory::{CreateExprFactoryRef, DefaultCreateExprFactory};
 use crate::frontend::FrontendOptions;
@@ -285,9 +288,6 @@ impl Instance {
         requests: InsertRequests,
         ctx: QueryContextRef,
     ) -> Result<Output> {
-        // TODO(LFC): Optimize concurrent table creation and table alteration.
-        // Currently table creation is guarded by a distributed lock in Metasrv. However, table
-        // alteration is not. We should all switch to procedures in Metasrv.
         let _ = future::join_all(
             requests
                 .inserts
@@ -509,6 +509,14 @@ impl SqlQueryHandler for Instance {
         }
     }
 
+    async fn do_exec_plan(&self, plan: LogicalPlan, query_ctx: QueryContextRef) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_EXEC_PLAN_ELAPSED);
+        self.query_engine
+            .execute(plan, query_ctx)
+            .await
+            .context(ExecLogicalPlanSnafu)
+    }
+
     async fn do_promql_query(
         &self,
         query: &PromQuery,
@@ -526,8 +534,11 @@ impl SqlQueryHandler for Instance {
         &self,
         stmt: Statement,
         query_ctx: QueryContextRef,
-    ) -> Result<Option<Schema>> {
-        if let Statement::Query(_) = stmt {
+    ) -> Result<Option<DescribeResult>> {
+        if matches!(
+            stmt,
+            Statement::Insert(_) | Statement::Query(_) | Statement::Delete(_)
+        ) {
             let plan = self
                 .query_engine
                 .planner()
@@ -563,6 +574,7 @@ impl PromHandler for Instance {
         let stmt = QueryLanguageParser::parse_promql(query).with_context(|_| ParsePromQLSnafu {
             query: query.clone(),
         })?;
+
         self.statement_executor
             .execute_stmt(stmt, query_ctx)
             .await
@@ -615,12 +627,15 @@ pub fn check_permission(
         Statement::DescribeTable(stmt) => {
             validate_param(stmt.name(), query_ctx)?;
         }
-        Statement::Copy(stmd) => match stmd {
+        Statement::Copy(sql::statements::copy::Copy::CopyTable(stmt)) => match stmt {
             CopyTable::To(copy_table_to) => validate_param(&copy_table_to.table_name, query_ctx)?,
             CopyTable::From(copy_table_from) => {
                 validate_param(&copy_table_from.table_name, query_ctx)?
             }
         },
+        Statement::Copy(sql::statements::copy::Copy::CopyDatabase(stmt)) => {
+            validate_param(&stmt.database_name, query_ctx)?
+        }
     }
     Ok(())
 }

src/frontend/src/instance/distributed.rs

@@ -598,7 +598,6 @@ impl DistInstance {
         Ok(Output::AffectedRows(affected_rows as usize))
     }
 
-    // TODO(LFC): Like insertions above, refactor GRPC deletion impl here.
     async fn handle_dist_delete(
         &self,
         request: DeleteRequest,
@@ -662,8 +661,6 @@ impl GrpcQueryHandler for DistInstance {
                 match expr {
                     DdlExpr::CreateDatabase(expr) => self.handle_create_database(expr, ctx).await,
                     DdlExpr::CreateTable(mut expr) => {
-                        // TODO(LFC): Support creating distributed table through GRPC interface.
-                        // Currently only SQL supports it; how to design the fields in CreateTableExpr?
                         let _ = self.create_table(&mut expr, None).await;
                         Ok(Output::AffectedRows(0))
                     }

src/frontend/src/instance/prometheus.rs

@@ -14,9 +14,8 @@
 
 use api::prometheus::remote::read_request::ResponseType;
 use api::prometheus::remote::{Query, QueryResult, ReadRequest, ReadResponse, WriteRequest};
-use api::v1::greptime_request::Request;
-use api::v1::{query_request, QueryRequest};
 use async_trait::async_trait;
+use common_catalog::format_full_table_name;
 use common_error::prelude::BoxedError;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
@@ -25,11 +24,14 @@ use metrics::counter;
 use prost::Message;
 use servers::error::{self, Result as ServerResult};
 use servers::prometheus::{self, Metrics};
-use servers::query_handler::grpc::GrpcQueryHandler;
 use servers::query_handler::{PrometheusProtocolHandler, PrometheusResponse};
 use session::context::QueryContextRef;
 use snafu::{OptionExt, ResultExt};
 
+use crate::error::{
+    CatalogSnafu, ExecLogicalPlanSnafu, PrometheusRemoteQueryPlanSnafu, ReadTableSnafu, Result,
+    TableNotFoundSnafu,
+};
 use crate::instance::Instance;
 use crate::metrics::PROMETHEUS_REMOTE_WRITE_SAMPLES;
 
@@ -75,6 +77,45 @@ async fn to_query_result(table_name: &str, output: Output) -> ServerResult<Query
 }
 
 impl Instance {
+    async fn handle_remote_query(
+        &self,
+        ctx: &QueryContextRef,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+        query: &Query,
+    ) -> Result<Output> {
+        let table = self
+            .catalog_manager
+            .table(catalog_name, schema_name, table_name)
+            .await
+            .context(CatalogSnafu)?
+            .with_context(|| TableNotFoundSnafu {
+                table_name: format_full_table_name(catalog_name, schema_name, table_name),
+            })?;
+
+        let dataframe = self
+            .query_engine
+            .read_table(table)
+            .with_context(|_| ReadTableSnafu {
+                table_name: format_full_table_name(catalog_name, schema_name, table_name),
+            })?;
+
+        let logical_plan =
+            prometheus::query_to_plan(dataframe, query).context(PrometheusRemoteQueryPlanSnafu)?;
+
+        logging::debug!(
+            "Prometheus remote read, table: {}, logical plan: {}",
+            table_name,
+            logical_plan.display_indent(),
+        );
+
+        self.query_engine
+            .execute(logical_plan, ctx.clone())
+            .await
+            .context(ExecLogicalPlanSnafu)
+    }
+
     async fn handle_remote_queries(
         &self,
         ctx: QueryContextRef,
@@ -82,22 +123,19 @@ impl Instance {
     ) -> ServerResult<Vec<(String, Output)>> {
         let mut results = Vec::with_capacity(queries.len());
 
-        for query in queries {
-            let (table_name, sql) = prometheus::query_to_sql(query)?;
-            logging::debug!(
-                "prometheus remote read, table: {}, sql: {}",
-                table_name,
-                sql
-            );
+        let catalog_name = ctx.current_catalog();
+        let schema_name = ctx.current_schema();
+
+        for query in queries {
+            let table_name = prometheus::table_name(query)?;
 
-            let query = Request::Query(QueryRequest {
-                query: Some(query_request::Query::Sql(sql.to_string())),
-            });
             let output = self
-                .do_query(query, ctx.clone())
+                .handle_remote_query(&ctx, &catalog_name, &schema_name, &table_name, query)
                 .await
                 .map_err(BoxedError::new)
-                .context(error::ExecuteGrpcQuerySnafu)?;
+                .with_context(|_| error::ExecuteQuerySnafu {
+                    query: format!("{query:#?}"),
+                })?;
 
             results.push((table_name, output));
         }

src/frontend/src/metrics.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 pub(crate) const METRIC_HANDLE_SQL_ELAPSED: &str = "frontend.handle_sql_elapsed";
+pub(crate) const METRIC_EXEC_PLAN_ELAPSED: &str = "frontend.exec_plan_elapsed";
 pub(crate) const METRIC_HANDLE_SCRIPTS_ELAPSED: &str = "frontend.handle_scripts_elapsed";
 pub(crate) const METRIC_RUN_SCRIPT_ELAPSED: &str = "frontend.run_script_elapsed";
 

src/frontend/src/statement.rs

@@ -12,32 +12,40 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod backup;
 mod copy_table_from;
 mod copy_table_to;
 mod describe;
 mod show;
 mod tql;
 
+use std::collections::HashMap;
+use std::str::FromStr;
+
 use catalog::CatalogManagerRef;
 use common_error::prelude::BoxedError;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
-use datanode::instance::sql::table_idents_to_full_name;
+use common_time::range::TimestampRange;
+use common_time::Timestamp;
+use datanode::instance::sql::{idents_to_full_database_name, table_idents_to_full_name};
 use query::parser::QueryStatement;
 use query::query_engine::SqlStatementExecutorRef;
 use query::QueryEngineRef;
 use session::context::QueryContextRef;
 use snafu::{ensure, OptionExt, ResultExt};
-use sql::statements::copy::{CopyTable, CopyTableArgument};
+use sql::statements::copy::{CopyDatabaseArgument, CopyTable, CopyTableArgument};
 use sql::statements::statement::Statement;
 use table::engine::TableReference;
-use table::requests::{CopyDirection, CopyTableRequest};
+use table::requests::{CopyDatabaseRequest, CopyDirection, CopyTableRequest};
 use table::TableRef;
 
+use crate::error;
 use crate::error::{
     CatalogSnafu, ExecLogicalPlanSnafu, ExecuteStatementSnafu, ExternalSnafu, PlanStatementSnafu,
     Result, SchemaNotFoundSnafu, TableNotFoundSnafu,
 };
+use crate::statement::backup::{COPY_DATABASE_TIME_END_KEY, COPY_DATABASE_TIME_START_KEY};
 
 #[derive(Clone)]
 pub struct StatementExecutor {
@@ -92,14 +100,23 @@ impl StatementExecutor {
 
             Statement::ShowTables(stmt) => self.show_tables(stmt, query_ctx).await,
 
-            Statement::Copy(stmt) => {
+            Statement::Copy(sql::statements::copy::Copy::CopyTable(stmt)) => {
                 let req = to_copy_table_request(stmt, query_ctx)?;
                 match req.direction {
-                    CopyDirection::Export => self.copy_table_to(req).await,
-                    CopyDirection::Import => self.copy_table_from(req).await,
+                    CopyDirection::Export => {
+                        self.copy_table_to(req).await.map(Output::AffectedRows)
+                    }
+                    CopyDirection::Import => {
+                        self.copy_table_from(req).await.map(Output::AffectedRows)
+                    }
                 }
             }
 
+            Statement::Copy(sql::statements::copy::Copy::CopyDatabase(arg)) => {
+                self.copy_database(to_copy_database_request(arg, &query_ctx)?)
+                    .await
+            }
+
             Statement::CreateDatabase(_)
             | Statement::CreateTable(_)
             | Statement::CreateExternalTable(_)
@@ -191,5 +208,47 @@ fn to_copy_table_request(stmt: CopyTable, query_ctx: QueryContextRef) -> Result<
         connection,
         pattern,
         direction,
+        // we copy the whole table by default.
+        timestamp_range: None,
     })
 }
+
+/// Converts [CopyDatabaseArgument] to [CopyDatabaseRequest].
+/// This function extracts the necessary info including catalog/database name, time range, etc.  
+fn to_copy_database_request(
+    arg: CopyDatabaseArgument,
+    query_ctx: &QueryContextRef,
+) -> Result<CopyDatabaseRequest> {
+    let (catalog_name, database_name) = idents_to_full_database_name(&arg.database_name, query_ctx)
+        .map_err(BoxedError::new)
+        .context(ExternalSnafu)?;
+
+    let start_timestamp = extract_timestamp(&arg.with, COPY_DATABASE_TIME_START_KEY)?;
+    let end_timestamp = extract_timestamp(&arg.with, COPY_DATABASE_TIME_END_KEY)?;
+
+    let time_range = match (start_timestamp, end_timestamp) {
+        (Some(start), Some(end)) => TimestampRange::new(start, end),
+        (Some(start), None) => Some(TimestampRange::from_start(start)),
+        (None, Some(end)) => Some(TimestampRange::until_end(end, false)), // exclusive end
+        (None, None) => None,
+    };
+
+    Ok(CopyDatabaseRequest {
+        catalog_name,
+        schema_name: database_name,
+        location: arg.location,
+        with: arg.with,
+        connection: arg.connection,
+        time_range,
+    })
+}
+
+/// Extracts timestamp from a [HashMap<String, String>] with given key.
+fn extract_timestamp(map: &HashMap<String, String>, key: &str) -> Result<Option<Timestamp>> {
+    map.get(key)
+        .map(|v| {
+            Timestamp::from_str(v)
+                .map_err(|_| error::InvalidCopyParameterSnafu { key, value: v }.build())
+        })
+        .transpose()
+}

src/frontend/src/statement/backup.rs

@@ -0,0 +1,97 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_datasource::file_format::Format;
+use common_query::Output;
+use common_telemetry::info;
+use snafu::{ensure, OptionExt, ResultExt};
+use table::requests::{CopyDatabaseRequest, CopyDirection, CopyTableRequest};
+
+use crate::error;
+use crate::error::{
+    CatalogNotFoundSnafu, CatalogSnafu, InvalidCopyParameterSnafu, SchemaNotFoundSnafu,
+};
+use crate::statement::StatementExecutor;
+
+pub(crate) const COPY_DATABASE_TIME_START_KEY: &str = "start_time";
+pub(crate) const COPY_DATABASE_TIME_END_KEY: &str = "end_time";
+
+impl StatementExecutor {
+    pub(crate) async fn copy_database(&self, req: CopyDatabaseRequest) -> error::Result<Output> {
+        // location must end with / so that every table is exported to a file.
+        ensure!(
+            req.location.ends_with('/'),
+            InvalidCopyParameterSnafu {
+                key: "location",
+                value: req.location,
+            }
+        );
+
+        info!(
+            "Copy database {}.{}, dir: {},. time: {:?}",
+            req.catalog_name, req.schema_name, req.location, req.time_range
+        );
+        let schema = self
+            .catalog_manager
+            .catalog(&req.catalog_name)
+            .await
+            .context(CatalogSnafu)?
+            .context(CatalogNotFoundSnafu {
+                catalog_name: &req.catalog_name,
+            })?
+            .schema(&req.schema_name)
+            .await
+            .context(CatalogSnafu)?
+            .context(SchemaNotFoundSnafu {
+                schema_info: &req.schema_name,
+            })?;
+
+        let suffix = Format::try_from(&req.with)
+            .context(error::ParseFileFormatSnafu)?
+            .suffix();
+
+        let table_names = schema.table_names().await.context(CatalogSnafu)?;
+
+        let mut exported_rows = 0;
+        for table_name in table_names {
+            // TODO(hl): remove this hardcode once we've removed numbers table.
+            if table_name == "numbers" {
+                continue;
+            }
+            let mut table_file = req.location.clone();
+            table_file.push_str(&table_name);
+            table_file.push_str(suffix);
+            info!(
+                "Copy table: {}.{}.{} to {}",
+                req.catalog_name, req.schema_name, table_name, table_file
+            );
+
+            let exported = self
+                .copy_table_to(CopyTableRequest {
+                    catalog_name: req.catalog_name.clone(),
+                    schema_name: req.schema_name.clone(),
+                    table_name,
+                    location: table_file,
+                    with: req.with.clone(),
+                    connection: req.connection.clone(),
+                    pattern: None,
+                    direction: CopyDirection::Export,
+                    timestamp_range: req.time_range,
+                })
+                .await?;
+            exported_rows += exported;
+        }
+        Ok(Output::AffectedRows(exported_rows))
+    }
+}

src/frontend/src/statement/copy_table_from.rs

@@ -24,7 +24,6 @@ use common_datasource::file_format::{FileFormat, Format};
 use common_datasource::lister::{Lister, Source};
 use common_datasource::object_store::{build_backend, parse_url};
 use common_datasource::util::find_dir_and_filename;
-use common_query::Output;
 use common_recordbatch::adapter::ParquetRecordBatchStreamAdapter;
 use common_recordbatch::DfSendableRecordBatchStream;
 use datafusion::datasource::listing::PartitionedFile;
@@ -205,7 +204,7 @@ impl StatementExecutor {
         }
     }
 
-    pub async fn copy_table_from(&self, req: CopyTableRequest) -> Result<Output> {
+    pub async fn copy_table_from(&self, req: CopyTableRequest) -> Result<usize> {
         let table_ref = TableReference {
             catalog: &req.catalog_name,
             schema: &req.schema_name,
@@ -313,7 +312,7 @@ impl StatementExecutor {
             }
         }
 
-        Ok(Output::AffectedRows(rows_inserted))
+        Ok(rows_inserted)
     }
 }
 

src/frontend/src/statement/copy_table_to.rs

@@ -18,7 +18,6 @@ use common_datasource::file_format::json::stream_to_json;
 use common_datasource::file_format::Format;
 use common_datasource::object_store::{build_backend, parse_url};
 use common_query::physical_plan::SessionContext;
-use common_query::Output;
 use common_recordbatch::adapter::DfRecordBatchStreamAdapter;
 use common_recordbatch::SendableRecordBatchStream;
 use object_store::ObjectStore;
@@ -72,7 +71,7 @@ impl StatementExecutor {
         }
     }
 
-    pub(crate) async fn copy_table_to(&self, req: CopyTableRequest) -> Result<Output> {
+    pub(crate) async fn copy_table_to(&self, req: CopyTableRequest) -> Result<usize> {
         let table_ref = TableReference {
             catalog: &req.catalog_name,
             schema: &req.schema_name,
@@ -82,12 +81,25 @@ impl StatementExecutor {
 
         let format = Format::try_from(&req.with).context(error::ParseFileFormatSnafu)?;
 
-        let stream = table
-            .scan(None, &[], None)
-            .await
-            .with_context(|_| error::CopyTableSnafu {
-                table_name: table_ref.to_string(),
-            })?;
+        let filters = table
+            .schema()
+            .timestamp_column()
+            .and_then(|c| {
+                common_query::logical_plan::build_filter_from_timestamp(
+                    &c.name,
+                    req.timestamp_range.as_ref(),
+                )
+            })
+            .into_iter()
+            .collect::<Vec<_>>();
+
+        let stream =
+            table
+                .scan(None, &filters, None)
+                .await
+                .with_context(|_| error::CopyTableSnafu {
+                    table_name: table_ref.to_string(),
+                })?;
 
         let stream = stream
             .execute(0, SessionContext::default().task_ctx())
@@ -101,6 +113,6 @@ impl StatementExecutor {
             .stream_to_file(stream, &format, object_store, &path)
             .await?;
 
-        Ok(Output::AffectedRows(rows_copied))
+        Ok(rows_copied)
     }
 }

src/log-store/src/error.rs

@@ -23,13 +23,13 @@ use snafu::Location;
 pub enum Error {
     #[snafu(display("Failed to start log store gc task, source: {}", source))]
     StartGcTask {
-        #[snafu(backtrace)]
+        location: Location,
         source: RuntimeError,
     },
 
     #[snafu(display("Failed to stop log store gc task, source: {}", source))]
     StopGcTask {
-        #[snafu(backtrace)]
+        location: Location,
         source: RuntimeError,
     },
 

src/meta-client/Cargo.toml

@@ -12,7 +12,7 @@ common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
 common-telemetry = { path = "../common/telemetry" }
 common-meta = { path = "../common/meta" }
-etcd-client = "0.10"
+etcd-client = "0.11"
 rand.workspace = true
 serde.workspace = true
 serde_json.workspace = true

src/meta-client/src/client.rs

@@ -755,16 +755,21 @@ mod tests {
     async fn test_batch_put() {
         let tc = new_client("test_batch_put").await;
 
-        let req = BatchPutRequest::new()
-            .add_kv(tc.key("key"), b"value".to_vec())
-            .add_kv(tc.key("key2"), b"value2".to_vec());
+        let mut req = BatchPutRequest::new();
+        for i in 0..275 {
+            req = req.add_kv(
+                tc.key(&format!("key-{}", i)),
+                format!("value-{}", i).into_bytes(),
+            );
+        }
+
         let res = tc.client.batch_put(req).await;
         assert_eq!(0, res.unwrap().take_prev_kvs().len());
 
-        let req = RangeRequest::new().with_range(tc.key("key"), tc.key("key3"));
+        let req = RangeRequest::new().with_prefix(tc.key("key-"));
         let res = tc.client.range(req).await;
         let kvs = res.unwrap().take_kvs();
-        assert_eq!(2, kvs.len());
+        assert_eq!(275, kvs.len());
     }
 
     #[tokio::test]
@@ -772,16 +777,17 @@ mod tests {
         let tc = new_client("test_batch_get").await;
         tc.gen_data().await;
 
-        let req = BatchGetRequest::default()
-            .add_key(tc.key("key-1"))
-            .add_key(tc.key("key-2"));
+        let mut req = BatchGetRequest::default();
+        for i in 0..256 {
+            req = req.add_key(tc.key(&format!("key-{}", i)));
+        }
         let mut res = tc.client.batch_get(req).await.unwrap();
 
-        assert_eq!(2, res.take_kvs().len());
+        assert_eq!(10, res.take_kvs().len());
 
         let req = BatchGetRequest::default()
             .add_key(tc.key("key-1"))
-            .add_key(tc.key("key-222"));
+            .add_key(tc.key("key-999"));
         let mut res = tc.client.batch_get(req).await.unwrap();
 
         assert_eq!(1, res.take_kvs().len());

src/meta-client/src/error.rs

@@ -35,7 +35,7 @@ pub enum Error {
 
     #[snafu(display("Failed to create gRPC channel, source: {}", source))]
     CreateChannel {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc::error::Error,
     },
 
@@ -50,19 +50,19 @@ pub enum Error {
 
     #[snafu(display("Invalid response header, source: {}", source))]
     InvalidResponseHeader {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_meta::error::Error,
     },
 
     #[snafu(display("Failed to convert Metasrv request, source: {}", source))]
     ConvertMetaRequest {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_meta::error::Error,
     },
 
     #[snafu(display("Failed to convert Metasrv response, source: {}", source))]
     ConvertMetaResponse {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_meta::error::Error,
     },
 }
@@ -86,9 +86,9 @@ impl ErrorExt for Error {
             | Error::CreateHeartbeatStream { .. }
             | Error::CreateChannel { .. } => StatusCode::Internal,
 
-            Error::InvalidResponseHeader { source }
-            | Error::ConvertMetaRequest { source }
-            | Error::ConvertMetaResponse { source } => source.status_code(),
+            Error::InvalidResponseHeader { source, .. }
+            | Error::ConvertMetaRequest { source, .. }
+            | Error::ConvertMetaResponse { source, .. } => source.status_code(),
         }
     }
 }

src/meta-srv/Cargo.toml

@@ -24,7 +24,7 @@ common-telemetry = { path = "../common/telemetry" }
 common-time = { path = "../common/time" }
 dashmap = "5.4"
 derive_builder = "0.12"
-etcd-client = "0.10"
+etcd-client = "0.11"
 futures.workspace = true
 h2 = "0.3"
 http-body = "0.4"
@@ -38,6 +38,7 @@ regex = "1.6"
 serde = "1.0"
 serde_json = "1.0"
 snafu.workspace = true
+store-api = { path = "../store-api" }
 table = { path = "../table" }
 tokio.workspace = true
 tokio-stream = { version = "0.1", features = ["net"] }

src/meta-srv/src/error.rs

@@ -26,7 +26,7 @@ pub enum Error {
 
     #[snafu(display("Failed to shutdown {} server, source: {}", server, source))]
     ShutdownServer {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
         server: String,
     },
@@ -60,7 +60,7 @@ pub enum Error {
     },
     #[snafu(display("Failed to start http server, source: {}", source))]
     StartHttp {
-        #[snafu(backtrace)]
+        location: Location,
         source: servers::error::Error,
     },
     #[snafu(display("Failed to parse address {}, source: {}", addr, source))]
@@ -130,7 +130,7 @@ pub enum Error {
 
     #[snafu(display("Cannot parse catalog value, source: {}", source))]
     InvalidCatalogValue {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_catalog::error::Error,
     },
 
@@ -190,7 +190,7 @@ pub enum Error {
 
     #[snafu(display("Failed to create gRPC channel, source: {}", source))]
     CreateChannel {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc::error::Error,
     },
 
@@ -273,7 +273,7 @@ pub enum Error {
 
     #[snafu(display("Failed to recover procedure, source: {source}"))]
     RecoverProcedure {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_procedure::Error,
     },
 
@@ -321,7 +321,7 @@ pub enum Error {
     ))]
     RegisterProcedureLoader {
         type_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_procedure::error::Error,
     },
 
@@ -350,7 +350,13 @@ pub enum Error {
 
     #[snafu(display("Failed to convert table route, source: {}", source))]
     TableRouteConversion {
-        #[snafu(backtrace)]
+        location: Location,
+        source: common_meta::error::Error,
+    },
+
+    #[snafu(display("Failed to convert proto data, source: {}", source))]
+    ConvertProtoData {
+        location: Location,
         source: common_meta::error::Error,
     },
 
@@ -434,15 +440,17 @@ impl ErrorExt for Error {
             | Error::Unexpected { .. } => StatusCode::Unexpected,
             Error::TableNotFound { .. } => StatusCode::TableNotFound,
             Error::InvalidCatalogValue { source, .. } => source.status_code(),
-            Error::RecoverProcedure { source } => source.status_code(),
-            Error::ShutdownServer { source, .. } | Error::StartHttp { source } => {
+            Error::RecoverProcedure { source, .. } => source.status_code(),
+            Error::ShutdownServer { source, .. } | Error::StartHttp { source, .. } => {
                 source.status_code()
             }
 
             Error::RegionFailoverCandidatesNotFound { .. } => StatusCode::RuntimeResourcesExhausted,
 
             Error::RegisterProcedureLoader { source, .. } => source.status_code(),
-            Error::TableRouteConversion { source } => source.status_code(),
+            Error::TableRouteConversion { source, .. } | Error::ConvertProtoData { source, .. } => {
+                source.status_code()
+            }
             Error::Other { source, .. } => source.status_code(),
         }
     }

src/meta-srv/src/handler.rs

@@ -19,18 +19,18 @@ use std::time::Duration;
 
 use api::v1::meta::mailbox_message::Payload;
 use api::v1::meta::{
-    HeartbeatRequest, HeartbeatResponse, MailboxMessage, RequestHeader, ResponseHeader, Role,
-    PROTOCOL_VERSION,
+    HeartbeatRequest, HeartbeatResponse, MailboxMessage, RegionLease, RequestHeader,
+    ResponseHeader, Role, PROTOCOL_VERSION,
 };
 pub use check_leader_handler::CheckLeaderHandler;
 pub use collect_stats_handler::CollectStatsHandler;
 use common_meta::instruction::{Instruction, InstructionReply};
-use common_telemetry::{debug, info, warn};
+use common_telemetry::{debug, info, timer, warn};
 use dashmap::DashMap;
 pub use failure_handler::RegionFailureHandler;
 pub use keep_lease_handler::KeepLeaseHandler;
 use metrics::{decrement_gauge, increment_gauge};
-pub use on_leader_start::OnLeaderStartHandler;
+pub use on_leader_start_handler::OnLeaderStartHandler;
 pub use persist_stats_handler::PersistStatsHandler;
 pub use response_header_handler::ResponseHeaderHandler;
 use snafu::{OptionExt, ResultExt};
@@ -40,7 +40,7 @@ use tokio::sync::{oneshot, Notify, RwLock};
 use self::node_stat::Stat;
 use crate::error::{self, DeserializeFromJsonSnafu, Result, UnexpectedInstructionReplySnafu};
 use crate::metasrv::Context;
-use crate::metrics::METRIC_META_HEARTBEAT_CONNECTION_NUM;
+use crate::metrics::{METRIC_META_HANDLER_EXECUTE, METRIC_META_HEARTBEAT_CONNECTION_NUM};
 use crate::sequence::Sequence;
 use crate::service::mailbox::{
     BroadcastChannel, Channel, Mailbox, MailboxReceiver, MailboxRef, MessageId,
@@ -52,14 +52,21 @@ pub(crate) mod failure_handler;
 mod keep_lease_handler;
 pub mod mailbox_handler;
 pub mod node_stat;
-mod on_leader_start;
+mod on_leader_start_handler;
 mod persist_stats_handler;
+pub(crate) mod region_lease_handler;
 mod response_header_handler;
 
 #[async_trait::async_trait]
 pub trait HeartbeatHandler: Send + Sync {
     fn is_acceptable(&self, role: Role) -> bool;
 
+    fn name(&self) -> &'static str {
+        let type_name = std::any::type_name::<Self>();
+        // short name
+        type_name.split("::").last().unwrap_or(type_name)
+    }
+
     async fn handle(
         &self,
         req: &HeartbeatRequest,
@@ -73,6 +80,7 @@ pub struct HeartbeatAccumulator {
     pub header: Option<ResponseHeader>,
     pub instructions: Vec<Instruction>,
     pub stat: Option<Stat>,
+    pub region_leases: Vec<RegionLease>,
 }
 
 impl HeartbeatAccumulator {
@@ -130,6 +138,7 @@ impl Pushers {
             .push(HeartbeatResponse {
                 header: Some(pusher.header()),
                 mailbox_message: Some(mailbox_message),
+                ..Default::default()
             })
             .await
     }
@@ -151,6 +160,7 @@ impl Pushers {
                 .push(HeartbeatResponse {
                     header: Some(pusher.header()),
                     mailbox_message: Some(mailbox_message),
+                    ..Default::default()
                 })
                 .await?;
         }
@@ -167,9 +177,22 @@ impl Pushers {
     }
 }
 
+struct NameCachedHandler {
+    name: &'static str,
+    handler: Box<dyn HeartbeatHandler>,
+}
+
+impl NameCachedHandler {
+    fn new(handler: impl HeartbeatHandler + 'static) -> Self {
+        let name = handler.name();
+        let handler = Box::new(handler);
+        Self { name, handler }
+    }
+}
+
 #[derive(Clone, Default)]
 pub struct HeartbeatHandlerGroup {
-    handlers: Arc<RwLock<Vec<Box<dyn HeartbeatHandler>>>>,
+    handlers: Arc<RwLock<Vec<NameCachedHandler>>>,
     pushers: Pushers,
 }
 
@@ -183,7 +206,7 @@ impl HeartbeatHandlerGroup {
 
     pub async fn add_handler(&self, handler: impl HeartbeatHandler + 'static) {
         let mut handlers = self.handlers.write().await;
-        handlers.push(Box::new(handler));
+        handlers.push(NameCachedHandler::new(handler));
     }
 
     pub async fn register(&self, key: impl AsRef<str>, pusher: Pusher) {
@@ -219,19 +242,21 @@ impl HeartbeatHandlerGroup {
                 err_msg: format!("invalid role: {:?}", req.header),
             })?;
 
-        for h in handlers.iter() {
+        for NameCachedHandler { name, handler } in handlers.iter() {
             if ctx.is_skip_all() {
                 break;
             }
 
-            if h.is_acceptable(role) {
-                h.handle(&req, &mut ctx, &mut acc).await?;
+            if handler.is_acceptable(role) {
+                let _timer = timer!(METRIC_META_HANDLER_EXECUTE, &[("name", *name)]);
+                handler.handle(&req, &mut ctx, &mut acc).await?;
             }
         }
         let header = std::mem::take(&mut acc.header);
         let res = HeartbeatResponse {
             header,
-            mailbox_message: acc.into_mailbox_message(),
+            region_leases: acc.region_leases,
+            ..Default::default()
         };
         Ok(res)
     }
@@ -378,7 +403,11 @@ mod tests {
     use api::v1::meta::{MailboxMessage, RequestHeader, Role, PROTOCOL_VERSION};
     use tokio::sync::mpsc;
 
-    use crate::handler::{HeartbeatHandlerGroup, HeartbeatMailbox, Pusher};
+    use crate::handler::mailbox_handler::MailboxHandler;
+    use crate::handler::{
+        CheckLeaderHandler, CollectStatsHandler, HeartbeatHandlerGroup, HeartbeatMailbox,
+        OnLeaderStartHandler, PersistStatsHandler, Pusher, ResponseHeaderHandler,
+    };
     use crate::sequence::Sequence;
     use crate::service::mailbox::{Channel, MailboxReceiver, MailboxRef};
     use crate::service::store::memory::MemStore;
@@ -447,4 +476,25 @@ mod tests {
 
         (mailbox, receiver)
     }
+
+    #[tokio::test]
+    async fn test_handler_name() {
+        let group = HeartbeatHandlerGroup::default();
+        group.add_handler(ResponseHeaderHandler::default()).await;
+        group.add_handler(CheckLeaderHandler::default()).await;
+        group.add_handler(OnLeaderStartHandler::default()).await;
+        group.add_handler(CollectStatsHandler::default()).await;
+        group.add_handler(MailboxHandler::default()).await;
+        group.add_handler(PersistStatsHandler::default()).await;
+
+        let handlers = group.handlers.read().await;
+
+        assert_eq!(6, handlers.len());
+        assert_eq!("ResponseHeaderHandler", handlers[0].handler.name());
+        assert_eq!("CheckLeaderHandler", handlers[1].handler.name());
+        assert_eq!("OnLeaderStartHandler", handlers[2].handler.name());
+        assert_eq!("CollectStatsHandler", handlers[3].handler.name());
+        assert_eq!("MailboxHandler", handlers[4].handler.name());
+        assert_eq!("PersistStatsHandler", handlers[5].handler.name());
+    }
 }

src/meta-srv/src/handler/collect_stats_handler.rs

@@ -20,6 +20,7 @@ use crate::error::Result;
 use crate::handler::{HeartbeatAccumulator, HeartbeatHandler};
 use crate::metasrv::Context;
 
+#[derive(Default)]
 pub struct CollectStatsHandler;
 
 #[async_trait::async_trait]

src/meta-srv/src/handler/failure_handler.rs

@@ -19,7 +19,7 @@ use std::sync::Arc;
 use api::v1::meta::{HeartbeatRequest, Role};
 use async_trait::async_trait;
 use common_catalog::consts::MITO_ENGINE;
-use common_meta::instruction::TableIdent;
+use common_meta::ident::TableIdent;
 use common_meta::RegionIdent;
 use table::engine::table_id;
 
@@ -36,6 +36,7 @@ pub(crate) struct DatanodeHeartbeat {
 
 pub struct RegionFailureHandler {
     failure_detect_runner: FailureDetectRunner,
+    region_failover_manager: Arc<RegionFailoverManager>,
 }
 
 impl RegionFailureHandler {
@@ -45,13 +46,19 @@ impl RegionFailureHandler {
     ) -> Result<Self> {
         region_failover_manager.try_start()?;
 
-        let mut failure_detect_runner = FailureDetectRunner::new(election, region_failover_manager);
+        let mut failure_detect_runner =
+            FailureDetectRunner::new(election, region_failover_manager.clone());
         failure_detect_runner.start().await;
 
         Ok(Self {
             failure_detect_runner,
+            region_failover_manager,
         })
     }
+
+    pub(crate) fn region_failover_manager(&self) -> &Arc<RegionFailoverManager> {
+        &self.region_failover_manager
+    }
 }
 
 #[async_trait]
@@ -74,12 +81,6 @@ impl HeartbeatHandler for RegionFailureHandler {
 
         let Some(stat) = acc.stat.as_ref() else { return Ok(()) };
 
-        // TODO(LFC): Filter out the stalled heartbeats:
-        // After the region failover is done, the distribution of region is changed.
-        // We can compare the heartbeat info here with the global region placement metadata,
-        // and remove the incorrect region ident keys in failure detect runner
-        // (by sending a control message).
-
         let heartbeat = DatanodeHeartbeat {
             region_idents: stat
                 .region_stats

src/meta-srv/src/handler/failure_handler/runner.rs

@@ -246,7 +246,7 @@ impl FailureDetectorContainer {
 #[cfg(test)]
 mod tests {
     use common_catalog::consts::MITO_ENGINE;
-    use common_meta::instruction::TableIdent;
+    use common_meta::ident::TableIdent;
     use rand::Rng;
 
     use super::*;

src/meta-srv/src/handler/node_stat.rs

@@ -42,6 +42,8 @@ pub struct Stat {
     pub write_io_rate: f64,
     /// Region stats on this node
     pub region_stats: Vec<RegionStat>,
+    // The node epoch is used to check whether the node has restarted or redeployed.
+    pub node_epoch: u64,
 }
 
 #[derive(Debug, Default, Serialize, Deserialize)]
@@ -79,6 +81,7 @@ impl TryFrom<HeartbeatRequest> for Stat {
             is_leader,
             node_stat,
             region_stats,
+            node_epoch,
             ..
         } = value;
 
@@ -104,6 +107,7 @@ impl TryFrom<HeartbeatRequest> for Stat {
                     read_io_rate: node_stat.read_io_rate,
                     write_io_rate: node_stat.write_io_rate,
                     region_stats: region_stats.into_iter().map(RegionStat::from).collect(),
+                    node_epoch,
                 })
             }
             _ => Err(()),

src/meta-srv/src/handler/persist_stats_handler.rs

@@ -23,9 +23,47 @@ use crate::metasrv::Context;
 
 const MAX_CACHED_STATS_PER_KEY: usize = 10;
 
+#[derive(Default)]
+struct EpochStats {
+    stats: Vec<Stat>,
+    epoch: Option<u64>,
+}
+
+impl EpochStats {
+    #[inline]
+    fn drain_all(&mut self) -> Vec<Stat> {
+        self.stats.drain(..).collect()
+    }
+
+    #[inline]
+    fn clear(&mut self) {
+        self.stats.clear();
+    }
+
+    #[inline]
+    fn push(&mut self, stat: Stat) {
+        self.stats.push(stat);
+    }
+
+    #[inline]
+    fn len(&self) -> usize {
+        self.stats.len()
+    }
+
+    #[inline]
+    fn epoch(&self) -> Option<u64> {
+        self.epoch
+    }
+
+    #[inline]
+    fn set_epoch(&mut self, epoch: u64) {
+        self.epoch = Some(epoch);
+    }
+}
+
 #[derive(Default)]
 pub struct PersistStatsHandler {
-    stats_cache: DashMap<StatKey, Vec<Stat>>,
+    stats_cache: DashMap<StatKey, EpochStats>,
 }
 
 #[async_trait::async_trait]
@@ -40,26 +78,47 @@ impl HeartbeatHandler for PersistStatsHandler {
         ctx: &mut Context,
         acc: &mut HeartbeatAccumulator,
     ) -> Result<()> {
-        let Some(stat) = acc.stat.take() else { return Ok(()) };
+        let Some(current_stat) = acc.stat.take() else { return Ok(()) };
 
-        let key = stat.stat_key();
+        let key = current_stat.stat_key();
         let mut entry = self
             .stats_cache
             .entry(key)
-            .or_insert_with(|| Vec::with_capacity(MAX_CACHED_STATS_PER_KEY));
-        let stats = entry.value_mut();
-        stats.push(stat);
+            .or_insert_with(EpochStats::default);
 
-        if stats.len() < MAX_CACHED_STATS_PER_KEY {
+        let key: Vec<u8> = key.into();
+        let epoch_stats = entry.value_mut();
+
+        let refresh = if let Some(epoch) = epoch_stats.epoch() {
+            // This node may have been redeployed.
+            if current_stat.node_epoch > epoch {
+                epoch_stats.set_epoch(current_stat.node_epoch);
+                epoch_stats.clear();
+                true
+            } else {
+                false
+            }
+        } else {
+            epoch_stats.set_epoch(current_stat.node_epoch);
+            // If the epoch is empty, it indicates that the current node sending the heartbeat
+            // for the first time to the current meta leader, so it is necessary to persist
+            // the data to the KV store as soon as possible.
+            true
+        };
+
+        epoch_stats.push(current_stat);
+
+        if !refresh && epoch_stats.len() < MAX_CACHED_STATS_PER_KEY {
             return Ok(());
         }
 
-        let stats = stats.drain(..).collect();
-        let val = StatValue { stats };
-
+        let value: Vec<u8> = StatValue {
+            stats: epoch_stats.drain_all(),
+        }
+        .try_into()?;
         let put = PutRequest {
-            key: key.into(),
-            value: val.try_into()?,
+            key,
+            value,
             ..Default::default()
         };
 
@@ -74,12 +133,11 @@ mod tests {
     use std::sync::atomic::AtomicBool;
     use std::sync::Arc;
 
-    use api::v1::meta::RangeRequest;
-
     use super::*;
     use crate::handler::{HeartbeatMailbox, Pushers};
     use crate::keys::StatKey;
     use crate::sequence::Sequence;
+    use crate::service::store::ext::KvStoreExt;
     use crate::service::store::memory::MemStore;
 
     #[tokio::test]
@@ -88,7 +146,7 @@ mod tests {
         let kv_store = Arc::new(MemStore::new());
         let seq = Sequence::new("test_seq", 0, 10, kv_store.clone());
         let mailbox = HeartbeatMailbox::create(Pushers::default(), seq);
-        let mut ctx = Context {
+        let ctx = Context {
             server_addr: "127.0.0.1:0000".to_string(),
             in_memory,
             kv_store,
@@ -98,9 +156,40 @@ mod tests {
             is_infancy: false,
         };
 
-        let req = HeartbeatRequest::default();
         let handler = PersistStatsHandler::default();
-        for i in 1..=MAX_CACHED_STATS_PER_KEY {
+        handle_request_many_times(ctx.clone(), &handler, 1).await;
+
+        let key = StatKey {
+            cluster_id: 3,
+            node_id: 101,
+        };
+        let res = ctx.in_memory.get(key.try_into().unwrap()).await.unwrap();
+        assert!(res.is_some());
+        let kv = res.unwrap();
+        let key: StatKey = kv.key.clone().try_into().unwrap();
+        assert_eq!(3, key.cluster_id);
+        assert_eq!(101, key.node_id);
+        let val: StatValue = kv.value.try_into().unwrap();
+        // first new stat must be set in kv store immediately
+        assert_eq!(1, val.stats.len());
+        assert_eq!(Some(1), val.stats[0].region_num);
+
+        handle_request_many_times(ctx.clone(), &handler, 10).await;
+        let res = ctx.in_memory.get(key.try_into().unwrap()).await.unwrap();
+        assert!(res.is_some());
+        let kv = res.unwrap();
+        let val: StatValue = kv.value.try_into().unwrap();
+        // refresh every 10 stats
+        assert_eq!(10, val.stats.len());
+    }
+
+    async fn handle_request_many_times(
+        mut ctx: Context,
+        handler: &PersistStatsHandler,
+        loop_times: i32,
+    ) {
+        let req = HeartbeatRequest::default();
+        for i in 1..=loop_times {
             let mut acc = HeartbeatAccumulator {
                 stat: Some(Stat {
                     cluster_id: 3,
@@ -112,30 +201,5 @@ mod tests {
             };
             handler.handle(&req, &mut ctx, &mut acc).await.unwrap();
         }
-
-        let key = StatKey {
-            cluster_id: 3,
-            node_id: 101,
-        };
-
-        let req = RangeRequest {
-            key: key.try_into().unwrap(),
-            ..Default::default()
-        };
-
-        let res = ctx.in_memory.range(req).await.unwrap();
-
-        assert_eq!(1, res.kvs.len());
-
-        let kv = &res.kvs[0];
-
-        let key: StatKey = kv.key.clone().try_into().unwrap();
-        assert_eq!(3, key.cluster_id);
-        assert_eq!(101, key.node_id);
-
-        let val: StatValue = kv.value.clone().try_into().unwrap();
-
-        assert_eq!(10, val.stats.len());
-        assert_eq!(Some(1), val.stats[0].region_num);
     }
 }

src/meta-srv/src/handler/region_lease_handler.rs

@@ -0,0 +1,226 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use api::v1::meta::{HeartbeatRequest, RegionLease, Role};
+use async_trait::async_trait;
+use catalog::helper::TableGlobalKey;
+use common_meta::ident::TableIdent;
+use common_meta::ClusterId;
+use store_api::storage::RegionNumber;
+
+use crate::error::Result;
+use crate::handler::{HeartbeatAccumulator, HeartbeatHandler};
+use crate::metasrv::Context;
+use crate::procedure::region_failover::{RegionFailoverKey, RegionFailoverManager};
+use crate::service::store::kv::KvStoreRef;
+use crate::table_routes;
+
+/// The lease seconds of a region. It's set by two default heartbeat intervals (5 second × 2) plus
+/// two roundtrip time (2 second × 2 × 2), plus some extra buffer (2 second).
+// TODO(LFC): Make region lease seconds calculated from Datanode heartbeat configuration.
+pub(crate) const REGION_LEASE_SECONDS: u64 = 20;
+
+pub(crate) struct RegionLeaseHandler {
+    kv_store: KvStoreRef,
+    region_failover_manager: Option<Arc<RegionFailoverManager>>,
+}
+
+impl RegionLeaseHandler {
+    pub(crate) fn new(
+        kv_store: KvStoreRef,
+        region_failover_manager: Option<Arc<RegionFailoverManager>>,
+    ) -> Self {
+        Self {
+            kv_store,
+            region_failover_manager,
+        }
+    }
+
+    /// Filter out the regions that are currently in failover.
+    /// It's meaningless to extend the lease of a region if it is in failover.
+    fn filter_failover_regions(
+        &self,
+        cluster_id: ClusterId,
+        table_ident: &TableIdent,
+        regions: Vec<RegionNumber>,
+    ) -> Vec<RegionNumber> {
+        if let Some(region_failover_manager) = &self.region_failover_manager {
+            let mut region_failover_key = RegionFailoverKey {
+                cluster_id,
+                table_ident: table_ident.clone(),
+                region_number: 0,
+            };
+
+            regions
+                .into_iter()
+                .filter(|region| {
+                    region_failover_key.region_number = *region;
+                    !region_failover_manager.is_region_failover_running(&region_failover_key)
+                })
+                .collect()
+        } else {
+            regions
+        }
+    }
+}
+
+#[async_trait]
+impl HeartbeatHandler for RegionLeaseHandler {
+    fn is_acceptable(&self, role: Role) -> bool {
+        role == Role::Datanode
+    }
+
+    async fn handle(
+        &self,
+        req: &HeartbeatRequest,
+        _: &mut Context,
+        acc: &mut HeartbeatAccumulator,
+    ) -> Result<()> {
+        let Some(stat) = acc.stat.as_ref() else { return Ok(()) };
+
+        let mut datanode_regions = HashMap::new();
+        stat.region_stats.iter().for_each(|x| {
+            let key = TableGlobalKey {
+                catalog_name: x.catalog.to_string(),
+                schema_name: x.schema.to_string(),
+                table_name: x.table.to_string(),
+            };
+            datanode_regions
+                .entry(key)
+                .or_insert_with(Vec::new)
+                .push(table::engine::region_number(x.id));
+        });
+
+        // TODO(LFC): Retrieve table global values from some cache here.
+        let table_global_values = table_routes::batch_get_table_global_value(
+            &self.kv_store,
+            datanode_regions.keys().collect::<Vec<_>>(),
+        )
+        .await?;
+
+        let mut region_leases = Vec::with_capacity(datanode_regions.len());
+        for (table_global_key, local_regions) in datanode_regions {
+            let Some(Some(table_global_value)) = table_global_values.get(&table_global_key) else { continue };
+
+            let Some(global_regions) = table_global_value.regions_id_map.get(&stat.id) else { continue };
+
+            // Filter out the designated regions from table global metadata for the given table on the given Datanode.
+            let designated_regions = local_regions
+                .into_iter()
+                .filter(|x| global_regions.contains(x))
+                .collect::<Vec<_>>();
+
+            let table_ident = TableIdent {
+                catalog: table_global_key.catalog_name.to_string(),
+                schema: table_global_key.schema_name.to_string(),
+                table: table_global_key.table_name.to_string(),
+                table_id: table_global_value.table_id(),
+                engine: table_global_value.engine().to_string(),
+            };
+            let designated_regions =
+                self.filter_failover_regions(stat.cluster_id, &table_ident, designated_regions);
+
+            region_leases.push(RegionLease {
+                table_ident: Some(table_ident.into()),
+                regions: designated_regions,
+                duration_since_epoch: req.duration_since_epoch,
+                lease_seconds: REGION_LEASE_SECONDS,
+            });
+        }
+        acc.region_leases = region_leases;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+
+    use super::*;
+    use crate::handler::node_stat::{RegionStat, Stat};
+    use crate::metasrv::builder::MetaSrvBuilder;
+    use crate::test_util;
+
+    #[tokio::test]
+    async fn test_handle_region_lease() {
+        let region_failover_manager = test_util::create_region_failover_manager();
+        let kv_store = region_failover_manager
+            .create_context()
+            .selector_ctx
+            .kv_store
+            .clone();
+
+        let table_name = "my_table";
+        let _ = table_routes::tests::prepare_table_global_value(&kv_store, table_name).await;
+
+        let table_ident = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: table_name.to_string(),
+            table_id: 1,
+            engine: "mito".to_string(),
+        };
+        region_failover_manager
+            .running_procedures()
+            .write()
+            .unwrap()
+            .insert(RegionFailoverKey {
+                cluster_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 1,
+            });
+
+        let handler = RegionLeaseHandler::new(kv_store, Some(region_failover_manager));
+
+        let req = HeartbeatRequest {
+            duration_since_epoch: 1234,
+            ..Default::default()
+        };
+
+        let builder = MetaSrvBuilder::new();
+        let metasrv = builder.build().await.unwrap();
+        let ctx = &mut metasrv.new_ctx();
+
+        let acc = &mut HeartbeatAccumulator::default();
+        let new_region_stat = |region_id: u64| -> RegionStat {
+            RegionStat {
+                id: region_id,
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+                table: table_name.to_string(),
+                ..Default::default()
+            }
+        };
+        acc.stat = Some(Stat {
+            cluster_id: 1,
+            id: 1,
+            region_stats: vec![new_region_stat(1), new_region_stat(2), new_region_stat(3)],
+            ..Default::default()
+        });
+
+        handler.handle(&req, ctx, acc).await.unwrap();
+
+        // region 1 is during failover and region 3 is not in table global value,
+        // so only region 2's lease is extended.
+        assert_eq!(acc.region_leases.len(), 1);
+        let lease = acc.region_leases.remove(0);
+        assert_eq!(lease.table_ident.unwrap(), table_ident.into());
+        assert_eq!(lease.regions, vec![2]);
+        assert_eq!(lease.duration_since_epoch, 1234);
+        assert_eq!(lease.lease_seconds, REGION_LEASE_SECONDS);
+    }
+}

src/meta-srv/src/handler/response_header_handler.rs

@@ -88,6 +88,7 @@ mod tests {
         let res = HeartbeatResponse {
             header,
             mailbox_message: acc.into_mailbox_message(),
+            ..Default::default()
         };
         assert_eq!(1, res.header.unwrap().cluster_id);
     }

src/meta-srv/src/metasrv.rs

@@ -49,6 +49,7 @@ pub struct MetaSrvOptions {
     pub datanode_lease_secs: i64,
     pub selector: SelectorType,
     pub use_memory_store: bool,
+    pub disable_region_failover: bool,
     pub http_opts: HttpOptions,
     pub logging: LoggingOptions,
 }
@@ -62,6 +63,7 @@ impl Default for MetaSrvOptions {
             datanode_lease_secs: 15,
             selector: SelectorType::default(),
             use_memory_store: false,
+            disable_region_failover: false,
             http_opts: HttpOptions::default(),
             logging: LoggingOptions::default(),
         }
@@ -146,25 +148,21 @@ impl MetaSrv {
             common_runtime::spawn_bg(async move {
                 loop {
                     match rx.recv().await {
-                        Ok(msg) => {
-                            match msg {
-                                LeaderChangeMessage::Elected(_) => {
-                                    if let Err(e) = procedure_manager.recover().await {
-                                        error!("Failed to recover procedures, error: {e}");
-                                    }
-                                }
-                                LeaderChangeMessage::StepDown(leader) => {
-                                    // TODO(LFC): TBC
-                                    error!("Leader :{:?} step down", leader);
+                        Ok(msg) => match msg {
+                            LeaderChangeMessage::Elected(_) => {
+                                if let Err(e) = procedure_manager.recover().await {
+                                    error!("Failed to recover procedures, error: {e}");
                                 }
                             }
-                        }
+                            LeaderChangeMessage::StepDown(leader) => {
+                                error!("Leader :{:?} step down", leader);
+                            }
+                        },
                         Err(RecvError::Closed) => {
                             error!("Not expected, is leader election loop still running?");
                             break;
                         }
                         Err(RecvError::Lagged(_)) => {
-                            // TODO(LFC): TBC
                             break;
                         }
                     }