feat: execute python script in distributed mode (#1264)

* feat: execute python script in distributed mode

* fix: rebase develop

.github/workflows/release.yml

@@ -5,6 +5,7 @@ on:
   schedule:
     # At 00:00 on Monday.
     - cron: '0 0 * * 1'
+  # Mannually trigger only builds binaries.
   workflow_dispatch:
 
 name: Release
@@ -32,38 +33,42 @@ jobs:
             os: ubuntu-2004-16-cores
             file: greptime-linux-amd64
             continue-on-error: false
+            opts: "-F servers/dashboard"
           - arch: aarch64-unknown-linux-gnu
             os: ubuntu-2004-16-cores
             file: greptime-linux-arm64
             continue-on-error: false
+            opts: "-F servers/dashboard"
           - arch: aarch64-apple-darwin
             os: macos-latest
             file: greptime-darwin-arm64
             continue-on-error: false
+            opts: "-F servers/dashboard"
           - arch: x86_64-apple-darwin
             os: macos-latest
             file: greptime-darwin-amd64
             continue-on-error: false
+            opts: "-F servers/dashboard"
           - arch: x86_64-unknown-linux-gnu
             os: ubuntu-2004-16-cores
             file: greptime-linux-amd64-pyo3
             continue-on-error: false
-            opts: "-F pyo3_backend"
+            opts: "-F pyo3_backend,servers/dashboard"
           - arch: aarch64-unknown-linux-gnu
             os: ubuntu-2004-16-cores
             file: greptime-linux-arm64-pyo3
             continue-on-error: false
-            opts: "-F pyo3_backend"
+            opts: "-F pyo3_backend,servers/dashboard"
           - arch: aarch64-apple-darwin
             os: macos-latest
             file: greptime-darwin-arm64-pyo3
             continue-on-error: false
-            opts: "-F pyo3_backend"
+            opts: "-F pyo3_backend,servers/dashboard"
           - arch: x86_64-apple-darwin
             os: macos-latest
             file: greptime-darwin-amd64-pyo3
             continue-on-error: false
-            opts: "-F pyo3_backend"
+            opts: "-F pyo3_backend,servers/dashboard"
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.continue-on-error }}
     if: github.repository == 'GreptimeTeam/greptimedb'
@@ -164,7 +169,7 @@ jobs:
           export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
           export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
           export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
-          
+
           echo "implementation=CPython" >> pyo3.config
           echo "version=3.10" >> pyo3.config
           echo "implementation=CPython" >> pyo3.config
@@ -212,7 +217,7 @@ jobs:
     name: Build docker image
     needs: [build]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb'
+    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -252,9 +257,8 @@ jobs:
 
       - name: Unzip the amd64 artifacts
         run: |
-          cd amd64
-          tar xvf greptime-linux-amd64-pyo3.tgz
-          rm greptime-linux-amd64-pyo3.tgz
+          tar xvf amd64/greptime-linux-amd64-pyo3.tgz -C amd64/ && rm amd64/greptime-linux-amd64-pyo3.tgz
+          cp -r amd64 docker/ci
 
       - name: Download arm64 binary
         id: download-arm64
@@ -267,15 +271,14 @@ jobs:
         id: unzip-arm64
         if: success() || steps.download-arm64.conclusion == 'success'
         run: |
-          cd arm64
-          tar xvf greptime-linux-arm64-pyo3.tgz
-          rm greptime-linux-arm64-pyo3.tgz
+          tar xvf arm64/greptime-linux-arm64-pyo3.tgz -C arm64/ && rm arm64/greptime-linux-arm64-pyo3.tgz
+          cp -r arm64 docker/ci
 
       - name: Build and push all
         uses: docker/build-push-action@v3
         if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds
         with:
-          context: .
+          context: ./docker/ci/
           file: ./docker/ci/Dockerfile
           push: true
           platforms: linux/amd64,linux/arm64
@@ -287,7 +290,7 @@ jobs:
         uses: docker/build-push-action@v3
         if: success() || steps.download-arm64.conclusion == 'failure' # Only build and push amd64 platform if download-arm64 fails
         with:
-          context: .
+          context: ./docker/ci/
           file: ./docker/ci/Dockerfile
           push: true
           platforms: linux/amd64
@@ -300,7 +303,7 @@ jobs:
     # Release artifacts only when all the artifacts are built successfully.
     needs: [build,docker]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb'
+    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -343,7 +346,7 @@ jobs:
     name: Push docker image to UCloud Container Registry
     needs: [docker]
     runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb'
+    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
     # Push to uhub may fail(500 error), but we don't want to block the release process. The failed job will be retried manually.
     continue-on-error: true
     steps:

.gitignore

@@ -35,3 +35,7 @@ benchmarks/data
 
 # dotenv
 .env
+
+# dashboard files
+!/src/servers/dashboard/VERSION
+/src/servers/dashboard/*

Cargo.toml

@@ -51,22 +51,22 @@ edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
-arrow = { version = "34.0" }
-arrow-array = "34.0"
-arrow-flight = "34.0"
-arrow-schema = { version = "34.0", features = ["serde"] }
+arrow = { version = "36.0" }
+arrow-array = "36.0"
+arrow-flight = "36.0"
+arrow-schema = { version = "36.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
-datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "146a949218ec970784974137277cde3b4e547d0a" }
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "8e125d2ecf242b4f4b81f06839900dbb2037cc2a" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "8e125d2ecf242b4f4b81f06839900dbb2037cc2a" }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "8e125d2ecf242b4f4b81f06839900dbb2037cc2a" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "8e125d2ecf242b4f4b81f06839900dbb2037cc2a" }
+datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "8e125d2ecf242b4f4b81f06839900dbb2037cc2a" }
+datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "8e125d2ecf242b4f4b81f06839900dbb2037cc2a" }
 futures = "0.3"
 futures-util = "0.3"
-parquet = "34.0"
+parquet = "36.0"
 paste = "1.0"
 prost = "0.11"
 rand = "0.8"

README.md

@@ -1,8 +1,8 @@
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: light)" srcset="/docs/logo-text-padding.png">
-    <source media="(prefers-color-scheme: dark)" srcset="/docs/logo-text-padding-dark.png">
-    <img alt="GreptimeDB Logo" src="/docs/logo-text-padding.png" width="400px">
+    <source media="(prefers-color-scheme: light)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png">
+    <source media="(prefers-color-scheme: dark)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding-dark.png">
+    <img alt="GreptimeDB Logo" src="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png" width="400px">
   </picture>
 </p>
 
@@ -158,6 +158,7 @@ You can always cleanup test database by removing `/tmp/greptimedb`.
 - GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts.html)
 - GreptimeDB [Developer
   Guide](https://docs.greptime.com/developer-guide/overview.html)
+- GreptimeDB [internal code document](https://greptimedb.rs)
 
 ### Dashboard
 - [The dashboard UI for GreptimeDB](https://github.com/GreptimeTeam/dashboard)

SECURITY.md

@@ -0,0 +1,19 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| >= v0.1.0 | :white_check_mark: |
+| < v0.1.0   | :x:                |
+
+## Reporting a Vulnerability
+
+We place great importance on the security of GreptimeDB code, software, 
+and cloud platform. If you come across a security vulnerability in GreptimeDB, 
+we kindly request that you inform us immediately. We will thoroughly investigate
+all valid reports and make every effort to resolve the issue promptly.
+
+To report any issues or vulnerabilities, please email us at info@greptime.com, rather than 
+posting publicly on GitHub. Be sure to provide us with the version identifier as well as details
+on how the vulnerability can be exploited.

benchmarks/src/bin/nyc-taxi.rs

@@ -126,12 +126,13 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
 
     for (array, field) in record_batch.columns().iter().zip(fields.iter()) {
         let (values, datatype) = build_values(array);
+
         let column = Column {
             column_name: field.name().to_owned(),
             values: Some(values),
             null_mask: array
                 .data()
-                .null_bitmap()
+                .nulls()
                 .map(|bitmap| bitmap.buffer().as_slice().to_vec())
                 .unwrap_or_default(),
             datatype: datatype.into(),
@@ -182,10 +183,10 @@ fn build_values(column: &ArrayRef) -> (Values, ColumnDataType) {
             let values = array.values();
             (
                 Values {
-                    i64_values: values.to_vec(),
+                    ts_microsecond_values: values.to_vec(),
                     ..Default::default()
                 },
-                ColumnDataType::Int64,
+                ColumnDataType::TimestampMicrosecond,
             )
         }
         DataType::Utf8 => {
@@ -252,13 +253,13 @@ fn create_table_expr() -> CreateTableExpr {
             },
             ColumnDef {
                 name: "tpep_pickup_datetime".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
             },
             ColumnDef {
                 name: "tpep_dropoff_datetime".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
             },

config/datanode.example.toml

@@ -37,11 +37,19 @@ type = "File"
 data_dir = "/tmp/greptimedb/data/"
 
 # Compaction options, see `standalone.example.toml`.
-[compaction]
+[storage.compaction]
 max_inflight_tasks = 4
 max_files_in_level0 = 8
 max_purge_tasks = 32
 
+# Storage manifest options
+[storage.manifest]
+# Region checkpoint actions margin.
+# Create a checkpoint every <checkpoint_margin> actions.
+checkpoint_margin = 10
+# Region manifest logs and checkpoints gc execution duration
+gc_duration = '30s'
+
 # Procedure storage options, see `standalone.example.toml`.
 # [procedure.store]
 # type = "File"

config/standalone.example.toml

@@ -99,7 +99,7 @@ type = "File"
 data_dir = "/tmp/greptimedb/data/"
 
 # Compaction options.
-[compaction]
+[storage.compaction]
 # Max task number that can concurrently run.
 max_inflight_tasks = 4
 # Max files in level 0 to trigger compaction.
@@ -107,6 +107,15 @@ max_files_in_level0 = 8
 # Max task number for SST purge task after compaction.
 max_purge_tasks = 32
 
+# Storage manifest options
+[storage.manifest]
+# Region checkpoint actions margin.
+# Create a checkpoint every <checkpoint_margin> actions.
+checkpoint_margin = 10
+# Region manifest logs and checkpoints gc execution duration
+gc_duration = '30s'
+
+
 # Procedure storage options.
 # Uncomment to enable.
 # [procedure.store]

docker/ci/Dockerfile

@@ -6,7 +6,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     python3.10-dev \
     python3-pip
 
-RUN python3 -m pip install pyarrow
+COPY requirements.txt /etc/greptime/requirements.txt
+
+RUN python3 -m pip install -r /etc/greptime/requirements.txt
 
 ARG TARGETARCH
 

docker/ci/requirements.txt

@@ -0,0 +1,5 @@
+numpy>=1.24.2
+pandas>=1.5.3
+pyarrow>=11.0.0
+requests>=2.28.2
+scipy>=1.10.1

scripts/fetch-dashboard-assets.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# This script is used to download built dashboard assets from the "GreptimeTeam/dashboard" repository.
+
+set -e
+
+declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
+declare -r ROOT_DIR=$(dirname ${SCRIPT_DIR})
+declare -r STATIC_DIR="$ROOT_DIR/src/servers/dashboard"
+
+RELEASE_VERSION="$(cat $STATIC_DIR/VERSION)"
+
+# Download the SHA256 checksum attached to the release. To verify the integrity
+# of the download, this checksum will be used to check the download tar file
+# containing the built dashboard assets.
+curl -Ls https://github.com/GreptimeTeam/dashboard/releases/download/$RELEASE_VERSION/sha256.txt --output sha256.txt
+
+# Download the tar file containing the built dashboard assets.
+curl -L https://github.com/GreptimeTeam/dashboard/releases/download/$RELEASE_VERSION/build.tar.gz --output build.tar.gz
+
+# Verify the checksums match; exit if they don't.
+case "$(uname -s)" in
+    FreeBSD | Darwin)
+        echo "$(cat sha256.txt)" | shasum --algorithm 256 --check \
+            || { echo "Checksums did not match for downloaded dashboard assets!"; exit 1; } ;;
+    Linux)
+        echo "$(cat sha256.txt)" | sha256sum --check -- \
+            || { echo "Checksums did not match for downloaded dashboard assets!"; exit 1; } ;;
+    *)
+        echo "The '$(uname -s)' operating system is not supported as a build host for the dashboard" >&2
+        exit 1
+esac
+
+# Extract the assets and clean up.
+tar -xzf build.tar.gz -C "$STATIC_DIR"
+rm sha256.txt
+rm build.tar.gz
+
+echo "Successfully download dashboard assets to $STATIC_DIR"

src/api/Cargo.toml

@@ -10,7 +10,7 @@ common-base = { path = "../common/base" }
 common-error = { path = "../common/error" }
 common-time = { path = "../common/time" }
 datatypes = { path = "../datatypes" }
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "eb760d219206c77dd3a105ecb6a3ba97d9d650ec" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "d3861c34f7920238869d0d4e50dc1e6b189d2a6b" }
 prost.workspace = true
 snafu = { version = "0.7", features = ["backtraces"] }
 tonic.workspace = true

src/catalog/src/error.rs

@@ -219,6 +219,12 @@ pub enum Error {
         #[snafu(backtrace)]
         source: table::error::Error,
     },
+
+    #[snafu(display("Invalid system table definition: {err_msg}"))]
+    InvalidSystemTableDef {
+        err_msg: String,
+        backtrace: Backtrace,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -231,7 +237,8 @@ impl ErrorExt for Error {
             | Error::TableNotFound { .. }
             | Error::IllegalManagerState { .. }
             | Error::CatalogNotFound { .. }
-            | Error::InvalidEntryType { .. } => StatusCode::Unexpected,
+            | Error::InvalidEntryType { .. }
+            | Error::InvalidSystemTableDef { .. } => StatusCode::Unexpected,
 
             Error::SystemCatalog { .. }
             | Error::EmptyValue { .. }

src/catalog/src/lib.rs

@@ -20,7 +20,7 @@ use std::sync::Arc;
 
 use api::v1::meta::{RegionStat, TableName};
 use common_telemetry::{info, warn};
-use snafu::{OptionExt, ResultExt};
+use snafu::ResultExt;
 use table::engine::{EngineContext, TableEngineRef};
 use table::metadata::TableId;
 use table::requests::CreateTableRequest;
@@ -228,34 +228,25 @@ pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
 
 /// The stat of regions in the datanode node.
 /// The number of regions can be got from len of vec.
-pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> Result<(u64, Vec<RegionStat>)> {
+///
+/// Ignores any errors occurred during iterating regions. The intention of this method is to
+/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
+/// "try our best" job.
+pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
     let mut region_number: u64 = 0;
     let mut region_stats = Vec::new();
 
-    for catalog_name in catalog_manager.catalog_names()? {
-        let catalog =
-            catalog_manager
-                .catalog(&catalog_name)?
-                .context(error::CatalogNotFoundSnafu {
-                    catalog_name: &catalog_name,
-                })?;
+    let Ok(catalog_names) = catalog_manager.catalog_names() else { return (region_number, region_stats) };
+    for catalog_name in catalog_names {
+        let Ok(Some(catalog)) = catalog_manager.catalog(&catalog_name) else { continue };
 
-        for schema_name in catalog.schema_names()? {
-            let schema = catalog
-                .schema(&schema_name)?
-                .context(error::SchemaNotFoundSnafu {
-                    catalog: &catalog_name,
-                    schema: &schema_name,
-                })?;
+        let Ok(schema_names) = catalog.schema_names() else { continue };
+        for schema_name in schema_names {
+            let Ok(Some(schema)) = catalog.schema(&schema_name) else { continue };
 
-            for table_name in schema.table_names()? {
-                let table =
-                    schema
-                        .table(&table_name)
-                        .await?
-                        .context(error::TableNotFoundSnafu {
-                            table_info: &table_name,
-                        })?;
+            let Ok(table_names) = schema.table_names() else { continue };
+            for table_name in table_names {
+                let Ok(Some(table)) = schema.table(&table_name).await else { continue };
 
                 let region_numbers = &table.table_info().meta.region_numbers;
                 region_number += region_numbers.len() as u64;
@@ -282,6 +273,5 @@ pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> Result<(u64,
             }
         }
     }
-
-    Ok((region_number, region_stats))
+    (region_number, region_stats)
 }

src/catalog/src/system.rs

@@ -400,7 +400,7 @@ mod tests {
     use log_store::NoopLogStore;
     use mito::config::EngineConfig;
     use mito::engine::MitoEngine;
-    use object_store::{ObjectStore, ObjectStoreBuilder};
+    use object_store::ObjectStore;
     use storage::compaction::noop::NoopCompactionScheduler;
     use storage::config::EngineConfig as StorageEngineConfig;
     use storage::EngineImpl;
@@ -482,11 +482,9 @@ mod tests {
     pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
         let dir = create_temp_dir("system-table-test");
         let store_dir = dir.path().to_string_lossy();
-        let accessor = object_store::services::Fs::default()
-            .root(&store_dir)
-            .build()
-            .unwrap();
-        let object_store = ObjectStore::new(accessor).finish();
+        let mut builder = object_store::services::Fs::default();
+        builder.root(&store_dir);
+        let object_store = ObjectStore::new(builder).unwrap().finish();
         let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
         let table_engine = Arc::new(MitoEngine::new(
             EngineConfig::default(),

src/catalog/src/table_source.rs

@@ -16,7 +16,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use common_catalog::format_full_table_name;
-use datafusion::common::{OwnedTableReference, ResolvedTableReference, TableReference};
+use datafusion::common::{ResolvedTableReference, TableReference};
 use datafusion::datasource::provider_as_source;
 use datafusion::logical_expr::TableSource;
 use session::context::QueryContext;
@@ -87,9 +87,8 @@ impl DfTableSourceProvider {
 
     pub async fn resolve_table(
         &mut self,
-        table_ref: OwnedTableReference,
+        table_ref: TableReference<'_>,
     ) -> Result<Arc<dyn TableSource>> {
-        let table_ref = table_ref.as_table_reference();
         let table_ref = self.resolve_table_ref(table_ref)?;
 
         let resolved_name = table_ref.to_string();

src/client/src/client.rs

@@ -15,6 +15,8 @@
 use std::sync::Arc;
 
 use api::v1::greptime_database_client::GreptimeDatabaseClient;
+use api::v1::health_check_client::HealthCheckClient;
+use api::v1::HealthCheckRequest;
 use arrow_flight::flight_service_client::FlightServiceClient;
 use common_grpc::channel_manager::ChannelManager;
 use parking_lot::RwLock;
@@ -153,6 +155,13 @@ impl Client {
             inner: GreptimeDatabaseClient::new(channel),
         })
     }
+
+    pub async fn health_check(&self) -> Result<()> {
+        let (_, channel) = self.find_channel()?;
+        let mut client = HealthCheckClient::new(channel);
+        client.health_check(HealthCheckRequest {}).await?;
+        Ok(())
+    }
 }
 
 #[cfg(test)]

src/client/src/database.rs

@@ -35,25 +35,44 @@ use crate::error::{
 };
 use crate::{error, Client, Result};
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 pub struct Database {
     // The "catalog" and "schema" to be used in processing the requests at the server side.
     // They are the "hint" or "context", just like how the "database" in "USE" statement is treated in MySQL.
     // They will be carried in the request header.
     catalog: String,
     schema: String,
+    // The dbname follows naming rule as out mysql, postgres and http
+    // protocol. The server treat dbname in priority of catalog/schema.
+    dbname: String,
 
     client: Client,
     ctx: FlightContext,
 }
 
 impl Database {
+    /// Create database service client using catalog and schema
     pub fn new(catalog: impl Into<String>, schema: impl Into<String>, client: Client) -> Self {
         Self {
             catalog: catalog.into(),
             schema: schema.into(),
             client,
-            ctx: FlightContext::default(),
+            ..Default::default()
+        }
+    }
+
+    /// Create database service client using dbname.
+    ///
+    /// This API is designed for external usage. `dbname` is:
+    ///
+    /// - the name of database when using GreptimeDB standalone or cluster
+    /// - the name provided by GreptimeCloud or other multi-tenant GreptimeDB
+    /// environment
+    pub fn new_with_dbname(dbname: impl Into<String>, client: Client) -> Self {
+        Self {
+            dbname: dbname.into(),
+            client,
+            ..Default::default()
         }
     }
 
@@ -73,6 +92,14 @@ impl Database {
         self.schema = schema.into();
     }
 
+    pub fn dbname(&self) -> &String {
+        &self.dbname
+    }
+
+    pub fn set_dbname(&mut self, dbname: impl Into<String>) {
+        self.dbname = dbname.into();
+    }
+
     pub fn set_auth(&mut self, auth: AuthScheme) {
         self.ctx.auth_header = Some(AuthHeader {
             auth_scheme: Some(auth),
@@ -86,6 +113,7 @@ impl Database {
                 catalog: self.catalog.clone(),
                 schema: self.schema.clone(),
                 authorization: self.ctx.auth_header.clone(),
+                dbname: self.dbname.clone(),
             }),
             request: Some(Request::Insert(request)),
         };
@@ -167,6 +195,7 @@ impl Database {
                 catalog: self.catalog.clone(),
                 schema: self.schema.clone(),
                 authorization: self.ctx.auth_header.clone(),
+                dbname: self.dbname.clone(),
             }),
             request: Some(request),
         };

src/cmd/src/bin/greptime.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![doc = include_str!("../../../../README.md")]
+
 use std::fmt;
 
 use clap::Parser;

src/cmd/src/datanode.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::time::Duration;
+
 use clap::Parser;
 use common_telemetry::logging;
 use datanode::datanode::{
@@ -86,6 +88,10 @@ struct StartCommand {
     wal_dir: Option<String>,
     #[clap(long)]
     procedure_dir: Option<String>,
+    #[clap(long)]
+    http_addr: Option<String>,
+    #[clap(long)]
+    http_timeout: Option<u64>,
 }
 
 impl StartCommand {
@@ -146,7 +152,7 @@ impl TryFrom<StartCommand> for DatanodeOptions {
         }
 
         if let Some(data_dir) = cmd.data_dir {
-            opts.storage = ObjectStoreConfig::File(FileConfig { data_dir });
+            opts.storage.store = ObjectStoreConfig::File(FileConfig { data_dir });
         }
 
         if let Some(wal_dir) = cmd.wal_dir {
@@ -155,6 +161,12 @@ impl TryFrom<StartCommand> for DatanodeOptions {
         if let Some(procedure_dir) = cmd.procedure_dir {
             opts.procedure = Some(ProcedureConfig::from_file_path(procedure_dir));
         }
+        if let Some(http_addr) = cmd.http_addr {
+            opts.http_opts.addr = http_addr
+        }
+        if let Some(http_timeout) = cmd.http_timeout {
+            opts.http_opts.timeout = Duration::from_secs(http_timeout)
+        }
 
         Ok(opts)
     }
@@ -166,8 +178,9 @@ mod tests {
     use std::io::Write;
     use std::time::Duration;
 
+    use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::datanode::{CompactionConfig, ObjectStoreConfig};
+    use datanode::datanode::{CompactionConfig, ObjectStoreConfig, RegionManifestConfig};
     use servers::Mode;
 
     use super::*;
@@ -203,10 +216,14 @@ mod tests {
             type = "File"
             data_dir = "/tmp/greptimedb/data/"
 
-            [compaction]
-            max_inflight_tasks = 4
-            max_files_in_level0 = 8
+            [storage.compaction]
+            max_inflight_tasks = 3
+            max_files_in_level0 = 7
             max_purge_tasks = 32
+
+            [storage.manifest]
+            checkpoint_margin = 9
+            gc_duration = '7s'
         "#;
         write!(file, "{}", toml_str).unwrap();
 
@@ -237,9 +254,9 @@ mod tests {
         assert_eq!(3000, timeout_millis);
         assert!(tcp_nodelay);
 
-        match options.storage {
-            ObjectStoreConfig::File(FileConfig { data_dir }) => {
-                assert_eq!("/tmp/greptimedb/data/".to_string(), data_dir)
+        match &options.storage.store {
+            ObjectStoreConfig::File(FileConfig { data_dir, .. }) => {
+                assert_eq!("/tmp/greptimedb/data/", data_dir)
             }
             ObjectStoreConfig::S3 { .. } => unreachable!(),
             ObjectStoreConfig::Oss { .. } => unreachable!(),
@@ -247,11 +264,19 @@ mod tests {
 
         assert_eq!(
             CompactionConfig {
-                max_inflight_tasks: 4,
-                max_files_in_level0: 8,
+                max_inflight_tasks: 3,
+                max_files_in_level0: 7,
                 max_purge_tasks: 32,
+                sst_write_buffer_size: ReadableSize::mb(8),
             },
-            options.compaction
+            options.storage.compaction,
+        );
+        assert_eq!(
+            RegionManifestConfig {
+                checkpoint_margin: Some(9),
+                gc_duration: Some(Duration::from_secs(7)),
+            },
+            options.storage.manifest,
         );
     }
 

src/cmd/src/metasrv.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::time::Duration;
+
 use clap::Parser;
 use common_telemetry::{info, logging, warn};
 use meta_srv::bootstrap::MetaSrvInstance;
@@ -80,6 +82,10 @@ struct StartCommand {
     selector: Option<String>,
     #[clap(long)]
     use_memory_store: bool,
+    #[clap(long)]
+    http_addr: Option<String>,
+    #[clap(long)]
+    http_timeout: Option<u64>,
 }
 
 impl StartCommand {
@@ -128,6 +134,13 @@ impl TryFrom<StartCommand> for MetaSrvOptions {
             opts.use_memory_store = true;
         }
 
+        if let Some(http_addr) = cmd.http_addr {
+            opts.http_opts.addr = http_addr;
+        }
+        if let Some(http_timeout) = cmd.http_timeout {
+            opts.http_opts.timeout = Duration::from_secs(http_timeout);
+        }
+
         Ok(opts)
     }
 }
@@ -150,6 +163,8 @@ mod tests {
             config_file: None,
             selector: Some("LoadBased".to_string()),
             use_memory_store: false,
+            http_addr: None,
+            http_timeout: None,
         };
         let options: MetaSrvOptions = cmd.try_into().unwrap();
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
@@ -178,6 +193,8 @@ mod tests {
             selector: None,
             config_file: Some(file.path().to_str().unwrap().to_string()),
             use_memory_store: false,
+            http_addr: None,
+            http_timeout: None,
         };
         let options: MetaSrvOptions = cmd.try_into().unwrap();
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);

src/cmd/src/standalone.rs

@@ -17,9 +17,7 @@ use std::sync::Arc;
 use clap::Parser;
 use common_base::Plugins;
 use common_telemetry::info;
-use datanode::datanode::{
-    CompactionConfig, Datanode, DatanodeOptions, ObjectStoreConfig, ProcedureConfig, WalConfig,
-};
+use datanode::datanode::{Datanode, DatanodeOptions, ProcedureConfig, StorageConfig, WalConfig};
 use datanode::instance::InstanceRef;
 use frontend::frontend::FrontendOptions;
 use frontend::grpc::GrpcOptions;
@@ -82,8 +80,7 @@ pub struct StandaloneOptions {
     pub prometheus_options: Option<PrometheusOptions>,
     pub prom_options: Option<PromOptions>,
     pub wal: WalConfig,
-    pub storage: ObjectStoreConfig,
-    pub compaction: CompactionConfig,
+    pub storage: StorageConfig,
     pub procedure: Option<ProcedureConfig>,
 }
 
@@ -101,8 +98,7 @@ impl Default for StandaloneOptions {
             prometheus_options: Some(PrometheusOptions::default()),
             prom_options: Some(PromOptions::default()),
             wal: WalConfig::default(),
-            storage: ObjectStoreConfig::default(),
-            compaction: CompactionConfig::default(),
+            storage: StorageConfig::default(),
             procedure: None,
         }
     }
@@ -129,7 +125,6 @@ impl StandaloneOptions {
             enable_memory_catalog: self.enable_memory_catalog,
             wal: self.wal,
             storage: self.storage,
-            compaction: self.compaction,
             procedure: self.procedure,
             ..Default::default()
         }
@@ -241,8 +236,9 @@ async fn build_frontend(
     plugins: Arc<Plugins>,
     datanode_instance: InstanceRef,
 ) -> Result<FeInstance> {
-    let mut frontend_instance = FeInstance::new_standalone(datanode_instance.clone());
-    frontend_instance.set_script_handler(datanode_instance);
+    let mut frontend_instance = FeInstance::try_new_standalone(datanode_instance.clone())
+        .await
+        .context(StartFrontendSnafu)?;
     frontend_instance.set_plugins(plugins.clone());
     Ok(frontend_instance)
 }

src/common/base/src/readable_size.rs

@@ -53,6 +53,10 @@ impl ReadableSize {
     pub const fn as_mb(self) -> u64 {
         self.0 / MIB
     }
+
+    pub const fn as_bytes(self) -> u64 {
+        self.0
+    }
 }
 
 impl Div<u64> for ReadableSize {

src/common/datasource/src/lister.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use futures::{future, TryStreamExt};
-use object_store::{Object, ObjectStore};
+use object_store::{Entry, ObjectStore};
 use regex::Regex;
 use snafu::ResultExt;
 
@@ -46,13 +46,12 @@ impl Lister {
         }
     }
 
-    pub async fn list(&self) -> Result<Vec<Object>> {
+    pub async fn list(&self) -> Result<Vec<Entry>> {
         match &self.source {
             Source::Dir => {
                 let streamer = self
                     .object_store
-                    .object(&self.path)
-                    .list()
+                    .list(&self.path)
                     .await
                     .context(error::ListObjectsSnafu { path: &self.path })?;
 
@@ -70,11 +69,14 @@ impl Lister {
                     .context(error::ListObjectsSnafu { path: &self.path })
             }
             Source::Filename(filename) => {
-                let obj = self
-                    .object_store
-                    .object(&format!("{}{}", self.path, filename));
-
-                Ok(vec![obj])
+                // make sure this file exists
+                let file_full_path = format!("{}{}", self.path, filename);
+                let _ = self.object_store.stat(&file_full_path).await.context(
+                    error::ListObjectsSnafu {
+                        path: &file_full_path,
+                    },
+                )?;
+                Ok(vec![Entry::new(&file_full_path)])
             }
         }
     }

src/common/datasource/src/object_store/fs.rs

@@ -13,16 +13,16 @@
 // limitations under the License.
 
 use object_store::services::Fs;
-use object_store::{ObjectStore, ObjectStoreBuilder};
+use object_store::ObjectStore;
 use snafu::ResultExt;
 
-use crate::error::{self, Result};
+use crate::error::{BuildBackendSnafu, Result};
 
 pub fn build_fs_backend(root: &str) -> Result<ObjectStore> {
-    let accessor = Fs::default()
-        .root(root)
-        .build()
-        .context(error::BuildBackendSnafu)?;
-
-    Ok(ObjectStore::new(accessor).finish())
+    let mut builder = Fs::default();
+    builder.root(root);
+    let object_store = ObjectStore::new(builder)
+        .context(BuildBackendSnafu)?
+        .finish();
+    Ok(object_store)
 }

src/common/datasource/src/object_store/s3.rs

@@ -15,7 +15,7 @@
 use std::collections::HashMap;
 
 use object_store::services::S3;
-use object_store::{ObjectStore, ObjectStoreBuilder};
+use object_store::ObjectStore;
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
@@ -73,7 +73,7 @@ pub fn build_s3_backend(
         }
     }
 
-    let accessor = builder.build().context(error::BuildBackendSnafu)?;
-
-    Ok(ObjectStore::new(accessor).finish())
+    Ok(ObjectStore::new(builder)
+        .context(error::BuildBackendSnafu)?
+        .finish())
 }

src/common/function-macro/src/range_fn.rs

@@ -36,6 +36,8 @@ macro_rules! ok {
 }
 
 pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
+    let mut result = TokenStream::new();
+
     // extract arg map
     let arg_pairs = parse_macro_input!(args as AttributeArgs);
     let arg_span = arg_pairs[0].span();
@@ -59,12 +61,17 @@ pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenSt
     let arg_types = ok!(extract_input_types(inputs));
 
     // build the struct and its impl block
-    let struct_code = build_struct(
-        attrs,
-        vis,
-        ok!(get_ident(&arg_map, "name", arg_span)),
-        ok!(get_ident(&arg_map, "display_name", arg_span)),
-    );
+    // only do this when `display_name` is specified
+    if let Ok(display_name) = get_ident(&arg_map, "display_name", arg_span) {
+        let struct_code = build_struct(
+            attrs,
+            vis,
+            ok!(get_ident(&arg_map, "name", arg_span)),
+            display_name,
+        );
+        result.extend(struct_code);
+    }
+
     let calc_fn_code = build_calc_fn(
         ok!(get_ident(&arg_map, "name", arg_span)),
         arg_types,
@@ -77,8 +84,6 @@ pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenSt
     }
     .into();
 
-    let mut result = TokenStream::new();
-    result.extend(struct_code);
     result.extend(calc_fn_code);
     result.extend(input_fn_code);
     result

src/common/function/src/scalars/timestamp.rs

@@ -11,11 +11,17 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+use std::sync::Arc;
+mod to_unixtime;
+
+use to_unixtime::ToUnixtimeFunction;
 
 use crate::scalars::function_registry::FunctionRegistry;
 
 pub(crate) struct TimestampFunction;
 
 impl TimestampFunction {
-    pub fn register(_registry: &FunctionRegistry) {}
+    pub fn register(registry: &FunctionRegistry) {
+        registry.register(Arc::new(ToUnixtimeFunction::default()));
+    }
 }

src/common/function/src/scalars/timestamp/to_unixtime.rs

@@ -0,0 +1,148 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use common_query::error::{self, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::{Signature, Volatility};
+use common_time::timestamp::TimeUnit;
+use common_time::Timestamp;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::types::StringType;
+use datatypes::vectors::{Int64Vector, StringVector, Vector, VectorRef};
+use snafu::ensure;
+
+use crate::scalars::function::{Function, FunctionContext};
+
+#[derive(Clone, Debug, Default)]
+pub struct ToUnixtimeFunction;
+
+const NAME: &str = "to_unixtime";
+
+fn convert_to_seconds(arg: &str) -> Option<i64> {
+    match Timestamp::from_str(arg) {
+        Ok(ts) => {
+            let sec_mul = (TimeUnit::Second.factor() / ts.unit().factor()) as i64;
+            Some(ts.value().div_euclid(sec_mul))
+        }
+        Err(_err) => None,
+    }
+}
+
+impl Function for ToUnixtimeFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::timestamp_second_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::exact(
+            vec![ConcreteDataType::String(StringType)],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            error::InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly one, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        match columns[0].data_type() {
+            ConcreteDataType::String(_) => {
+                let array = columns[0].to_arrow_array();
+                let vector = StringVector::try_from_arrow_array(&array).unwrap();
+                Ok(Arc::new(Int64Vector::from(
+                    (0..vector.len())
+                        .map(|i| convert_to_seconds(&vector.get(i).to_string()))
+                        .collect::<Vec<_>>(),
+                )))
+            }
+            _ => UnsupportedInputDataTypeSnafu {
+                function: NAME,
+                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+            }
+            .fail(),
+        }
+    }
+}
+
+impl fmt::Display for ToUnixtimeFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "TO_UNIXTIME")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use common_query::prelude::TypeSignature;
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::types::StringType;
+    use datatypes::value::Value;
+    use datatypes::vectors::StringVector;
+
+    use super::{ToUnixtimeFunction, *};
+    use crate::scalars::Function;
+
+    #[test]
+    fn test_to_unixtime() {
+        let f = ToUnixtimeFunction::default();
+        assert_eq!("to_unixtime", f.name());
+        assert_eq!(
+            ConcreteDataType::timestamp_second_datatype(),
+            f.return_type(&[]).unwrap()
+        );
+
+        assert!(matches!(f.signature(),
+                         Signature {
+                            type_signature: TypeSignature::Exact(valid_types),
+                            volatility: Volatility::Immutable
+                         } if  valid_types == vec![ConcreteDataType::String(StringType)]
+        ));
+
+        let times = vec![
+            Some("2023-03-01T06:35:02Z"),
+            None,
+            Some("2022-06-30T23:59:60Z"),
+            Some("invalid_time_stamp"),
+        ];
+        let results = vec![Some(1677652502), None, Some(1656633600), None];
+        let args: Vec<VectorRef> = vec![Arc::new(StringVector::from(times.clone()))];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+        assert_eq!(4, vector.len());
+        for (i, _t) in times.iter().enumerate() {
+            let v = vector.get(i);
+            if i == 1 || i == 3 {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Int64(ts) => {
+                    assert_eq!(ts, (*results.get(i).unwrap()).unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+}

src/common/procedure/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 async-trait.workspace = true
+async-stream.workspace = true
 common-error = { path = "../error" }
 common-runtime = { path = "../runtime" }
 common-telemetry = { path = "../telemetry" }

src/common/procedure/src/error.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::any::Any;
+use std::string::FromUtf8Error;
 use std::sync::Arc;
 
 use common_error::prelude::*;
@@ -47,10 +48,11 @@ pub enum Error {
         backtrace: Backtrace,
     },
 
-    #[snafu(display("Failed to put {}, source: {}", key, source))]
+    #[snafu(display("Failed to put state, key: '{key}', source: {source}"))]
     PutState {
         key: String,
-        source: object_store::Error,
+        #[snafu(backtrace)]
+        source: BoxedError,
     },
 
     #[snafu(display("Failed to delete {}, source: {}", key, source))]
@@ -59,10 +61,18 @@ pub enum Error {
         source: object_store::Error,
     },
 
-    #[snafu(display("Failed to list {}, source: {}", path, source))]
+    #[snafu(display("Failed to delete keys: '{keys}', source: {source}"))]
+    DeleteStates {
+        keys: String,
+        #[snafu(backtrace)]
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to list state, path: '{path}', source: {source}"))]
     ListState {
         path: String,
-        source: object_store::Error,
+        #[snafu(backtrace)]
+        source: BoxedError,
     },
 
     #[snafu(display("Failed to read {}, source: {}", key, source))]
@@ -107,6 +117,9 @@ pub enum Error {
         source: Arc<Error>,
         procedure_id: ProcedureId,
     },
+
+    #[snafu(display("Corrupted data, error: {source}"))]
+    CorruptedData { source: FromUtf8Error },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -114,11 +127,13 @@ pub type Result<T> = std::result::Result<T, Error>;
 impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
-            Error::External { source } => source.status_code(),
+            Error::External { source }
+            | Error::PutState { source, .. }
+            | Error::DeleteStates { source, .. }
+            | Error::ListState { source, .. } => source.status_code(),
+
             Error::ToJson { .. }
-            | Error::PutState { .. }
             | Error::DeleteState { .. }
-            | Error::ListState { .. }
             | Error::ReadState { .. }
             | Error::FromJson { .. }
             | Error::RetryTimesExceeded { .. }
@@ -127,7 +142,7 @@ impl ErrorExt for Error {
             Error::LoaderConflict { .. } | Error::DuplicateProcedure { .. } => {
                 StatusCode::InvalidArguments
             }
-            Error::ProcedurePanic { .. } => StatusCode::Unexpected,
+            Error::ProcedurePanic { .. } | Error::CorruptedData { .. } => StatusCode::Unexpected,
             Error::ProcedureExec { source, .. } => source.status_code(),
         }
     }

src/common/procedure/src/lib.rs

@@ -17,7 +17,7 @@
 pub mod error;
 pub mod local;
 mod procedure;
-mod store;
+pub mod store;
 pub mod watcher;
 
 pub use crate::error::{Error, Result};

src/common/procedure/src/local.rs

@@ -22,7 +22,6 @@ use std::time::Duration;
 use async_trait::async_trait;
 use backon::ExponentialBuilder;
 use common_telemetry::logging;
-use object_store::ObjectStore;
 use snafu::ensure;
 use tokio::sync::watch::{self, Receiver, Sender};
 use tokio::sync::Notify;
@@ -31,7 +30,7 @@ use crate::error::{DuplicateProcedureSnafu, LoaderConflictSnafu, Result};
 use crate::local::lock::LockMap;
 use crate::local::runner::Runner;
 use crate::procedure::BoxedProcedureLoader;
-use crate::store::{ObjectStateStore, ProcedureMessage, ProcedureStore, StateStoreRef};
+use crate::store::{ProcedureMessage, ProcedureStore, StateStoreRef};
 use crate::{
     BoxedProcedure, ContextProvider, LockKey, ProcedureId, ProcedureManager, ProcedureState,
     ProcedureWithId, Watcher,
@@ -291,12 +290,19 @@ impl ManagerContext {
 /// Config for [LocalManager].
 #[derive(Debug)]
 pub struct ManagerConfig {
-    /// Object store
-    pub object_store: ObjectStore,
     pub max_retry_times: usize,
     pub retry_delay: Duration,
 }
 
+impl Default for ManagerConfig {
+    fn default() -> Self {
+        Self {
+            max_retry_times: 3,
+            retry_delay: Duration::from_millis(500),
+        }
+    }
+}
+
 /// A [ProcedureManager] that maintains procedure states locally.
 pub struct LocalManager {
     manager_ctx: Arc<ManagerContext>,
@@ -307,10 +313,10 @@ pub struct LocalManager {
 
 impl LocalManager {
     /// Create a new [LocalManager] with specific `config`.
-    pub fn new(config: ManagerConfig) -> LocalManager {
+    pub fn new(config: ManagerConfig, state_store: StateStoreRef) -> LocalManager {
         LocalManager {
             manager_ctx: Arc::new(ManagerContext::new()),
-            state_store: Arc::new(ObjectStateStore::new(config.object_store)),
+            state_store,
             max_retry_times: config.max_retry_times,
             retry_delay: config.retry_delay,
         }
@@ -423,7 +429,7 @@ impl ProcedureManager for LocalManager {
 mod test_util {
     use common_test_util::temp_dir::TempDir;
     use object_store::services::Fs as Builder;
-    use object_store::ObjectStoreBuilder;
+    use object_store::ObjectStore;
 
     use super::*;
 
@@ -433,8 +439,9 @@ mod test_util {
 
     pub(crate) fn new_object_store(dir: &TempDir) -> ObjectStore {
         let store_dir = dir.path().to_str().unwrap();
-        let accessor = Builder::default().root(store_dir).build().unwrap();
-        ObjectStore::new(accessor).finish()
+        let mut builder = Builder::default();
+        builder.root(store_dir);
+        ObjectStore::new(builder).unwrap().finish()
     }
 }
 
@@ -446,6 +453,7 @@ mod tests {
 
     use super::*;
     use crate::error::Error;
+    use crate::store::ObjectStateStore;
     use crate::{Context, Procedure, Status};
 
     #[test]
@@ -554,11 +562,11 @@ mod tests {
     fn test_register_loader() {
         let dir = create_temp_dir("register");
         let config = ManagerConfig {
-            object_store: test_util::new_object_store(&dir),
             max_retry_times: 3,
             retry_delay: Duration::from_millis(500),
         };
-        let manager = LocalManager::new(config);
+        let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
+        let manager = LocalManager::new(config, state_store);
 
         manager
             .register_loader("ProcedureToLoad", ProcedureToLoad::loader())
@@ -575,11 +583,11 @@ mod tests {
         let dir = create_temp_dir("recover");
         let object_store = test_util::new_object_store(&dir);
         let config = ManagerConfig {
-            object_store: object_store.clone(),
             max_retry_times: 3,
             retry_delay: Duration::from_millis(500),
         };
-        let manager = LocalManager::new(config);
+        let state_store = Arc::new(ObjectStateStore::new(object_store.clone()));
+        let manager = LocalManager::new(config, state_store);
 
         manager
             .register_loader("ProcedureToLoad", ProcedureToLoad::loader())
@@ -621,11 +629,11 @@ mod tests {
     async fn test_submit_procedure() {
         let dir = create_temp_dir("submit");
         let config = ManagerConfig {
-            object_store: test_util::new_object_store(&dir),
             max_retry_times: 3,
             retry_delay: Duration::from_millis(500),
         };
-        let manager = LocalManager::new(config);
+        let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
+        let manager = LocalManager::new(config, state_store);
 
         let procedure_id = ProcedureId::random();
         assert!(manager
@@ -669,11 +677,11 @@ mod tests {
     async fn test_state_changed_on_err() {
         let dir = create_temp_dir("on_err");
         let config = ManagerConfig {
-            object_store: test_util::new_object_store(&dir),
             max_retry_times: 3,
             retry_delay: Duration::from_millis(500),
         };
-        let manager = LocalManager::new(config);
+        let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
+        let manager = LocalManager::new(config, state_store);
 
         #[derive(Debug)]
         struct MockProcedure {

src/common/procedure/src/local/runner.rs

@@ -473,8 +473,7 @@ mod tests {
 
     async fn check_files(object_store: &ObjectStore, procedure_id: ProcedureId, files: &[&str]) {
         let dir = format!("{procedure_id}/");
-        let object = object_store.object(&dir);
-        let lister = object.list().await.unwrap();
+        let lister = object_store.list(&dir).await.unwrap();
         let mut files_in_dir: Vec<_> = lister
             .map_ok(|de| de.name().to_string())
             .try_collect()

src/common/procedure/src/store.rs

@@ -26,7 +26,7 @@ use crate::error::{Result, ToJsonSnafu};
 pub(crate) use crate::store::state_store::{ObjectStateStore, StateStoreRef};
 use crate::{BoxedProcedure, ProcedureId};
 
-mod state_store;
+pub mod state_store;
 
 /// Serialized data of a procedure.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -248,15 +248,15 @@ mod tests {
     use async_trait::async_trait;
     use common_test_util::temp_dir::{create_temp_dir, TempDir};
     use object_store::services::Fs as Builder;
-    use object_store::ObjectStoreBuilder;
 
     use super::*;
     use crate::{Context, LockKey, Procedure, Status};
 
     fn procedure_store_for_test(dir: &TempDir) -> ProcedureStore {
         let store_dir = dir.path().to_str().unwrap();
-        let accessor = Builder::default().root(store_dir).build().unwrap();
-        let object_store = ObjectStore::new(accessor).finish();
+        let mut builder = Builder::default();
+        builder.root(store_dir);
+        let object_store = ObjectStore::new(builder).unwrap().finish();
 
         ProcedureStore::from(object_store)
     }

src/common/procedure/src/store/state_store.rs

@@ -15,22 +15,25 @@
 use std::pin::Pin;
 use std::sync::Arc;
 
+use async_stream::try_stream;
 use async_trait::async_trait;
-use futures::{Stream, TryStreamExt};
-use object_store::{ObjectMode, ObjectStore};
+use common_error::ext::PlainError;
+use common_error::prelude::{BoxedError, StatusCode};
+use futures::{Stream, StreamExt};
+use object_store::{EntryMode, Metakey, ObjectStore};
 use snafu::ResultExt;
 
-use crate::error::{DeleteStateSnafu, Error, PutStateSnafu, Result};
+use crate::error::{DeleteStateSnafu, ListStateSnafu, PutStateSnafu, Result};
 
 /// Key value from state store.
-type KeyValue = (String, Vec<u8>);
+pub type KeyValue = (String, Vec<u8>);
 
 /// Stream that yields [KeyValue].
-type KeyValueStream = Pin<Box<dyn Stream<Item = Result<KeyValue>> + Send>>;
+pub type KeyValueStream = Pin<Box<dyn Stream<Item = Result<KeyValue>> + Send>>;
 
 /// Storage layer for persisting procedure's state.
 #[async_trait]
-pub(crate) trait StateStore: Send + Sync {
+pub trait StateStore: Send + Sync {
     /// Puts `key` and `value` into the store.
     async fn put(&self, key: &str, value: Vec<u8>) -> Result<()>;
 
@@ -50,13 +53,13 @@ pub(crate) type StateStoreRef = Arc<dyn StateStore>;
 
 /// [StateStore] based on [ObjectStore].
 #[derive(Debug)]
-pub(crate) struct ObjectStateStore {
+pub struct ObjectStateStore {
     store: ObjectStore,
 }
 
 impl ObjectStateStore {
     /// Returns a new [ObjectStateStore] with specific `store`.
-    pub(crate) fn new(store: ObjectStore) -> ObjectStateStore {
+    pub fn new(store: ObjectStore) -> ObjectStateStore {
         ObjectStateStore { store }
     }
 }
@@ -64,49 +67,83 @@ impl ObjectStateStore {
 #[async_trait]
 impl StateStore for ObjectStateStore {
     async fn put(&self, key: &str, value: Vec<u8>) -> Result<()> {
-        let object = self.store.object(key);
-        object.write(value).await.context(PutStateSnafu { key })
+        self.store
+            .write(key, value)
+            .await
+            .map_err(|e| {
+                BoxedError::new(PlainError::new(
+                    e.to_string(),
+                    StatusCode::StorageUnavailable,
+                ))
+            })
+            .context(PutStateSnafu { key })
     }
 
     async fn walk_top_down(&self, path: &str) -> Result<KeyValueStream> {
         let path_string = path.to_string();
 
-        let lister = self
+        let mut lister = self
             .store
-            .object(path)
-            .scan()
+            .scan(path)
             .await
-            .map_err(|e| Error::ListState {
+            .map_err(|e| {
+                BoxedError::new(PlainError::new(
+                    e.to_string(),
+                    StatusCode::StorageUnavailable,
+                ))
+            })
+            .with_context(|_| ListStateSnafu {
                 path: path_string.clone(),
-                source: e,
             })?;
 
-        let stream = lister
-            .try_filter_map(|entry| async move {
+        let store = self.store.clone();
+
+        let stream = try_stream!({
+            while let Some(res) = lister.next().await {
+                let entry = res
+                    .map_err(|e| {
+                        BoxedError::new(PlainError::new(
+                            e.to_string(),
+                            StatusCode::StorageUnavailable,
+                        ))
+                    })
+                    .context(ListStateSnafu { path: &path_string })?;
                 let key = entry.path();
-                let key_value = match entry.mode().await? {
-                    ObjectMode::FILE => {
-                        let value = entry.read().await?;
-
-                        Some((key.to_string(), value))
-                    }
-                    ObjectMode::DIR | ObjectMode::Unknown => None,
-                };
-
-                Ok(key_value)
-            })
-            .map_err(move |e| Error::ListState {
-                path: path_string.clone(),
-                source: e,
-            });
+                let metadata = store
+                    .metadata(&entry, Metakey::Mode)
+                    .await
+                    .map_err(|e| {
+                        BoxedError::new(PlainError::new(
+                            e.to_string(),
+                            StatusCode::StorageUnavailable,
+                        ))
+                    })
+                    .context(ListStateSnafu { path: key })?;
+                if let EntryMode::FILE = metadata.mode() {
+                    let value = store
+                        .read(key)
+                        .await
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                e.to_string(),
+                                StatusCode::StorageUnavailable,
+                            ))
+                        })
+                        .context(ListStateSnafu { path: key })?;
+                    yield (key.to_string(), value);
+                }
+            }
+        });
 
         Ok(Box::pin(stream))
     }
 
     async fn delete(&self, keys: &[String]) -> Result<()> {
         for key in keys {
-            let object = self.store.object(key);
-            object.delete().await.context(DeleteStateSnafu { key })?;
+            self.store
+                .delete(key)
+                .await
+                .context(DeleteStateSnafu { key })?;
         }
 
         Ok(())
@@ -116,8 +153,8 @@ impl StateStore for ObjectStateStore {
 #[cfg(test)]
 mod tests {
     use common_test_util::temp_dir::create_temp_dir;
+    use futures_util::TryStreamExt;
     use object_store::services::Fs as Builder;
-    use object_store::ObjectStoreBuilder;
 
     use super::*;
 
@@ -125,8 +162,10 @@ mod tests {
     async fn test_object_state_store() {
         let dir = create_temp_dir("state_store");
         let store_dir = dir.path().to_str().unwrap();
-        let accessor = Builder::default().root(store_dir).build().unwrap();
-        let object_store = ObjectStore::new(accessor).finish();
+        let mut builder = Builder::default();
+        builder.root(store_dir);
+
+        let object_store = ObjectStore::new(builder).unwrap().finish();
         let state_store = ObjectStateStore::new(object_store);
 
         let data: Vec<_> = state_store

src/common/recordbatch/src/error.rs

@@ -17,6 +17,7 @@ use std::any::Any;
 
 use common_error::ext::BoxedError;
 use common_error::prelude::*;
+use datatypes::prelude::ConcreteDataType;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -70,6 +71,26 @@ pub enum Error {
         source: datafusion_common::DataFusionError,
         backtrace: Backtrace,
     },
+
+    #[snafu(display("Column {} not exists in table {}", column_name, table_name))]
+    ColumnNotExists {
+        column_name: String,
+        table_name: String,
+        backtrace: Backtrace,
+    },
+
+    #[snafu(display(
+        "Failed to cast vector of type '{:?}' to type '{:?}', source: {}",
+        from_type,
+        to_type,
+        source
+    ))]
+    CastVector {
+        from_type: ConcreteDataType,
+        to_type: ConcreteDataType,
+        #[snafu(backtrace)]
+        source: datatypes::error::Error,
+    },
 }
 
 impl ErrorExt for Error {
@@ -81,11 +102,14 @@ impl ErrorExt for Error {
             | Error::CreateRecordBatches { .. }
             | Error::PollStream { .. }
             | Error::Format { .. }
-            | Error::InitRecordbatchStream { .. } => StatusCode::Internal,
+            | Error::InitRecordbatchStream { .. }
+            | Error::ColumnNotExists { .. } => StatusCode::Internal,
 
             Error::External { source } => source.status_code(),
 
-            Error::SchemaConversion { source, .. } => source.status_code(),
+            Error::SchemaConversion { source, .. } | Error::CastVector { source, .. } => {
+                source.status_code()
+            }
         }
     }
 

src/common/recordbatch/src/recordbatch.rs

@@ -12,14 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
+
 use datatypes::schema::SchemaRef;
 use datatypes::value::Value;
 use datatypes::vectors::{Helper, VectorRef};
 use serde::ser::{Error, SerializeStruct};
 use serde::{Serialize, Serializer};
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 
-use crate::error::{self, Result};
+use crate::error::{self, CastVectorSnafu, ColumnNotExistsSnafu, Result};
 use crate::DfRecordBatch;
 
 /// A two-dimensional batch of column-oriented data with a defined schema.
@@ -108,6 +110,41 @@ impl RecordBatch {
     pub fn rows(&self) -> RecordBatchRowIterator<'_> {
         RecordBatchRowIterator::new(self)
     }
+
+    pub fn column_vectors(
+        &self,
+        table_name: &str,
+        table_schema: SchemaRef,
+    ) -> Result<HashMap<String, VectorRef>> {
+        let mut vectors = HashMap::with_capacity(self.num_columns());
+
+        // column schemas in recordbatch must match its vectors, otherwise it's corrupted
+        for (vector_schema, vector) in self.schema.column_schemas().iter().zip(self.columns.iter())
+        {
+            let column_name = &vector_schema.name;
+            let column_schema =
+                table_schema
+                    .column_schema_by_name(column_name)
+                    .context(ColumnNotExistsSnafu {
+                        table_name,
+                        column_name,
+                    })?;
+            let vector = if vector_schema.data_type != column_schema.data_type {
+                vector
+                    .cast(&column_schema.data_type)
+                    .with_context(|_| CastVectorSnafu {
+                        from_type: vector.data_type(),
+                        to_type: column_schema.data_type.clone(),
+                    })?
+            } else {
+                vector.clone()
+            };
+
+            vectors.insert(column_name.clone(), vector);
+        }
+
+        Ok(vectors)
+    }
 }
 
 impl Serialize for RecordBatch {

src/common/runtime/Cargo.toml

@@ -5,6 +5,7 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
+async-trait.workspace = true
 common-error = { path = "../error" }
 common-telemetry = { path = "../telemetry" }
 metrics = "0.20"
@@ -12,6 +13,7 @@ once_cell = "1.12"
 paste.workspace = true
 snafu.workspace = true
 tokio.workspace = true
+tokio-util.workspace = true
 
 [dev-dependencies]
 tokio-test = "0.4"

src/common/runtime/src/error.rs

@@ -15,6 +15,7 @@
 use std::any::Any;
 
 use common_error::prelude::*;
+use tokio::task::JoinError;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -26,6 +27,19 @@ pub enum Error {
         source: std::io::Error,
         backtrace: Backtrace,
     },
+    #[snafu(display("Repeated task {} not started yet", name))]
+    IllegalState { name: String, backtrace: Backtrace },
+
+    #[snafu(display(
+        "Failed to wait for repeated task {} to stop, source: {}",
+        name,
+        source
+    ))]
+    WaitGcTaskStop {
+        name: String,
+        source: JoinError,
+        backtrace: Backtrace,
+    },
 }
 
 impl ErrorExt for Error {

src/common/runtime/src/lib.rs

@@ -14,7 +14,8 @@
 
 pub mod error;
 mod global;
-pub mod metric;
+mod metrics;
+mod repeated_task;
 pub mod runtime;
 
 pub use global::{
@@ -23,4 +24,5 @@ pub use global::{
     spawn_read, spawn_write, write_runtime,
 };
 
+pub use crate::repeated_task::{RepeatedTask, TaskFunction, TaskFunctionRef};
 pub use crate::runtime::{Builder, JoinError, JoinHandle, Runtime};

src/common/runtime/src/repeated_task.rs

@@ -0,0 +1,174 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use common_error::prelude::ErrorExt;
+use common_telemetry::logging;
+use snafu::{ensure, OptionExt, ResultExt};
+use tokio::sync::Mutex;
+use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
+
+use crate::error::{IllegalStateSnafu, Result, WaitGcTaskStopSnafu};
+use crate::Runtime;
+
+#[async_trait::async_trait]
+pub trait TaskFunction<E: ErrorExt> {
+    async fn call(&self) -> std::result::Result<(), E>;
+    fn name(&self) -> &str;
+}
+
+pub type TaskFunctionRef<E> = Arc<dyn TaskFunction<E> + Send + Sync>;
+
+pub struct RepeatedTask<E> {
+    cancel_token: Mutex<Option<CancellationToken>>,
+    task_handle: Mutex<Option<JoinHandle<()>>>,
+    started: AtomicBool,
+    interval: Duration,
+    task_fn: TaskFunctionRef<E>,
+}
+
+impl<E: ErrorExt> std::fmt::Display for RepeatedTask<E> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "RepeatedTask({})", self.task_fn.name())
+    }
+}
+
+impl<E: ErrorExt> std::fmt::Debug for RepeatedTask<E> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("RepeatedTask")
+            .field(&self.task_fn.name())
+            .finish()
+    }
+}
+
+impl<E: ErrorExt + 'static> RepeatedTask<E> {
+    pub fn new(interval: Duration, task_fn: TaskFunctionRef<E>) -> Self {
+        Self {
+            cancel_token: Mutex::new(None),
+            task_handle: Mutex::new(None),
+            started: AtomicBool::new(false),
+            interval,
+            task_fn,
+        }
+    }
+
+    pub fn started(&self) -> bool {
+        self.started.load(Ordering::Relaxed)
+    }
+
+    pub async fn start(&self, runtime: Runtime) -> Result<()> {
+        let token = CancellationToken::new();
+        let interval = self.interval;
+        let child = token.child_token();
+        let task_fn = self.task_fn.clone();
+        // TODO(hl): Maybe spawn to a blocking runtime.
+        let handle = runtime.spawn(async move {
+            loop {
+                tokio::select! {
+                    _ = tokio::time::sleep(interval) => {}
+                    _ = child.cancelled() => {
+                        return;
+                    }
+                }
+                if let Err(e) = task_fn.call().await {
+                    logging::error!(e; "Failed to run repeated task: {}", task_fn.name());
+                }
+            }
+        });
+        *self.cancel_token.lock().await = Some(token);
+        *self.task_handle.lock().await = Some(handle);
+        self.started.store(true, Ordering::Relaxed);
+
+        logging::debug!(
+            "Repeated task {} started with interval: {:?}",
+            self.task_fn.name(),
+            self.interval
+        );
+
+        Ok(())
+    }
+
+    pub async fn stop(&self) -> Result<()> {
+        let name = self.task_fn.name();
+        ensure!(
+            self.started
+                .compare_exchange(true, false, Ordering::Relaxed, Ordering::Relaxed)
+                .is_ok(),
+            IllegalStateSnafu { name }
+        );
+        let token = self
+            .cancel_token
+            .lock()
+            .await
+            .take()
+            .context(IllegalStateSnafu { name })?;
+        let handle = self
+            .task_handle
+            .lock()
+            .await
+            .take()
+            .context(IllegalStateSnafu { name })?;
+
+        token.cancel();
+        handle.await.context(WaitGcTaskStopSnafu { name })?;
+
+        logging::debug!("Repeated task {} stopped", name);
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::atomic::AtomicI32;
+
+    use super::*;
+
+    struct TickTask {
+        n: AtomicI32,
+    }
+
+    #[async_trait::async_trait]
+    impl TaskFunction<crate::error::Error> for TickTask {
+        fn name(&self) -> &str {
+            "test"
+        }
+
+        async fn call(&self) -> Result<()> {
+            self.n.fetch_add(1, Ordering::Relaxed);
+
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_repeated_task() {
+        common_telemetry::init_default_ut_logging();
+
+        let task_fn = Arc::new(TickTask {
+            n: AtomicI32::new(0),
+        });
+
+        let task = RepeatedTask::new(Duration::from_millis(100), task_fn.clone());
+
+        task.start(crate::bg_runtime()).await.unwrap();
+        tokio::time::sleep(Duration::from_millis(550)).await;
+        task.stop().await.unwrap();
+
+        assert_eq!(task_fn.n.load(Ordering::Relaxed), 5);
+    }
+}

src/common/runtime/src/runtime.rs

@@ -24,7 +24,7 @@ use tokio::sync::oneshot;
 pub use tokio::task::{JoinError, JoinHandle};
 
 use crate::error::*;
-use crate::metric::*;
+use crate::metrics::*;
 
 /// A runtime to run future tasks
 #[derive(Clone, Debug)]

src/common/substrait/src/df_expr.rs

@@ -581,7 +581,7 @@ pub fn expression_from_df_expr(
         | Expr::ScalarSubquery(..)
         | Expr::Placeholder { .. }
         | Expr::QualifiedWildcard { .. } => todo!(),
-        Expr::GroupingSet(_) => UnsupportedExprSnafu {
+        Expr::GroupingSet(_) | Expr::OuterReferenceColumn(_, _) => UnsupportedExprSnafu {
             name: expr.to_string(),
         }
         .fail()?,

src/common/substrait/src/df_logical.rs

@@ -22,9 +22,10 @@ use catalog::CatalogManagerRef;
 use common_catalog::format_full_table_name;
 use common_telemetry::debug;
 use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
-use datafusion::common::{DFField, DFSchema, OwnedTableReference};
+use datafusion::common::{DFField, DFSchema};
 use datafusion::datasource::DefaultTableSource;
 use datafusion::physical_plan::project_schema;
+use datafusion::sql::TableReference;
 use datafusion_expr::{Filter, LogicalPlan, TableScan};
 use prost::Message;
 use session::context::QueryContext;
@@ -240,13 +241,13 @@ impl DFLogicalSubstraitConvertor {
             .projection
             .map(|mask_expr| self.convert_mask_expression(mask_expr));
 
-        let table_ref = OwnedTableReference::Full {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-            table: table_name.clone(),
-        };
+        let table_ref = TableReference::full(
+            catalog_name.clone(),
+            schema_name.clone(),
+            table_name.clone(),
+        );
         let adapter = table_provider
-            .resolve_table(table_ref)
+            .resolve_table(table_ref.clone())
             .await
             .with_context(|_| ResolveTableSnafu {
                 table_name: format_full_table_name(&catalog_name, &schema_name, &table_name),
@@ -272,14 +273,13 @@ impl DFLogicalSubstraitConvertor {
         };
 
         // Calculate the projected schema
-        let qualified = &format_full_table_name(&catalog_name, &schema_name, &table_name);
         let projected_schema = Arc::new(
             project_schema(&stored_schema, projection.as_ref())
                 .and_then(|x| {
                     DFSchema::new_with_metadata(
                         x.fields()
                             .iter()
-                            .map(|f| DFField::from_qualified(qualified, f.clone()))
+                            .map(|f| DFField::from_qualified(table_ref.clone(), f.clone()))
                             .collect(),
                         x.metadata().clone(),
                     )
@@ -291,7 +291,7 @@ impl DFLogicalSubstraitConvertor {
 
         // TODO(ruihang): Support limit(fetch)
         Ok(LogicalPlan::TableScan(TableScan {
-            table_name: qualified.to_string(),
+            table_name: table_ref,
             source: adapter,
             projection,
             projected_schema,
@@ -620,10 +620,13 @@ mod test {
         let projected_schema =
             Arc::new(DFSchema::new_with_metadata(projected_fields, Default::default()).unwrap());
 
+        let table_name = TableReference::full(
+            DEFAULT_CATALOG_NAME,
+            DEFAULT_SCHEMA_NAME,
+            DEFAULT_TABLE_NAME,
+        );
         let table_scan_plan = LogicalPlan::TableScan(TableScan {
-            table_name: format!(
-                "{DEFAULT_CATALOG_NAME}.{DEFAULT_SCHEMA_NAME}.{DEFAULT_TABLE_NAME}",
-            ),
+            table_name,
             source: adapter,
             projection: Some(projection),
             projected_schema,

src/common/telemetry/Cargo.toml

@@ -12,7 +12,7 @@ deadlock_detection = ["parking_lot"]
 backtrace = "0.3"
 common-error = { path = "../error" }
 console-subscriber = { version = "0.1", optional = true }
-metrics = "0.20"
+metrics = "0.20.1"
 metrics-exporter-prometheus = { version = "0.11", default-features = false }
 once_cell = "1.10"
 opentelemetry = { version = "0.17", default-features = false, features = [

src/datanode/Cargo.toml

@@ -4,10 +4,6 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
-[features]
-default = ["python"]
-python = ["dep:script"]
-
 [dependencies]
 async-compat = "0.2"
 async-stream.workspace = true
@@ -21,6 +17,7 @@ common-base = { path = "../common/base" }
 common-catalog = { path = "../common/catalog" }
 common-error = { path = "../common/error" }
 common-datasource = { path = "../common/datasource" }
+common-function = { path = "../common/function" }
 common-grpc = { path = "../common/grpc" }
 common-grpc-expr = { path = "../common/grpc-expr" }
 common-procedure = { path = "../common/procedure" }
@@ -37,6 +34,7 @@ futures = "0.3"
 futures-util.workspace = true
 hyper = { version = "0.14", features = ["full"] }
 humantime-serde = "1.1"
+log = "0.4"
 log-store = { path = "../log-store" }
 meta-client = { path = "../meta-client" }
 meta-srv = { path = "../meta-srv", features = ["mock"] }
@@ -47,7 +45,6 @@ pin-project = "1.0"
 prost.workspace = true
 query = { path = "../query" }
 regex = "1.6"
-script = { path = "../script", features = ["python"], optional = true }
 serde = "1.0"
 serde_json = "1.0"
 servers = { path = "../servers" }
@@ -65,7 +62,6 @@ tonic.workspace = true
 tower = { version = "0.4", features = ["full"] }
 tower-http = { version = "0.3", features = ["full"] }
 url = "2.3.1"
-uuid.workspace = true
 
 [dev-dependencies]
 axum-test-helper = { git = "https://github.com/sunng87/axum-test-helper.git", branch = "patch-1" }

src/datanode/src/datanode.rs

@@ -19,6 +19,7 @@ use common_base::readable_size::ReadableSize;
 use common_telemetry::info;
 use meta_client::MetaClientOptions;
 use serde::{Deserialize, Serialize};
+use servers::http::HttpOptions;
 use servers::Mode;
 use storage::config::EngineConfig as StorageEngineConfig;
 use storage::scheduler::SchedulerConfig;
@@ -29,6 +30,7 @@ use crate::server::Services;
 
 pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize(1024);
 
+/// Object storage config
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(tag = "type")]
 pub enum ObjectStoreConfig {
@@ -37,6 +39,16 @@ pub enum ObjectStoreConfig {
     Oss(OssConfig),
 }
 
+/// Storage engine config
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(default)]
+pub struct StorageConfig {
+    #[serde(flatten)]
+    pub store: ObjectStoreConfig,
+    pub compaction: CompactionConfig,
+    pub manifest: RegionManifestConfig,
+}
+
 #[derive(Debug, Clone, Serialize, Default, Deserialize)]
 #[serde(default)]
 pub struct FileConfig {
@@ -107,6 +119,27 @@ impl Default for WalConfig {
     }
 }
 
+/// Options for region manifest
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+#[serde(default)]
+pub struct RegionManifestConfig {
+    /// Region manifest checkpoint actions margin.
+    /// Manifest service create a checkpoint every [checkpoint_margin] actions.
+    pub checkpoint_margin: Option<u16>,
+    /// Region manifest logs and checkpoints gc task execution duration.
+    #[serde(with = "humantime_serde")]
+    pub gc_duration: Option<Duration>,
+}
+
+impl Default for RegionManifestConfig {
+    fn default() -> Self {
+        Self {
+            checkpoint_margin: Some(10u16),
+            gc_duration: Some(Duration::from_secs(30)),
+        }
+    }
+}
+
 /// Options for table compaction
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
 #[serde(default)]
@@ -117,6 +150,8 @@ pub struct CompactionConfig {
     pub max_files_in_level0: usize,
     /// Max task number for SST purge task after compaction.
     pub max_purge_tasks: usize,
+    /// Buffer threshold while writing SST files
+    pub sst_write_buffer_size: ReadableSize,
 }
 
 impl Default for CompactionConfig {
@@ -125,6 +160,7 @@ impl Default for CompactionConfig {
             max_inflight_tasks: 4,
             max_files_in_level0: 8,
             max_purge_tasks: 32,
+            sst_write_buffer_size: ReadableSize::mb(8),
         }
     }
 }
@@ -132,7 +168,7 @@ impl Default for CompactionConfig {
 impl From<&DatanodeOptions> for SchedulerConfig {
     fn from(value: &DatanodeOptions) -> Self {
         Self {
-            max_inflight_tasks: value.compaction.max_inflight_tasks,
+            max_inflight_tasks: value.storage.compaction.max_inflight_tasks,
         }
     }
 }
@@ -140,8 +176,11 @@ impl From<&DatanodeOptions> for SchedulerConfig {
 impl From<&DatanodeOptions> for StorageEngineConfig {
     fn from(value: &DatanodeOptions) -> Self {
         Self {
-            max_files_in_l0: value.compaction.max_files_in_level0,
-            max_purge_tasks: value.compaction.max_purge_tasks,
+            manifest_checkpoint_margin: value.storage.manifest.checkpoint_margin,
+            manifest_gc_duration: value.storage.manifest.gc_duration,
+            max_files_in_l0: value.storage.compaction.max_files_in_level0,
+            max_purge_tasks: value.storage.compaction.max_purge_tasks,
+            sst_write_buffer_size: value.storage.compaction.sst_write_buffer_size,
         }
     }
 }
@@ -190,10 +229,10 @@ pub struct DatanodeOptions {
     pub rpc_runtime_size: usize,
     pub mysql_addr: String,
     pub mysql_runtime_size: usize,
+    pub http_opts: HttpOptions,
     pub meta_client_options: Option<MetaClientOptions>,
     pub wal: WalConfig,
-    pub storage: ObjectStoreConfig,
-    pub compaction: CompactionConfig,
+    pub storage: StorageConfig,
     pub procedure: Option<ProcedureConfig>,
 }
 
@@ -208,10 +247,10 @@ impl Default for DatanodeOptions {
             rpc_runtime_size: 8,
             mysql_addr: "127.0.0.1:4406".to_string(),
             mysql_runtime_size: 2,
+            http_opts: HttpOptions::default(),
             meta_client_options: None,
             wal: WalConfig::default(),
-            storage: ObjectStoreConfig::default(),
-            compaction: CompactionConfig::default(),
+            storage: StorageConfig::default(),
             procedure: None,
         }
     }
@@ -220,14 +259,17 @@ impl Default for DatanodeOptions {
 /// Datanode service.
 pub struct Datanode {
     opts: DatanodeOptions,
-    services: Services,
+    services: Option<Services>,
     instance: InstanceRef,
 }
 
 impl Datanode {
     pub async fn new(opts: DatanodeOptions) -> Result<Datanode> {
         let instance = Arc::new(Instance::new(&opts).await?);
-        let services = Services::try_new(instance.clone(), &opts).await?;
+        let services = match opts.mode {
+            Mode::Distributed => Some(Services::try_new(instance.clone(), &opts).await?),
+            Mode::Standalone => None,
+        };
         Ok(Self {
             opts,
             services,
@@ -248,7 +290,11 @@ impl Datanode {
 
     /// Start services of datanode. This method call will block until services are shutdown.
     pub async fn start_services(&mut self) -> Result<()> {
-        self.services.start(&self.opts).await
+        if let Some(service) = self.services.as_mut() {
+            service.start(&self.opts).await
+        } else {
+            Ok(())
+        }
     }
 
     pub fn get_instance(&self) -> InstanceRef {
@@ -260,7 +306,11 @@ impl Datanode {
     }
 
     async fn shutdown_services(&self) -> Result<()> {
-        self.services.shutdown().await
+        if let Some(service) = self.services.as_ref() {
+            service.shutdown().await
+        } else {
+            Ok(())
+        }
     }
 
     pub async fn shutdown(&self) -> Result<()> {

src/datanode/src/error.rs

@@ -17,9 +17,7 @@ use std::any::Any;
 use common_datasource::error::Error as DataSourceError;
 use common_error::prelude::*;
 use common_procedure::ProcedureId;
-use common_recordbatch::error::Error as RecordBatchError;
 use datafusion::parquet;
-use datatypes::prelude::ConcreteDataType;
 use storage::error::Error as StorageError;
 use table::error::Error as TableError;
 use url::ParseError;
@@ -125,24 +123,6 @@ pub enum Error {
     ))]
     ColumnValuesNumberMismatch { columns: usize, values: usize },
 
-    #[snafu(display(
-        "Column type mismatch, column: {}, expected type: {:?}, actual: {:?}",
-        column,
-        expected,
-        actual,
-    ))]
-    ColumnTypeMismatch {
-        column: String,
-        expected: ConcreteDataType,
-        actual: ConcreteDataType,
-    },
-
-    #[snafu(display("Failed to collect record batch, source: {}", source))]
-    CollectRecords {
-        #[snafu(backtrace)]
-        source: RecordBatchError,
-    },
-
     #[snafu(display("Failed to parse sql value, source: {}", source))]
     ParseSqlValue {
         #[snafu(backtrace)]
@@ -201,6 +181,9 @@ pub enum Error {
     #[snafu(display("Failed to create directory {}, source: {}", dir, source))]
     CreateDir { dir: String, source: std::io::Error },
 
+    #[snafu(display("Failed to remove directory {}, source: {}", dir, source))]
+    RemoveDir { dir: String, source: std::io::Error },
+
     #[snafu(display("Failed to open log store, source: {}", source))]
     OpenLogStore {
         #[snafu(backtrace)]
@@ -331,12 +314,6 @@ pub enum Error {
         source: sql::error::Error,
     },
 
-    #[snafu(display("Failed to start script manager, source: {}", source))]
-    StartScriptManager {
-        #[snafu(backtrace)]
-        source: script::error::Error,
-    },
-
     #[snafu(display(
         "Failed to parse string to timestamp, string: {}, source: {}",
         raw,
@@ -456,12 +433,6 @@ pub enum Error {
         backtrace: Backtrace,
     },
 
-    #[snafu(display("Failed to write parquet file, source: {}", source))]
-    WriteParquet {
-        source: parquet::errors::ParquetError,
-        backtrace: Backtrace,
-    },
-
     #[snafu(display("Failed to poll stream, source: {}", source))]
     PollStream {
         source: datafusion_common::DataFusionError,
@@ -531,6 +502,12 @@ pub enum Error {
         #[snafu(backtrace)]
         source: BoxedError,
     },
+
+    #[snafu(display("Failed to copy table to parquet file, source: {}", source))]
+    WriteParquet {
+        #[snafu(backtrace)]
+        source: storage::error::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -556,8 +533,6 @@ impl ErrorExt for Error {
 
             Insert { source, .. } => source.status_code(),
             Delete { source, .. } => source.status_code(),
-            CollectRecords { source, .. } => source.status_code(),
-
             TableNotFound { .. } => StatusCode::TableNotFound,
             ColumnNotFound { .. } => StatusCode::TableColumnNotFound,
 
@@ -570,7 +545,6 @@ impl ErrorExt for Error {
             ConvertSchema { source, .. } | VectorComputation { source } => source.status_code(),
 
             ColumnValuesNumberMismatch { .. }
-            | ColumnTypeMismatch { .. }
             | InvalidSql { .. }
             | InvalidUrl { .. }
             | InvalidPath { .. }
@@ -599,6 +573,7 @@ impl ErrorExt for Error {
             | TcpBind { .. }
             | StartGrpc { .. }
             | CreateDir { .. }
+            | RemoveDir { .. }
             | InsertSystemCatalog { .. }
             | RenameTable { .. }
             | Catalog { .. }
@@ -620,7 +595,6 @@ impl ErrorExt for Error {
             | WriteObject { .. }
             | ListObjects { .. } => StatusCode::StorageUnavailable,
             OpenLogStore { source } => source.status_code(),
-            StartScriptManager { source } => source.status_code(),
             OpenStorageEngine { source } => source.status_code(),
             RuntimeResource { .. } => StatusCode::RuntimeResourcesExhausted,
             MetaClientInit { source, .. } => source.status_code(),

src/datanode/src/heartbeat.rs

@@ -106,13 +106,7 @@ impl HeartbeatTask {
         let mut tx = Self::create_streams(&meta_client, running.clone()).await?;
         common_runtime::spawn_bg(async move {
             while running.load(Ordering::Acquire) {
-                let (region_num, region_stats) = match datanode_stat(&catalog_manager_clone).await {
-                    Ok(datanode_stat) => (datanode_stat.0 as i64, datanode_stat.1),
-                    Err(e) => {
-                        error!("failed to get region status, err: {e:?}");
-                        (-1, vec![])
-                    }
-                };
+                let (region_num, region_stats) = datanode_stat(&catalog_manager_clone).await;
 
                 let req = HeartbeatRequest {
                     peer: Some(Peer {
@@ -120,7 +114,7 @@ impl HeartbeatTask {
                         addr: addr.clone(),
                     }),
                     node_stat: Some(NodeStat {
-                        region_num,
+                        region_num: region_num as _,
                         ..Default::default()
                     }),
                     region_stats,

src/datanode/src/instance.rs

@@ -23,6 +23,7 @@ use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER
 use common_error::prelude::BoxedError;
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
 use common_procedure::local::{LocalManager, ManagerConfig};
+use common_procedure::store::state_store::ObjectStateStore;
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::logging::info;
 use log_store::raft_engine::log_store::RaftEngineLogStore;
@@ -37,12 +38,14 @@ use object_store::services::{Fs as FsBuilder, Oss as OSSBuilder, S3 as S3Builder
 use object_store::{util, ObjectStore, ObjectStoreBuilder};
 use query::query_engine::{QueryEngineFactory, QueryEngineRef};
 use servers::Mode;
+use session::context::QueryContext;
 use snafu::prelude::*;
 use storage::compaction::{CompactionHandler, CompactionSchedulerRef, SimplePicker};
 use storage::config::EngineConfig as StorageEngineConfig;
 use storage::scheduler::{LocalScheduler, SchedulerConfig};
 use storage::EngineImpl;
 use store_api::logstore::LogStore;
+use table::requests::FlushTableRequest;
 use table::table::numbers::NumbersTable;
 use table::table::TableIdProviderRef;
 use table::Table;
@@ -55,11 +58,9 @@ use crate::error::{
     NewCatalogSnafu, OpenLogStoreSnafu, RecoverProcedureSnafu, Result, ShutdownInstanceSnafu,
 };
 use crate::heartbeat::HeartbeatTask;
-use crate::script::ScriptExecutor;
-use crate::sql::SqlHandler;
+use crate::sql::{SqlHandler, SqlRequest};
 
 mod grpc;
-mod script;
 pub mod sql;
 
 pub(crate) type DefaultEngine = MitoEngine<EngineImpl<RaftEngineLogStore>>;
@@ -69,9 +70,9 @@ pub struct Instance {
     pub(crate) query_engine: QueryEngineRef,
     pub(crate) sql_handler: SqlHandler,
     pub(crate) catalog_manager: CatalogManagerRef,
-    pub(crate) script_executor: ScriptExecutor,
     pub(crate) table_id_provider: Option<TableIdProviderRef>,
     pub(crate) heartbeat_task: Option<HeartbeatTask>,
+    procedure_manager: Option<ProcedureManagerRef>,
 }
 
 pub type InstanceRef = Arc<Instance>;
@@ -102,7 +103,7 @@ impl Instance {
         meta_client: Option<Arc<MetaClient>>,
         compaction_scheduler: CompactionSchedulerRef<RaftEngineLogStore>,
     ) -> Result<Self> {
-        let object_store = new_object_store(&opts.storage).await?;
+        let object_store = new_object_store(&opts.storage.store).await?;
         let log_store = Arc::new(create_log_store(&opts.wal).await?);
 
         let table_engine = Arc::new(DefaultEngine::new(
@@ -166,8 +167,6 @@ impl Instance {
 
         let factory = QueryEngineFactory::new(catalog_manager.clone());
         let query_engine = factory.query_engine();
-        let script_executor =
-            ScriptExecutor::new(catalog_manager.clone(), query_engine.clone()).await?;
 
         let heartbeat_task = match opts.mode {
             Mode::Standalone => None,
@@ -181,6 +180,7 @@ impl Instance {
         };
 
         let procedure_manager = create_procedure_manager(&opts.procedure).await?;
+        // Register all procedures.
         if let Some(procedure_manager) = &procedure_manager {
             table_engine.register_procedure_loaders(&**procedure_manager);
             table_procedure::register_procedure_loaders(
@@ -189,12 +189,6 @@ impl Instance {
                 table_engine.clone(),
                 &**procedure_manager,
             );
-
-            // Recover procedures.
-            procedure_manager
-                .recover()
-                .await
-                .context(RecoverProcedureSnafu)?;
         }
 
         Ok(Self {
@@ -202,14 +196,13 @@ impl Instance {
             sql_handler: SqlHandler::new(
                 table_engine.clone(),
                 catalog_manager.clone(),
-                query_engine.clone(),
                 table_engine,
-                procedure_manager,
+                procedure_manager.clone(),
             ),
             catalog_manager,
-            script_executor,
             heartbeat_task,
             table_id_provider,
+            procedure_manager,
         })
     }
 
@@ -221,6 +214,15 @@ impl Instance {
         if let Some(task) = &self.heartbeat_task {
             task.start().await?;
         }
+
+        // Recover procedures after the catalog manager is started, so we can
+        // ensure we can access all tables from the catalog manager.
+        if let Some(procedure_manager) = &self.procedure_manager {
+            procedure_manager
+                .recover()
+                .await
+                .context(RecoverProcedureSnafu)?;
+        }
         Ok(())
     }
 
@@ -233,6 +235,8 @@ impl Instance {
                 .context(ShutdownInstanceSnafu)?;
         }
 
+        self.flush_tables().await?;
+
         self.sql_handler
             .close()
             .await
@@ -240,6 +244,43 @@ impl Instance {
             .context(ShutdownInstanceSnafu)
     }
 
+    pub async fn flush_tables(&self) -> Result<()> {
+        info!("going to flush all schemas");
+        let schema_list = self
+            .catalog_manager
+            .catalog(DEFAULT_CATALOG_NAME)
+            .map_err(BoxedError::new)
+            .context(ShutdownInstanceSnafu)?
+            .expect("Default schema not found")
+            .schema_names()
+            .map_err(BoxedError::new)
+            .context(ShutdownInstanceSnafu)?;
+        let flush_requests = schema_list
+            .into_iter()
+            .map(|schema_name| {
+                SqlRequest::FlushTable(FlushTableRequest {
+                    catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+                    schema_name,
+                    table_name: None,
+                    region_number: None,
+                    wait: Some(true),
+                })
+            })
+            .collect::<Vec<_>>();
+        let flush_result = futures::future::try_join_all(
+            flush_requests
+                .into_iter()
+                .map(|request| self.sql_handler.execute(request, QueryContext::arc())),
+        )
+        .await
+        .map_err(BoxedError::new)
+        .context(ShutdownInstanceSnafu);
+        info!("Flushed all tables result: {}", flush_result.is_ok());
+        flush_result?;
+
+        Ok(())
+    }
+
     pub fn sql_handler(&self) -> &SqlHandler {
         &self.sql_handler
     }
@@ -268,11 +309,22 @@ pub(crate) async fn new_object_store(store_config: &ObjectStoreConfig) -> Result
         ObjectStoreConfig::Oss { .. } => new_oss_object_store(store_config).await,
     };
 
+    // Don't enable retry layer when using local file backend.
+    let object_store = if !matches!(store_config, ObjectStoreConfig::File(..)) {
+        object_store.map(|object_store| object_store.layer(RetryLayer::new().with_jitter()))
+    } else {
+        object_store
+    };
+
     object_store.map(|object_store| {
         object_store
-            .layer(RetryLayer::new().with_jitter())
             .layer(MetricsLayer)
-            .layer(LoggingLayer::default())
+            .layer(
+                LoggingLayer::default()
+                    // Print the expected error only in DEBUG level.
+                    // See https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html#method.with_error_level
+                    .with_error_level(Some(log::Level::Debug)),
+            )
             .layer(TracingLayer)
     })
 }
@@ -290,18 +342,20 @@ pub(crate) async fn new_oss_object_store(store_config: &ObjectStoreConfig) -> Re
     );
 
     let mut builder = OSSBuilder::default();
-    let builder = builder
+    builder
         .root(&root)
         .bucket(&oss_config.bucket)
         .endpoint(&oss_config.endpoint)
         .access_key_id(&oss_config.access_key_id)
         .access_key_secret(&oss_config.access_key_secret);
 
-    let accessor = builder.build().with_context(|_| error::InitBackendSnafu {
-        config: store_config.clone(),
-    })?;
+    let object_store = ObjectStore::new(builder)
+        .with_context(|_| error::InitBackendSnafu {
+            config: store_config.clone(),
+        })?
+        .finish();
 
-    create_object_store_with_cache(ObjectStore::new(accessor).finish(), store_config)
+    create_object_store_with_cache(object_store, store_config)
 }
 
 fn create_object_store_with_cache(
@@ -354,24 +408,27 @@ pub(crate) async fn new_s3_object_store(store_config: &ObjectStoreConfig) -> Res
     );
 
     let mut builder = S3Builder::default();
-    let mut builder = builder
+    builder
         .root(&root)
         .bucket(&s3_config.bucket)
         .access_key_id(&s3_config.access_key_id)
         .secret_access_key(&s3_config.secret_access_key);
 
     if s3_config.endpoint.is_some() {
-        builder = builder.endpoint(s3_config.endpoint.as_ref().unwrap());
+        builder.endpoint(s3_config.endpoint.as_ref().unwrap());
     }
     if s3_config.region.is_some() {
-        builder = builder.region(s3_config.region.as_ref().unwrap());
+        builder.region(s3_config.region.as_ref().unwrap());
     }
 
-    let accessor = builder.build().with_context(|_| error::InitBackendSnafu {
-        config: store_config.clone(),
-    })?;
-
-    create_object_store_with_cache(ObjectStore::new(accessor).finish(), store_config)
+    create_object_store_with_cache(
+        ObjectStore::new(builder)
+            .with_context(|_| error::InitBackendSnafu {
+                config: store_config.clone(),
+            })?
+            .finish(),
+        store_config,
+    )
 }
 
 pub(crate) async fn new_fs_object_store(store_config: &ObjectStoreConfig) -> Result<ObjectStore> {
@@ -385,16 +442,27 @@ pub(crate) async fn new_fs_object_store(store_config: &ObjectStoreConfig) -> Res
     info!("The file storage directory is: {}", &data_dir);
 
     let atomic_write_dir = format!("{data_dir}/.tmp/");
+    if path::Path::new(&atomic_write_dir).exists() {
+        info!(
+            "Begin to clean temp storage directory: {}",
+            &atomic_write_dir
+        );
+        fs::remove_dir_all(&atomic_write_dir).context(error::RemoveDirSnafu {
+            dir: &atomic_write_dir,
+        })?;
+        info!("Cleaned temp storage directory: {}", &atomic_write_dir);
+    }
 
-    let accessor = FsBuilder::default()
-        .root(&data_dir)
-        .atomic_write_dir(&atomic_write_dir)
-        .build()
+    let mut builder = FsBuilder::default();
+    builder.root(&data_dir).atomic_write_dir(&atomic_write_dir);
+
+    let object_store = ObjectStore::new(builder)
         .context(error::InitBackendSnafu {
             config: store_config.clone(),
-        })?;
+        })?
+        .finish();
 
-    Ok(ObjectStore::new(accessor).finish())
+    Ok(object_store)
 }
 
 /// Create metasrv client instance and spawn heartbeat loop.
@@ -460,11 +528,15 @@ pub(crate) async fn create_procedure_manager(
     );
 
     let object_store = new_object_store(&procedure_config.store).await?;
+    let state_store = Arc::new(ObjectStateStore::new(object_store));
+
     let manager_config = ManagerConfig {
-        object_store,
         max_retry_times: procedure_config.max_retry_times,
         retry_delay: procedure_config.retry_delay,
     };
 
-    Ok(Some(Arc::new(LocalManager::new(manager_config))))
+    Ok(Some(Arc::new(LocalManager::new(
+        manager_config,
+        state_store,
+    ))))
 }

src/datanode/src/instance/grpc.rs

@@ -21,7 +21,7 @@ use common_query::Output;
 use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
 use query::plan::LogicalPlan;
 use servers::query_handler::grpc::GrpcQueryHandler;
-use session::context::QueryContextRef;
+use session::context::{QueryContext, QueryContextRef};
 use snafu::prelude::*;
 use sql::statements::statement::Statement;
 use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
@@ -53,7 +53,7 @@ impl Instance {
             .context(DecodeLogicalPlanSnafu)?;
 
         self.query_engine
-            .execute(&LogicalPlan::DfPlan(logical_plan))
+            .execute(LogicalPlan::DfPlan(logical_plan), QueryContext::arc())
             .await
             .context(ExecuteLogicalPlanSnafu)
     }
@@ -69,11 +69,11 @@ impl Instance {
                         let plan = self
                             .query_engine
                             .planner()
-                            .plan(stmt, ctx)
+                            .plan(stmt, ctx.clone())
                             .await
                             .context(PlanStatementSnafu)?;
                         self.query_engine
-                            .execute(&plan)
+                            .execute(plan, ctx)
                             .await
                             .context(ExecuteLogicalPlanSnafu)
                     }
@@ -175,7 +175,7 @@ mod test {
             .plan(stmt, QueryContext::arc())
             .await
             .unwrap();
-        engine.execute(&plan).await.unwrap()
+        engine.execute(plan, QueryContext::arc()).await.unwrap()
     }
 
     #[tokio::test(flavor = "multi_thread")]

src/datanode/src/instance/sql.rs

@@ -19,7 +19,6 @@ use common_error::prelude::BoxedError;
 use common_query::Output;
 use common_telemetry::logging::info;
 use common_telemetry::timer;
-use futures::StreamExt;
 use query::error::QueryExecutionSnafu;
 use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
 use query::query_engine::StatementHandler;
@@ -38,9 +37,8 @@ use crate::error::{
     TableIdProviderNotFoundSnafu,
 };
 use crate::instance::Instance;
-use crate::metric;
-use crate::sql::insert::InsertRequests;
-use crate::sql::SqlRequest;
+use crate::metrics;
+use crate::sql::{SqlHandler, SqlRequest};
 
 impl Instance {
     pub async fn execute_stmt(
@@ -50,37 +48,10 @@ impl Instance {
     ) -> Result<Output> {
         match stmt {
             QueryStatement::Sql(Statement::Insert(insert)) => {
-                let requests = self
-                    .sql_handler
-                    .insert_to_requests(self.catalog_manager.clone(), *insert, query_ctx.clone())
-                    .await?;
-
-                match requests {
-                    InsertRequests::Request(request) => {
-                        self.sql_handler.execute(request, query_ctx.clone()).await
-                    }
-
-                    InsertRequests::Stream(mut s) => {
-                        let mut rows = 0;
-                        while let Some(request) = s.next().await {
-                            match self
-                                .sql_handler
-                                .execute(request?, query_ctx.clone())
-                                .await?
-                            {
-                                Output::AffectedRows(n) => {
-                                    rows += n;
-                                }
-                                _ => unreachable!(),
-                            }
-                        }
-                        Ok(Output::AffectedRows(rows))
-                    }
-                }
-            }
-            QueryStatement::Sql(Statement::Delete(delete)) => {
-                let request = SqlRequest::Delete(*delete);
-                self.sql_handler.execute(request, query_ctx).await
+                let request =
+                    SqlHandler::insert_to_request(self.catalog_manager.clone(), *insert, query_ctx)
+                        .await?;
+                self.sql_handler.insert(request).await
             }
             QueryStatement::Sql(Statement::CreateDatabase(create_database)) => {
                 let request = CreateDatabaseRequest {
@@ -119,6 +90,9 @@ impl Instance {
                     .execute(SqlRequest::CreateTable(request), query_ctx)
                     .await
             }
+            QueryStatement::Sql(Statement::CreateExternalTable(_create_external_table)) => {
+                unimplemented!()
+            }
             QueryStatement::Sql(Statement::Alter(alter_table)) => {
                 let name = alter_table.table_name().clone();
                 let (catalog, schema, table) = table_idents_to_full_name(&name, query_ctx.clone())?;
@@ -150,11 +124,6 @@ impl Instance {
                     .execute(SqlRequest::ShowTables(show_tables), query_ctx)
                     .await
             }
-            QueryStatement::Sql(Statement::DescribeTable(describe_table)) => {
-                self.sql_handler
-                    .execute(SqlRequest::DescribeTable(describe_table), query_ctx)
-                    .await
-            }
             QueryStatement::Sql(Statement::ShowCreateTable(_show_create_table)) => {
                 unimplemented!("SHOW CREATE TABLE is unimplemented yet");
             }
@@ -210,6 +179,8 @@ impl Instance {
             | QueryStatement::Sql(Statement::Explain(_))
             | QueryStatement::Sql(Statement::Use(_))
             | QueryStatement::Sql(Statement::Tql(_))
+            | QueryStatement::Sql(Statement::Delete(_))
+            | QueryStatement::Sql(Statement::DescribeTable(_))
             | QueryStatement::Promql(_) => unreachable!(),
         }
     }
@@ -219,17 +190,20 @@ impl Instance {
         promql: &PromQuery,
         query_ctx: QueryContextRef,
     ) -> Result<Output> {
-        let _timer = timer!(metric::METRIC_HANDLE_PROMQL_ELAPSED);
+        let _timer = timer!(metrics::METRIC_HANDLE_PROMQL_ELAPSED);
 
         let stmt = QueryLanguageParser::parse_promql(promql).context(ExecuteSqlSnafu)?;
 
         let engine = self.query_engine();
         let plan = engine
             .planner()
-            .plan(stmt, query_ctx)
+            .plan(stmt, query_ctx.clone())
             .await
             .context(PlanStatementSnafu)?;
-        engine.execute(&plan).await.context(ExecuteStatementSnafu)
+        engine
+            .execute(plan, query_ctx)
+            .await
+            .context(ExecuteStatementSnafu)
     }
 
     // TODO(ruihang): merge this and `execute_promql` after #951 landed
@@ -262,10 +236,13 @@ impl Instance {
         let engine = self.query_engine();
         let plan = engine
             .planner()
-            .plan(stmt, query_ctx)
+            .plan(stmt, query_ctx.clone())
             .await
             .context(PlanStatementSnafu)?;
-        engine.execute(&plan).await.context(ExecuteStatementSnafu)
+        engine
+            .execute(plan, query_ctx)
+            .await
+            .context(ExecuteStatementSnafu)
     }
 }
 
@@ -317,7 +294,7 @@ impl StatementHandler for Instance {
 #[async_trait]
 impl PromHandler for Instance {
     async fn do_query(&self, query: &PromQuery) -> server_error::Result<Output> {
-        let _timer = timer!(metric::METRIC_HANDLE_PROMQL_ELAPSED);
+        let _timer = timer!(metrics::METRIC_HANDLE_PROMQL_ELAPSED);
 
         self.execute_promql(query, QueryContext::arc())
             .await

src/datanode/src/lib.rs

@@ -19,9 +19,8 @@ pub mod datanode;
 pub mod error;
 mod heartbeat;
 pub mod instance;
-pub mod metric;
+pub mod metrics;
 mod mock;
-mod script;
 pub mod server;
 pub mod sql;
 #[cfg(test)]

src/datanode/src/metrics.rs

@@ -15,6 +15,4 @@
 //! datanode metrics
 
 pub const METRIC_HANDLE_SQL_ELAPSED: &str = "datanode.handle_sql_elapsed";
-pub const METRIC_HANDLE_SCRIPTS_ELAPSED: &str = "datanode.handle_scripts_elapsed";
-pub const METRIC_RUN_SCRIPT_ELAPSED: &str = "datanode.run_script_elapsed";
 pub const METRIC_HANDLE_PROMQL_ELAPSED: &str = "datanode.handle_promql_elapsed";

src/datanode/src/server.rs

@@ -18,9 +18,12 @@ use std::sync::Arc;
 
 use common_runtime::Builder as RuntimeBuilder;
 use servers::grpc::GrpcServer;
+use servers::http::{HttpServer, HttpServerBuilder};
+use servers::metrics_handler::MetricsHandler;
 use servers::query_handler::grpc::ServerGrpcQueryHandlerAdaptor;
 use servers::server::Server;
 use snafu::ResultExt;
+use tokio::select;
 
 use crate::datanode::DatanodeOptions;
 use crate::error::{
@@ -33,6 +36,7 @@ pub mod grpc;
 /// All rpc services.
 pub struct Services {
     grpc_server: GrpcServer,
+    http_server: HttpServer,
 }
 
 impl Services {
@@ -51,6 +55,9 @@ impl Services {
                 None,
                 grpc_runtime,
             ),
+            http_server: HttpServerBuilder::new(opts.http_opts.clone())
+                .with_metrics_handler(MetricsHandler)
+                .build(),
         })
     }
 
@@ -58,10 +65,15 @@ impl Services {
         let grpc_addr: SocketAddr = opts.rpc_addr.parse().context(ParseAddrSnafu {
             addr: &opts.rpc_addr,
         })?;
-        self.grpc_server
-            .start(grpc_addr)
-            .await
-            .context(StartServerSnafu)?;
+        let http_addr = opts.http_opts.addr.parse().context(ParseAddrSnafu {
+            addr: &opts.http_opts.addr,
+        })?;
+        let grpc = self.grpc_server.start(grpc_addr);
+        let http = self.http_server.start(http_addr);
+        select!(
+            v = grpc => v.context(StartServerSnafu)?,
+            v = http => v.context(StartServerSnafu)?,
+        );
         Ok(())
     }
 
@@ -69,6 +81,11 @@ impl Services {
         self.grpc_server
             .shutdown()
             .await
-            .context(ShutdownServerSnafu)
+            .context(ShutdownServerSnafu)?;
+        self.http_server
+            .shutdown()
+            .await
+            .context(ShutdownServerSnafu)?;
+        Ok(())
     }
 }

src/datanode/src/sql.rs

@@ -17,19 +17,16 @@ use common_error::prelude::BoxedError;
 use common_procedure::ProcedureManagerRef;
 use common_query::Output;
 use common_telemetry::error;
-use query::query_engine::QueryEngineRef;
-use query::sql::{describe_table, show_databases, show_tables};
+use query::sql::{show_databases, show_tables};
 use session::context::QueryContextRef;
 use snafu::{OptionExt, ResultExt};
-use sql::statements::delete::Delete;
-use sql::statements::describe::DescribeTable;
 use sql::statements::show::{ShowDatabases, ShowTables};
 use table::engine::{EngineContext, TableEngineProcedureRef, TableEngineRef, TableReference};
 use table::requests::*;
 use table::TableRef;
 
 use crate::error::{
-    self, CloseTableEngineSnafu, ExecuteSqlSnafu, GetTableSnafu, Result, TableNotFoundSnafu,
+    CloseTableEngineSnafu, ExecuteSqlSnafu, GetTableSnafu, Result, TableNotFoundSnafu,
 };
 use crate::instance::sql::table_idents_to_full_name;
 
@@ -37,14 +34,12 @@ mod alter;
 mod copy_table_from;
 mod copy_table_to;
 mod create;
-mod delete;
 mod drop_table;
 mod flush_table;
 pub(crate) mod insert;
 
 #[derive(Debug)]
 pub enum SqlRequest {
-    Insert(InsertRequest),
     CreateTable(CreateTableRequest),
     CreateDatabase(CreateDatabaseRequest),
     Alter(AlterTableRequest),
@@ -52,16 +47,14 @@ pub enum SqlRequest {
     FlushTable(FlushTableRequest),
     ShowDatabases(ShowDatabases),
     ShowTables(ShowTables),
-    DescribeTable(DescribeTable),
-    Delete(Delete),
     CopyTable(CopyTableRequest),
 }
 
 // Handler to execute SQL except query
+#[derive(Clone)]
 pub struct SqlHandler {
     table_engine: TableEngineRef,
     catalog_manager: CatalogManagerRef,
-    query_engine: QueryEngineRef,
     engine_procedure: TableEngineProcedureRef,
     procedure_manager: Option<ProcedureManagerRef>,
 }
@@ -70,14 +63,12 @@ impl SqlHandler {
     pub fn new(
         table_engine: TableEngineRef,
         catalog_manager: CatalogManagerRef,
-        query_engine: QueryEngineRef,
         engine_procedure: TableEngineProcedureRef,
         procedure_manager: Option<ProcedureManagerRef>,
     ) -> Self {
         Self {
             table_engine,
             catalog_manager,
-            query_engine,
             engine_procedure,
             procedure_manager,
         }
@@ -89,12 +80,10 @@ impl SqlHandler {
     // there, instead of executing here in a "static" fashion.
     pub async fn execute(&self, request: SqlRequest, query_ctx: QueryContextRef) -> Result<Output> {
         let result = match request {
-            SqlRequest::Insert(req) => self.insert(req).await,
             SqlRequest::CreateTable(req) => self.create_table(req).await,
             SqlRequest::CreateDatabase(req) => self.create_database(req, query_ctx.clone()).await,
             SqlRequest::Alter(req) => self.alter(req).await,
             SqlRequest::DropTable(req) => self.drop_table(req).await,
-            SqlRequest::Delete(req) => self.delete(query_ctx.clone(), req).await,
             SqlRequest::CopyTable(req) => match req.direction {
                 CopyDirection::Export => self.copy_table_to(req).await,
                 CopyDirection::Import => self.copy_table_from(req).await,
@@ -106,19 +95,6 @@ impl SqlHandler {
                 show_tables(req, self.catalog_manager.clone(), query_ctx.clone())
                     .context(ExecuteSqlSnafu)
             }
-            SqlRequest::DescribeTable(req) => {
-                let (catalog, schema, table) =
-                    table_idents_to_full_name(req.name(), query_ctx.clone())?;
-                let table = self
-                    .catalog_manager
-                    .table(&catalog, &schema, &table)
-                    .await
-                    .context(error::CatalogSnafu)?
-                    .with_context(|| TableNotFoundSnafu {
-                        table_name: req.name().to_string(),
-                    })?;
-                describe_table(table).context(ExecuteSqlSnafu)
-            }
             SqlRequest::FlushTable(req) => self.flush_table(req).await,
         };
         if let Err(e) = &result {
@@ -150,239 +126,3 @@ impl SqlHandler {
             .context(CloseTableEngineSnafu)
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use std::any::Any;
-    use std::sync::Arc;
-
-    use catalog::{CatalogManager, RegisterTableRequest};
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-    use common_query::logical_plan::Expr;
-    use common_query::physical_plan::PhysicalPlanRef;
-    use common_test_util::temp_dir::create_temp_dir;
-    use common_time::timestamp::Timestamp;
-    use datatypes::prelude::ConcreteDataType;
-    use datatypes::schema::{ColumnSchema, SchemaBuilder, SchemaRef};
-    use datatypes::value::Value;
-    use futures::StreamExt;
-    use log_store::NoopLogStore;
-    use mito::config::EngineConfig as TableEngineConfig;
-    use mito::engine::MitoEngine;
-    use object_store::services::Fs as Builder;
-    use object_store::{ObjectStore, ObjectStoreBuilder};
-    use query::parser::{QueryLanguageParser, QueryStatement};
-    use query::QueryEngineFactory;
-    use session::context::QueryContext;
-    use sql::statements::statement::Statement;
-    use storage::compaction::noop::NoopCompactionScheduler;
-    use storage::config::EngineConfig as StorageEngineConfig;
-    use storage::EngineImpl;
-    use table::error::Result as TableResult;
-    use table::metadata::TableInfoRef;
-    use table::Table;
-
-    use super::*;
-    use crate::error::Error;
-    use crate::sql::insert::InsertRequests;
-
-    struct DemoTable;
-
-    #[async_trait::async_trait]
-    impl Table for DemoTable {
-        fn as_any(&self) -> &dyn Any {
-            self
-        }
-
-        fn schema(&self) -> SchemaRef {
-            let column_schemas = vec![
-                ColumnSchema::new("host", ConcreteDataType::string_datatype(), false),
-                ColumnSchema::new("cpu", ConcreteDataType::float64_datatype(), true),
-                ColumnSchema::new("memory", ConcreteDataType::float64_datatype(), true),
-                ColumnSchema::new(
-                    "ts",
-                    ConcreteDataType::timestamp_millisecond_datatype(),
-                    true,
-                )
-                .with_time_index(true),
-            ];
-
-            Arc::new(
-                SchemaBuilder::try_from(column_schemas)
-                    .unwrap()
-                    .build()
-                    .unwrap(),
-            )
-        }
-
-        fn table_info(&self) -> TableInfoRef {
-            unimplemented!()
-        }
-
-        async fn scan(
-            &self,
-            _projection: Option<&Vec<usize>>,
-            _filters: &[Expr],
-            _limit: Option<usize>,
-        ) -> TableResult<PhysicalPlanRef> {
-            unimplemented!();
-        }
-    }
-
-    #[tokio::test]
-    async fn test_statement_to_request() {
-        let dir = create_temp_dir("setup_test_engine_and_table");
-        let store_dir = dir.path().to_string_lossy();
-        let accessor = Builder::default().root(&store_dir).build().unwrap();
-        let object_store = ObjectStore::new(accessor).finish();
-        let compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
-        let sql = r#"insert into demo(host, cpu, memory, ts) values
-                           ('host1', 66.6, 1024, 1655276557000),
-                           ('host2', 88.8,  333.3, 1655276558000)
-                           "#;
-
-        let table_engine = Arc::new(MitoEngine::<EngineImpl<NoopLogStore>>::new(
-            TableEngineConfig::default(),
-            EngineImpl::new(
-                StorageEngineConfig::default(),
-                Arc::new(NoopLogStore::default()),
-                object_store.clone(),
-                compaction_scheduler,
-            ),
-            object_store,
-        ));
-
-        let catalog_list = Arc::new(
-            catalog::local::LocalCatalogManager::try_new(table_engine.clone())
-                .await
-                .unwrap(),
-        );
-        catalog_list.start().await.unwrap();
-        assert!(catalog_list
-            .register_table(RegisterTableRequest {
-                catalog: DEFAULT_CATALOG_NAME.to_string(),
-                schema: DEFAULT_SCHEMA_NAME.to_string(),
-                table_name: "demo".to_string(),
-                table_id: 1,
-                table: Arc::new(DemoTable),
-            })
-            .await
-            .unwrap());
-
-        let factory = QueryEngineFactory::new(catalog_list.clone());
-        let query_engine = factory.query_engine();
-        let sql_handler = SqlHandler::new(
-            table_engine.clone(),
-            catalog_list.clone(),
-            query_engine.clone(),
-            table_engine,
-            None,
-        );
-
-        let stmt = match QueryLanguageParser::parse_sql(sql).unwrap() {
-            QueryStatement::Sql(Statement::Insert(i)) => i,
-            _ => {
-                unreachable!()
-            }
-        };
-        let request = sql_handler
-            .insert_to_requests(catalog_list.clone(), *stmt, QueryContext::arc())
-            .await
-            .unwrap();
-
-        match request {
-            InsertRequests::Request(SqlRequest::Insert(req)) => {
-                assert_eq!(req.table_name, "demo");
-                let columns_values = req.columns_values;
-                assert_eq!(4, columns_values.len());
-
-                let hosts = &columns_values["host"];
-                assert_eq!(2, hosts.len());
-                assert_eq!(Value::from("host1"), hosts.get(0));
-                assert_eq!(Value::from("host2"), hosts.get(1));
-
-                let cpus = &columns_values["cpu"];
-                assert_eq!(2, cpus.len());
-                assert_eq!(Value::from(66.6f64), cpus.get(0));
-                assert_eq!(Value::from(88.8f64), cpus.get(1));
-
-                let memories = &columns_values["memory"];
-                assert_eq!(2, memories.len());
-                assert_eq!(Value::from(1024f64), memories.get(0));
-                assert_eq!(Value::from(333.3f64), memories.get(1));
-
-                let ts = &columns_values["ts"];
-                assert_eq!(2, ts.len());
-                assert_eq!(
-                    Value::from(Timestamp::new_millisecond(1655276557000i64)),
-                    ts.get(0)
-                );
-                assert_eq!(
-                    Value::from(Timestamp::new_millisecond(1655276558000i64)),
-                    ts.get(1)
-                );
-            }
-            _ => {
-                panic!("Not supposed to reach here")
-            }
-        }
-
-        // test inert into select
-
-        // type mismatch
-        let sql = "insert into demo(ts) select number from numbers limit 3";
-
-        let stmt = match QueryLanguageParser::parse_sql(sql).unwrap() {
-            QueryStatement::Sql(Statement::Insert(i)) => i,
-            _ => {
-                unreachable!()
-            }
-        };
-        let request = sql_handler
-            .insert_to_requests(catalog_list.clone(), *stmt, QueryContext::arc())
-            .await
-            .unwrap();
-
-        match request {
-            InsertRequests::Stream(mut stream) => {
-                assert!(matches!(
-                    stream.next().await.unwrap().unwrap_err(),
-                    Error::ColumnTypeMismatch { .. }
-                ));
-            }
-            _ => unreachable!(),
-        }
-
-        let sql = "insert into demo(cpu) select cast(number as double) from numbers limit 3";
-        let stmt = match QueryLanguageParser::parse_sql(sql).unwrap() {
-            QueryStatement::Sql(Statement::Insert(i)) => i,
-            _ => {
-                unreachable!()
-            }
-        };
-        let request = sql_handler
-            .insert_to_requests(catalog_list.clone(), *stmt, QueryContext::arc())
-            .await
-            .unwrap();
-
-        match request {
-            InsertRequests::Stream(mut stream) => {
-                let mut times = 0;
-                while let Some(Ok(SqlRequest::Insert(req))) = stream.next().await {
-                    times += 1;
-                    assert_eq!(req.table_name, "demo");
-                    let columns_values = req.columns_values;
-                    assert_eq!(1, columns_values.len());
-
-                    let memories = &columns_values["cpu"];
-                    assert_eq!(3, memories.len());
-                    assert_eq!(Value::from(0.0f64), memories.get(0));
-                    assert_eq!(Value::from(1.0f64), memories.get(1));
-                    assert_eq!(Value::from(2.0f64), memories.get(2));
-                }
-                assert_eq!(1, times);
-            }
-            _ => unreachable!(),
-        }
-    }
-}

src/datanode/src/sql/copy_table_from.rs

@@ -48,7 +48,6 @@ impl SqlHandler {
             build_backend(&req.location, req.connection).context(error::BuildBackendSnafu)?;
 
         let (dir, filename) = find_dir_and_filename(&path);
-
         let regex = req
             .pattern
             .as_ref()
@@ -62,16 +61,18 @@ impl SqlHandler {
             Source::Dir
         };
 
-        let lister = Lister::new(object_store, source, dir, regex);
+        let lister = Lister::new(object_store.clone(), source, dir, regex);
 
-        let objects = lister.list().await.context(error::ListObjectsSnafu)?;
+        let entries = lister.list().await.context(error::ListObjectsSnafu)?;
 
         let mut buf: Vec<RecordBatch> = Vec::new();
 
-        for obj in objects.iter() {
-            let reader = obj.reader().await.context(error::ReadObjectSnafu {
-                path: &obj.path().to_string(),
-            })?;
+        for entry in entries.iter() {
+            let path = entry.path();
+            let reader = object_store
+                .reader(path)
+                .await
+                .context(error::ReadObjectSnafu { path })?;
 
             let buf_reader = BufReader::new(reader.compat());
 

src/datanode/src/sql/copy_table_to.rs

@@ -12,24 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::pin::Pin;
-
 use common_datasource;
 use common_datasource::object_store::{build_backend, parse_url};
 use common_query::physical_plan::SessionContext;
 use common_query::Output;
-use common_recordbatch::adapter::DfRecordBatchStreamAdapter;
-use datafusion::parquet::arrow::ArrowWriter;
-use datafusion::parquet::basic::{Compression, Encoding};
-use datafusion::parquet::file::properties::WriterProperties;
-use datafusion::physical_plan::RecordBatchStream;
-use futures::TryStreamExt;
-use object_store::ObjectStore;
 use snafu::ResultExt;
+use storage::sst::SstInfo;
+use storage::{ParquetWriter, Source};
 use table::engine::TableReference;
 use table::requests::CopyTableRequest;
 
-use crate::error::{self, Result};
+use crate::error::{self, Result, WriteParquetSnafu};
 use crate::sql::SqlHandler;
 
 impl SqlHandler {
@@ -51,99 +44,20 @@ impl SqlHandler {
         let stream = stream
             .execute(0, SessionContext::default().task_ctx())
             .context(error::TableScanExecSnafu)?;
-        let stream = Box::pin(DfRecordBatchStreamAdapter::new(stream));
 
         let (_schema, _host, path) = parse_url(&req.location).context(error::ParseUrlSnafu)?;
         let object_store =
             build_backend(&req.location, req.connection).context(error::BuildBackendSnafu)?;
 
-        let mut parquet_writer = ParquetWriter::new(path.to_string(), stream, object_store);
-        // TODO(jiachun):
-        // For now, COPY is implemented synchronously.
-        // When copying large table, it will be blocked for a long time.
-        // Maybe we should make "copy" runs in background?
-        // Like PG: https://www.postgresql.org/docs/current/sql-copy.html
-        let rows = parquet_writer.flush().await?;
+        let writer = ParquetWriter::new(&path, Source::Stream(stream), object_store);
 
-        Ok(Output::AffectedRows(rows))
-    }
-}
-
-type DfRecordBatchStream = Pin<Box<DfRecordBatchStreamAdapter>>;
-
-struct ParquetWriter {
-    file_name: String,
-    stream: DfRecordBatchStream,
-    object_store: ObjectStore,
-    max_row_group_size: usize,
-    max_rows_in_segment: usize,
-}
-
-impl ParquetWriter {
-    pub fn new(file_name: String, stream: DfRecordBatchStream, object_store: ObjectStore) -> Self {
-        Self {
-            file_name,
-            stream,
-            object_store,
-            // TODO(jiachun): make these configurable: WITH (max_row_group_size=xxx, max_rows_in_segment=xxx)
-            max_row_group_size: 4096,
-            max_rows_in_segment: 5000000, // default 5M rows per segment
-        }
-    }
-
-    pub async fn flush(&mut self) -> Result<usize> {
-        let schema = self.stream.as_ref().schema();
-        let writer_props = WriterProperties::builder()
-            .set_compression(Compression::ZSTD)
-            .set_encoding(Encoding::PLAIN)
-            .set_max_row_group_size(self.max_row_group_size)
-            .build();
-        let mut total_rows = 0;
-        loop {
-            let mut buf = vec![];
-            let mut arrow_writer =
-                ArrowWriter::try_new(&mut buf, schema.clone(), Some(writer_props.clone()))
-                    .context(error::WriteParquetSnafu)?;
-
-            let mut rows = 0;
-            let mut end_loop = true;
-            // TODO(hl & jiachun): Since OpenDAL's writer is async and ArrowWriter requires a `std::io::Write`,
-            // here we use a Vec<u8> to buffer all parquet bytes in memory and write to object store
-            // at a time. Maybe we should find a better way to bridge ArrowWriter and OpenDAL's object.
-            while let Some(batch) = self
-                .stream
-                .try_next()
-                .await
-                .context(error::PollStreamSnafu)?
-            {
-                arrow_writer
-                    .write(&batch)
-                    .context(error::WriteParquetSnafu)?;
-                rows += batch.num_rows();
-                if rows >= self.max_rows_in_segment {
-                    end_loop = false;
-                    break;
-                }
-            }
-
-            let start_row_num = total_rows + 1;
-            total_rows += rows;
-            arrow_writer.close().context(error::WriteParquetSnafu)?;
-
-            // if rows == 0, we just end up with an empty file.
-            //
-            // file_name like:
-            // "file_name_1_1000000"        (row num: 1 ~ 1000000),
-            // "file_name_1000001_xxx"      (row num: 1000001 ~ xxx)
-            let file_name = format!("{}_{}_{}", self.file_name, start_row_num, total_rows);
-            let object = self.object_store.object(&file_name);
-            object.write(buf).await.context(error::WriteObjectSnafu {
-                path: object.path(),
-            })?;
-
-            if end_loop {
-                return Ok(total_rows);
-            }
-        }
+        let rows_copied = writer
+            .write_sst(&storage::sst::WriteOptions::default())
+            .await
+            .context(WriteParquetSnafu)?
+            .map(|SstInfo { num_rows, .. }| num_rows)
+            .unwrap_or(0);
+
+        Ok(Output::AffectedRows(rows_copied))
     }
 }

src/datanode/src/sql/create.rs

@@ -313,13 +313,15 @@ mod tests {
     use common_base::readable_size::ReadableSize;
     use datatypes::prelude::ConcreteDataType;
     use datatypes::schema::Schema;
+    use query::parser::QueryLanguageParser;
+    use session::context::QueryContext;
     use sql::dialect::GenericDialect;
     use sql::parser::ParserContext;
     use sql::statements::statement::Statement;
 
     use super::*;
     use crate::error::Error;
-    use crate::tests::test_util::create_mock_sql_handler;
+    use crate::tests::test_util::{create_mock_sql_handler, MockInstance};
 
     fn sql_to_statement(sql: &str) -> CreateTable {
         let mut res = ParserContext::create_with_dialect(sql, &GenericDialect {}).unwrap();
@@ -522,4 +524,42 @@ mod tests {
             schema.column_schema_by_name("memory").unwrap().data_type
         );
     }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn create_table_by_procedure() {
+        let instance = MockInstance::with_procedure_enabled("create_table_by_procedure").await;
+
+        let sql = r#"create table test_table(
+                            host string,
+                            ts timestamp,
+                            cpu double default 0,
+                            memory double,
+                            TIME INDEX (ts),
+                            PRIMARY KEY(host)
+                        ) engine=mito with(regions=1);"#;
+        let stmt = QueryLanguageParser::parse_sql(sql).unwrap();
+        let output = instance
+            .inner()
+            .execute_stmt(stmt, QueryContext::arc())
+            .await
+            .unwrap();
+        assert!(matches!(output, Output::AffectedRows(0)));
+
+        // create if not exists
+        let sql = r#"create table if not exists test_table(
+                            host string,
+                            ts timestamp,
+                            cpu double default 0,
+                            memory double,
+                            TIME INDEX (ts),
+                            PRIMARY KEY(host)
+                        ) engine=mito with(regions=1);"#;
+        let stmt = QueryLanguageParser::parse_sql(sql).unwrap();
+        let output = instance
+            .inner()
+            .execute_stmt(stmt, QueryContext::arc())
+            .await
+            .unwrap();
+        assert!(matches!(output, Output::AffectedRows(0)));
+    }
 }

src/datanode/src/sql/delete.rs

@@ -1,142 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use common_query::Output;
-use datatypes::data_type::DataType;
-use datatypes::prelude::VectorRef;
-use datatypes::vectors::StringVector;
-use session::context::QueryContextRef;
-use snafu::{OptionExt, ResultExt};
-use sql::ast::{BinaryOperator, Expr, Value};
-use sql::statements::delete::Delete;
-use sql::statements::sql_value_to_value;
-use table::engine::TableReference;
-use table::requests::DeleteRequest;
-use table::TableRef;
-
-use crate::error::{ColumnNotFoundSnafu, DeleteSnafu, InvalidSqlSnafu, NotSupportSqlSnafu, Result};
-use crate::instance::sql::table_idents_to_full_name;
-use crate::sql::SqlHandler;
-
-impl SqlHandler {
-    pub(crate) async fn delete(&self, query_ctx: QueryContextRef, stmt: Delete) -> Result<Output> {
-        let (catalog_name, schema_name, table_name) =
-            table_idents_to_full_name(stmt.table_name(), query_ctx)?;
-        let table_ref = TableReference {
-            catalog: &catalog_name.to_string(),
-            schema: &schema_name.to_string(),
-            table: &table_name.to_string(),
-        };
-
-        let table = self.get_table(&table_ref)?;
-
-        let req = DeleteRequest {
-            key_column_values: parse_selection(stmt.selection(), &table)?,
-        };
-
-        let affected_rows = table.delete(req).await.with_context(|_| DeleteSnafu {
-            table_name: table_ref.to_string(),
-        })?;
-
-        Ok(Output::AffectedRows(affected_rows))
-    }
-}
-
-/// parse selection, currently supported format is `tagkey1 = 'tagvalue1' and 'ts' = 'value'`.
-/// (only uses =, and in the where clause and provides all columns needed by the key.)
-fn parse_selection(
-    selection: &Option<Expr>,
-    table: &TableRef,
-) -> Result<HashMap<String, VectorRef>> {
-    let mut key_column_values = HashMap::new();
-    if let Some(expr) = selection {
-        parse_expr(expr, &mut key_column_values, table)?;
-    }
-    Ok(key_column_values)
-}
-
-fn parse_expr(
-    expr: &Expr,
-    key_column_values: &mut HashMap<String, VectorRef>,
-    table: &TableRef,
-) -> Result<()> {
-    // match BinaryOp
-    if let Expr::BinaryOp { left, op, right } = expr {
-        match (&**left, op, &**right) {
-            // match And operator
-            (Expr::BinaryOp { .. }, BinaryOperator::And, Expr::BinaryOp { .. }) => {
-                parse_expr(left, key_column_values, table)?;
-                parse_expr(right, key_column_values, table)?;
-                return Ok(());
-            }
-            // match Eq operator
-            (Expr::Identifier(column_name), BinaryOperator::Eq, Expr::Value(value)) => {
-                key_column_values.insert(
-                    column_name.to_string(),
-                    value_to_vector(&column_name.to_string(), value, table)?,
-                );
-                return Ok(());
-            }
-            (Expr::Identifier(column_name), BinaryOperator::Eq, Expr::Identifier(value)) => {
-                key_column_values.insert(
-                    column_name.to_string(),
-                    Arc::new(StringVector::from(vec![value.to_string()])),
-                );
-                return Ok(());
-            }
-            _ => {}
-        }
-    }
-    NotSupportSqlSnafu {
-        msg: format!(
-            "Not support sql expr:{expr},correct format is tagkey1 = tagvalue1 and ts = value"
-        ),
-    }
-    .fail()
-}
-
-/// parse value to vector
-fn value_to_vector(column_name: &String, sql_value: &Value, table: &TableRef) -> Result<VectorRef> {
-    let schema = table.schema();
-    let column_schema =
-        schema
-            .column_schema_by_name(column_name)
-            .with_context(|| ColumnNotFoundSnafu {
-                table_name: table.table_info().name.clone(),
-                column_name: column_name.to_string(),
-            })?;
-    let data_type = &column_schema.data_type;
-    let value = sql_value_to_value(column_name, data_type, sql_value);
-    match value {
-        Ok(value) => {
-            let mut vec = data_type.create_mutable_vector(1);
-            if vec.try_push_value_ref(value.as_value_ref()).is_err() {
-                return InvalidSqlSnafu {
-                    msg: format!(
-                        "invalid sql, column name is {column_name}, value is {sql_value}",
-                    ),
-                }
-                    .fail();
-            }
-            Ok(vec.to_vector())
-        }
-        _ => InvalidSqlSnafu {
-            msg: format!("invalid sql, column name is {column_name}, value is {sql_value}",),
-        }
-        .fail(),
-    }
-}

src/datanode/src/sql/flush_table.rs

@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use catalog::SchemaProviderRef;
 use common_query::Output;
 use snafu::{OptionExt, ResultExt};
-use table::engine::TableReference;
 use table::requests::FlushTableRequest;
 
 use crate::error::{self, CatalogSnafu, DatabaseNotFoundSnafu, Result};
@@ -22,34 +22,22 @@ use crate::sql::SqlHandler;
 
 impl SqlHandler {
     pub(crate) async fn flush_table(&self, req: FlushTableRequest) -> Result<Output> {
-        if let Some(table) = &req.table_name {
-            self.flush_table_inner(
-                &req.catalog_name,
-                &req.schema_name,
-                table,
-                req.region_number,
-                req.wait,
-            )
-            .await?;
-        } else {
-            let schema = self
-                .catalog_manager
-                .schema(&req.catalog_name, &req.schema_name)
-                .context(CatalogSnafu)?
-                .context(DatabaseNotFoundSnafu {
-                    catalog: &req.catalog_name,
-                    schema: &req.schema_name,
-                })?;
+        let schema = self
+            .catalog_manager
+            .schema(&req.catalog_name, &req.schema_name)
+            .context(CatalogSnafu)?
+            .context(DatabaseNotFoundSnafu {
+                catalog: &req.catalog_name,
+                schema: &req.schema_name,
+            })?;
 
+        if let Some(table) = &req.table_name {
+            self.flush_table_inner(schema, table, req.region_number, req.wait)
+                .await?;
+        } else {
             let all_table_names = schema.table_names().context(CatalogSnafu)?;
             futures::future::join_all(all_table_names.iter().map(|table| {
-                self.flush_table_inner(
-                    &req.catalog_name,
-                    &req.schema_name,
-                    table,
-                    req.region_number,
-                    req.wait,
-                )
+                self.flush_table_inner(schema.clone(), table, req.region_number, req.wait)
             }))
             .await
             .into_iter()
@@ -60,25 +48,18 @@ impl SqlHandler {
 
     async fn flush_table_inner(
         &self,
-        catalog: &str,
-        schema: &str,
-        table: &str,
+        schema: SchemaProviderRef,
+        table_name: &str,
         region: Option<u32>,
         wait: Option<bool>,
     ) -> Result<()> {
-        let table_ref = TableReference {
-            catalog,
-            schema,
-            table,
-        };
-
-        let full_table_name = table_ref.to_string();
-        let table = self.get_table(&table_ref)?;
-        table
+        schema
+            .table(table_name)
+            .await
+            .context(error::FindTableSnafu { table_name })?
+            .context(error::TableNotFoundSnafu { table_name })?
             .flush(region, wait)
             .await
-            .context(error::FlushTableSnafu {
-                table_name: full_table_name,
-            })
+            .context(error::FlushTableSnafu { table_name })
     }
 }

src/datanode/src/sql/insert.rs

@@ -11,49 +11,31 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-use std::collections::HashMap;
-use std::pin::Pin;
 
 use catalog::CatalogManagerRef;
 use common_catalog::format_full_table_name;
 use common_query::Output;
-use common_recordbatch::RecordBatch;
-use datafusion_expr::type_coercion::binary::coerce_types;
-use datafusion_expr::Operator;
 use datatypes::data_type::DataType;
 use datatypes::schema::ColumnSchema;
 use datatypes::vectors::MutableVector;
-use futures::stream::{self, StreamExt};
-use futures::Stream;
-use query::parser::QueryStatement;
 use session::context::QueryContextRef;
 use snafu::{ensure, OptionExt, ResultExt};
 use sql::ast::Value as SqlValue;
 use sql::statements::insert::Insert;
-use sql::statements::statement::Statement;
 use sql::statements::{self};
 use table::engine::TableReference;
 use table::requests::*;
 use table::TableRef;
 
 use crate::error::{
-    CatalogSnafu, CollectRecordsSnafu, ColumnDefaultValueSnafu, ColumnNoneDefaultValueSnafu,
-    ColumnNotFoundSnafu, ColumnTypeMismatchSnafu, ColumnValuesNumberMismatchSnafu, Error,
-    ExecuteLogicalPlanSnafu, InsertSnafu, MissingInsertBodySnafu, ParseSqlSnafu,
-    ParseSqlValueSnafu, PlanStatementSnafu, Result, TableNotFoundSnafu,
+    CatalogSnafu, ColumnDefaultValueSnafu, ColumnNoneDefaultValueSnafu, ColumnNotFoundSnafu,
+    ColumnValuesNumberMismatchSnafu, InsertSnafu, MissingInsertBodySnafu, ParseSqlSnafu,
+    ParseSqlValueSnafu, Result, TableNotFoundSnafu,
 };
-use crate::sql::{table_idents_to_full_name, SqlHandler, SqlRequest};
+use crate::sql::{table_idents_to_full_name, SqlHandler};
 
 const DEFAULT_PLACEHOLDER_VALUE: &str = "default";
 
-type InsertRequestStream = Pin<Box<dyn Stream<Item = Result<SqlRequest>> + Send>>;
-pub(crate) enum InsertRequests {
-    // Single request
-    Request(SqlRequest),
-    // Streaming requests
-    Stream(InsertRequestStream),
-}
-
 impl SqlHandler {
     pub(crate) async fn insert(&self, req: InsertRequest) -> Result<Output> {
         // FIXME(dennis): table_ref is used in InsertSnafu and the req is consumed
@@ -77,7 +59,7 @@ impl SqlHandler {
         table_ref: TableReference,
         table: &TableRef,
         stmt: Insert,
-    ) -> Result<SqlRequest> {
+    ) -> Result<InsertRequest> {
         let values = stmt
             .values_body()
             .context(ParseSqlValueSnafu)?
@@ -129,7 +111,7 @@ impl SqlHandler {
             }
         }
 
-        Ok(SqlRequest::Insert(InsertRequest {
+        Ok(InsertRequest {
             catalog_name: table_ref.catalog.to_string(),
             schema_name: table_ref.schema.to_string(),
             table_name: table_ref.table.to_string(),
@@ -138,150 +120,14 @@ impl SqlHandler {
                 .map(|(cs, mut b)| (cs.name.to_string(), b.to_vector()))
                 .collect(),
             region_number: 0,
-        }))
+        })
     }
 
-    fn build_request_from_batch(
-        stmt: Insert,
-        table: TableRef,
-        batch: RecordBatch,
-        query_ctx: QueryContextRef,
-    ) -> Result<SqlRequest> {
-        let (catalog_name, schema_name, table_name) =
-            table_idents_to_full_name(stmt.table_name(), query_ctx)?;
-
-        let schema = table.schema();
-        let columns: Vec<_> = if stmt.columns().is_empty() {
-            schema
-                .column_schemas()
-                .iter()
-                .map(|c| c.name.to_string())
-                .collect()
-        } else {
-            stmt.columns().iter().map(|c| (*c).clone()).collect()
-        };
-        let columns_num = columns.len();
-
-        ensure!(
-            batch.num_columns() == columns_num,
-            ColumnValuesNumberMismatchSnafu {
-                columns: columns_num,
-                values: batch.num_columns(),
-            }
-        );
-
-        let batch_schema = &batch.schema;
-        let batch_columns = batch_schema.column_schemas();
-        assert_eq!(batch_columns.len(), columns_num);
-        let mut columns_values = HashMap::with_capacity(columns_num);
-
-        for (i, column_name) in columns.into_iter().enumerate() {
-            let column_schema = schema
-                .column_schema_by_name(&column_name)
-                .with_context(|| ColumnNotFoundSnafu {
-                    table_name: &table_name,
-                    column_name: &column_name,
-                })?;
-            let expect_datatype = column_schema.data_type.as_arrow_type();
-            // It's safe to retrieve the column schema by index, we already
-            // check columns number is the same above.
-            let batch_datatype = batch_columns[i].data_type.as_arrow_type();
-            let coerced_type = coerce_types(&expect_datatype, &Operator::Eq, &batch_datatype)
-                .map_err(|_| Error::ColumnTypeMismatch {
-                    column: column_name.clone(),
-                    expected: column_schema.data_type.clone(),
-                    actual: batch_columns[i].data_type.clone(),
-                })?;
-
-            ensure!(
-                expect_datatype == coerced_type,
-                ColumnTypeMismatchSnafu {
-                    column: column_name,
-                    expected: column_schema.data_type.clone(),
-                    actual: batch_columns[i].data_type.clone(),
-                }
-            );
-            let vector = batch
-                .column(i)
-                .cast(&column_schema.data_type)
-                .map_err(|_| Error::ColumnTypeMismatch {
-                    column: column_name.clone(),
-                    expected: column_schema.data_type.clone(),
-                    actual: batch_columns[i].data_type.clone(),
-                })?;
-
-            columns_values.insert(column_name, vector);
-        }
-
-        Ok(SqlRequest::Insert(InsertRequest {
-            catalog_name,
-            schema_name,
-            table_name,
-            columns_values,
-            region_number: 0,
-        }))
-    }
-
-    // FIXME(dennis): move it to frontend when refactor is done.
-    async fn build_stream_from_query(
-        &self,
-        table: TableRef,
-        stmt: Insert,
-        query_ctx: QueryContextRef,
-    ) -> Result<InsertRequestStream> {
-        let query = stmt
-            .query_body()
-            .context(ParseSqlValueSnafu)?
-            .context(MissingInsertBodySnafu)?;
-
-        let logical_plan = self
-            .query_engine
-            .planner()
-            .plan(
-                QueryStatement::Sql(Statement::Query(Box::new(query))),
-                query_ctx.clone(),
-            )
-            .await
-            .context(PlanStatementSnafu)?;
-
-        let output = self
-            .query_engine
-            .execute(&logical_plan)
-            .await
-            .context(ExecuteLogicalPlanSnafu)?;
-
-        let stream: InsertRequestStream = match output {
-            Output::RecordBatches(batches) => {
-                Box::pin(stream::iter(batches.take()).map(move |batch| {
-                    Self::build_request_from_batch(
-                        stmt.clone(),
-                        table.clone(),
-                        batch,
-                        query_ctx.clone(),
-                    )
-                }))
-            }
-
-            Output::Stream(stream) => Box::pin(stream.map(move |batch| {
-                Self::build_request_from_batch(
-                    stmt.clone(),
-                    table.clone(),
-                    batch.context(CollectRecordsSnafu)?,
-                    query_ctx.clone(),
-                )
-            })),
-            _ => unreachable!(),
-        };
-
-        Ok(stream)
-    }
-
-    pub(crate) async fn insert_to_requests(
-        &self,
+    pub async fn insert_to_request(
         catalog_manager: CatalogManagerRef,
         stmt: Insert,
         query_ctx: QueryContextRef,
-    ) -> Result<InsertRequests> {
+    ) -> Result<InsertRequest> {
         let (catalog_name, schema_name, table_name) =
             table_idents_to_full_name(stmt.table_name(), query_ctx.clone())?;
 
@@ -293,16 +139,8 @@ impl SqlHandler {
                 table_name: format_full_table_name(&catalog_name, &schema_name, &table_name),
             })?;
 
-        if stmt.is_insert_select() {
-            Ok(InsertRequests::Stream(
-                self.build_stream_from_query(table, stmt, query_ctx).await?,
-            ))
-        } else {
-            let table_ref = TableReference::full(&catalog_name, &schema_name, &table_name);
-            Ok(InsertRequests::Request(Self::build_request_from_values(
-                table_ref, &table, stmt,
-            )?))
-        }
+        let table_ref = TableReference::full(&catalog_name, &schema_name, &table_name);
+        Self::build_request_from_values(table_ref, &table, stmt)
     }
 }
 

src/datanode/src/tests.rs

@@ -13,6 +13,5 @@
 // limitations under the License.
 
 // TODO(LFC): These tests should be moved to frontend crate. They are actually standalone instance tests.
-mod instance_test;
 mod promql_test;
 pub(crate) mod test_util;

src/datanode/src/tests/test_util.rs

@@ -24,7 +24,6 @@ use datatypes::schema::{ColumnSchema, RawSchema};
 use mito::config::EngineConfig;
 use mito::table::test_util::{new_test_object_store, MockEngine, MockMitoEngine};
 use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
-use query::QueryEngineFactory;
 use servers::Mode;
 use session::context::QueryContext;
 use snafu::ResultExt;
@@ -33,7 +32,9 @@ use sql::statements::tql::Tql;
 use table::engine::{EngineContext, TableEngineRef};
 use table::requests::{CreateTableRequest, TableOptions};
 
-use crate::datanode::{DatanodeOptions, FileConfig, ObjectStoreConfig, ProcedureConfig, WalConfig};
+use crate::datanode::{
+    DatanodeOptions, FileConfig, ObjectStoreConfig, ProcedureConfig, StorageConfig, WalConfig,
+};
 use crate::error::{CreateTableSnafu, Result};
 use crate::instance::Instance;
 use crate::sql::SqlHandler;
@@ -87,7 +88,7 @@ impl MockInstance {
         match stmt {
             QueryStatement::Sql(Statement::Query(_)) => {
                 let plan = planner.plan(stmt, QueryContext::arc()).await.unwrap();
-                engine.execute(&plan).await.unwrap()
+                engine.execute(plan, QueryContext::arc()).await.unwrap()
             }
             QueryStatement::Sql(Statement::Tql(tql)) => {
                 let plan = match tql {
@@ -103,7 +104,7 @@ impl MockInstance {
                     }
                     Tql::Explain(_) => unimplemented!(),
                 };
-                engine.execute(&plan).await.unwrap()
+                engine.execute(plan, QueryContext::arc()).await.unwrap()
             }
             _ => self
                 .inner()
@@ -131,9 +132,12 @@ fn create_tmp_dir_and_datanode_opts(name: &str) -> (DatanodeOptions, TestGuard)
             dir: wal_tmp_dir.path().to_str().unwrap().to_string(),
             ..Default::default()
         },
-        storage: ObjectStoreConfig::File(FileConfig {
-            data_dir: data_tmp_dir.path().to_str().unwrap().to_string(),
-        }),
+        storage: StorageConfig {
+            store: ObjectStoreConfig::File(FileConfig {
+                data_dir: data_tmp_dir.path().to_str().unwrap().to_string(),
+            }),
+            ..Default::default()
+        },
         mode: Mode::Standalone,
         ..Default::default()
     };
@@ -201,33 +205,13 @@ pub async fn create_mock_sql_handler() -> SqlHandler {
             .await
             .unwrap(),
     );
-
-    let catalog_list = catalog::local::new_memory_catalog_list().unwrap();
-    let factory = QueryEngineFactory::new(catalog_list);
-
-    SqlHandler::new(
-        mock_engine.clone(),
-        catalog_manager,
-        factory.query_engine(),
-        mock_engine,
-        None,
-    )
+    SqlHandler::new(mock_engine.clone(), catalog_manager, mock_engine, None)
 }
 
 pub(crate) async fn setup_test_instance(test_name: &str) -> MockInstance {
     MockInstance::new(test_name).await
 }
 
-pub async fn check_output_stream(output: Output, expected: String) {
-    let recordbatches = match output {
-        Output::Stream(stream) => util::collect_batches(stream).await.unwrap(),
-        Output::RecordBatches(recordbatches) => recordbatches,
-        _ => unreachable!(),
-    };
-    let pretty_print = recordbatches.pretty_print().unwrap();
-    assert_eq!(pretty_print, expected, "{}", pretty_print);
-}
-
 pub async fn check_unordered_output_stream(output: Output, expected: String) {
     let sort_table = |table: String| -> String {
         let replaced = table.replace("\\n", "\n");

src/datatypes/src/vectors/validity.rs

@@ -13,13 +13,13 @@
 // limitations under the License.
 
 use arrow::array::ArrayData;
-use arrow::bitmap::Bitmap;
+use arrow::buffer::NullBuffer;
 
 #[derive(Debug, PartialEq)]
 enum ValidityKind<'a> {
     /// Whether the array slot is valid or not (null).
     Slots {
-        bitmap: &'a Bitmap,
+        bitmap: &'a NullBuffer,
         len: usize,
         null_count: usize,
     },
@@ -38,7 +38,7 @@ pub struct Validity<'a> {
 impl<'a> Validity<'a> {
     /// Creates a `Validity` from [`ArrayData`].
     pub fn from_array_data(data: &'a ArrayData) -> Validity<'a> {
-        match data.null_bitmap() {
+        match data.nulls() {
             Some(bitmap) => Validity {
                 kind: ValidityKind::Slots {
                     bitmap,
@@ -67,7 +67,7 @@ impl<'a> Validity<'a> {
     /// Returns whether `i-th` bit is set.
     pub fn is_set(&self, i: usize) -> bool {
         match self.kind {
-            ValidityKind::Slots { bitmap, .. } => bitmap.is_set(i),
+            ValidityKind::Slots { bitmap, .. } => bitmap.is_valid(i),
             ValidityKind::AllValid { len } => i < len,
             ValidityKind::AllNull { .. } => false,
         }

src/frontend/Cargo.toml

@@ -4,6 +4,10 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+[features]
+default = ["python"]
+python = ["dep:script"]
+
 [dependencies]
 api = { path = "../api" }
 async-stream.workspace = true
@@ -30,12 +34,14 @@ futures = "0.3"
 futures-util.workspace = true
 itertools = "0.10"
 meta-client = { path = "../meta-client" }
+mito = { path = "../mito", features = ["test"] }
 moka = { version = "0.9", features = ["future"] }
 openmetrics-parser = "0.4"
 partition = { path = "../partition" }
 prost.workspace = true
 query = { path = "../query" }
 rustls = "0.20"
+script = { path = "../script", features = ["python"], optional = true }
 serde = "1.0"
 serde_json = "1.0"
 servers = { path = "../servers" }
@@ -56,3 +62,4 @@ meta-srv = { path = "../meta-srv", features = ["mock"] }
 strfmt = "0.2"
 toml = "0.5"
 tower = "0.4"
+uuid.workspace = true

src/frontend/src/catalog.rs

@@ -16,8 +16,12 @@ use std::any::Any;
 use std::collections::HashSet;
 use std::sync::Arc;
 
+use api::v1::CreateTableExpr;
 use async_trait::async_trait;
-use catalog::error::{self as catalog_err, InvalidCatalogValueSnafu, Result as CatalogResult};
+use catalog::error::{
+    self as catalog_err, InternalSnafu, InvalidCatalogValueSnafu, InvalidSystemTableDefSnafu,
+    Result as CatalogResult, UnimplementedSnafu,
+};
 use catalog::helper::{
     build_catalog_prefix, build_schema_prefix, build_table_global_prefix, CatalogKey, SchemaKey,
     TableGlobalKey, TableGlobalValue,
@@ -28,6 +32,7 @@ use catalog::{
     RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
     SchemaProvider, SchemaProviderRef,
 };
+use common_error::prelude::BoxedError;
 use common_telemetry::error;
 use futures::StreamExt;
 use meta_client::rpc::TableName;
@@ -36,6 +41,8 @@ use snafu::prelude::*;
 use table::TableRef;
 
 use crate::datanode::DatanodeClients;
+use crate::expr_factory;
+use crate::instance::distributed::DistInstance;
 use crate::table::DistTable;
 
 #[derive(Clone)]
@@ -43,6 +50,12 @@ pub struct FrontendCatalogManager {
     backend: KvBackendRef,
     partition_manager: PartitionRuleManagerRef,
     datanode_clients: Arc<DatanodeClients>,
+
+    // TODO(LFC): Remove this field.
+    // DistInstance in FrontendCatalogManager is only used for creating distributed script table now.
+    // Once we have some standalone distributed table creator (like create distributed table procedure),
+    // we should use that.
+    dist_instance: Option<Arc<DistInstance>>,
 }
 
 impl FrontendCatalogManager {
@@ -55,9 +68,14 @@ impl FrontendCatalogManager {
             backend,
             partition_manager,
             datanode_clients,
+            dist_instance: None,
         }
     }
 
+    pub(crate) fn set_dist_instance(&mut self, dist_instance: Arc<DistInstance>) {
+        self.dist_instance = Some(dist_instance)
+    }
+
     pub(crate) fn backend(&self) -> KvBackendRef {
         self.backend.clone()
     }
@@ -106,9 +124,93 @@ impl CatalogManager for FrontendCatalogManager {
 
     async fn register_system_table(
         &self,
-        _request: RegisterSystemTableRequest,
+        request: RegisterSystemTableRequest,
     ) -> catalog::error::Result<()> {
-        unimplemented!()
+        if let Some(dist_instance) = &self.dist_instance {
+            let open_hook = request.open_hook;
+            let request = request.create_table_request;
+
+            if let Some(table) = self
+                .table(
+                    &request.catalog_name,
+                    &request.schema_name,
+                    &request.table_name,
+                )
+                .await?
+            {
+                if let Some(hook) = open_hook {
+                    (hook)(table)?;
+                }
+                return Ok(());
+            }
+
+            let time_index = request
+                .schema
+                .column_schemas
+                .iter()
+                .find_map(|x| {
+                    if x.is_time_index() {
+                        Some(x.name.clone())
+                    } else {
+                        None
+                    }
+                })
+                .context(InvalidSystemTableDefSnafu {
+                    err_msg: "Time index is not defined.",
+                })?;
+
+            let primary_keys = request
+                .schema
+                .column_schemas
+                .iter()
+                .enumerate()
+                .filter_map(|(i, x)| {
+                    if request.primary_key_indices.contains(&i) {
+                        Some(x.name.clone())
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+
+            let column_defs = expr_factory::column_schemas_to_defs(request.schema.column_schemas)
+                .map_err(|e| {
+                InvalidSystemTableDefSnafu {
+                    err_msg: e.to_string(),
+                }
+                .build()
+            })?;
+
+            let mut create_table = CreateTableExpr {
+                catalog_name: request.catalog_name,
+                schema_name: request.schema_name,
+                table_name: request.table_name,
+                desc: request.desc.unwrap_or("".to_string()),
+                column_defs,
+                time_index,
+                primary_keys,
+                create_if_not_exists: request.create_if_not_exists,
+                table_options: (&request.table_options).into(),
+                table_id: None, // Should and will be assigned by Meta.
+                region_ids: vec![0],
+            };
+
+            let table = dist_instance
+                .create_table(&mut create_table, None)
+                .await
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?;
+
+            if let Some(hook) = open_hook {
+                (hook)(table)?;
+            }
+            Ok(())
+        } else {
+            UnimplementedSnafu {
+                operation: "register system table",
+            }
+            .fail()
+        }
     }
 
     fn schema(
@@ -330,3 +432,70 @@ impl SchemaProvider for FrontendSchemaProvider {
         Ok(self.table_names()?.contains(&name.to_string()))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use script::table::{build_scripts_schema, SCRIPTS_TABLE_NAME};
+    use table::requests::{CreateTableRequest, TableOptions};
+
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_register_system_table() {
+        let instance =
+            crate::tests::create_distributed_instance("test_register_system_table").await;
+
+        let catalog_name = DEFAULT_CATALOG_NAME;
+        let schema_name = DEFAULT_SCHEMA_NAME;
+        let table_name = SCRIPTS_TABLE_NAME;
+        let request = CreateTableRequest {
+            id: 1,
+            catalog_name: catalog_name.to_string(),
+            schema_name: schema_name.to_string(),
+            table_name: table_name.to_string(),
+            desc: Some("Scripts table".to_string()),
+            schema: build_scripts_schema(),
+            region_numbers: vec![0],
+            primary_key_indices: vec![0, 1],
+            create_if_not_exists: true,
+            table_options: TableOptions::default(),
+        };
+
+        let result = instance
+            .catalog_manager
+            .register_system_table(RegisterSystemTableRequest {
+                create_table_request: request,
+                open_hook: None,
+            })
+            .await;
+        assert!(result.is_ok());
+
+        assert!(
+            instance
+                .catalog_manager
+                .table(catalog_name, schema_name, table_name)
+                .await
+                .unwrap()
+                .is_some(),
+            "the registered system table cannot be found in catalog"
+        );
+
+        let mut actually_created_table_in_datanode = 0;
+        for datanode in instance.datanodes.values() {
+            if datanode
+                .catalog_manager()
+                .table(catalog_name, schema_name, table_name)
+                .await
+                .unwrap()
+                .is_some()
+            {
+                actually_created_table_in_datanode += 1;
+            }
+        }
+        assert_eq!(
+            actually_created_table_in_datanode, 1,
+            "system table should be actually created at one and only one datanode"
+        )
+    }
+}

src/frontend/src/error.rs

@@ -378,6 +378,12 @@ pub enum Error {
         #[snafu(backtrace)]
         source: table::error::Error,
     },
+
+    #[snafu(display("Failed to start script manager, source: {}", source))]
+    StartScriptManager {
+        #[snafu(backtrace)]
+        source: script::error::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -462,6 +468,8 @@ impl ErrorExt for Error {
                 source.status_code()
             }
             Error::UnrecognizedTableOption { .. } => StatusCode::InvalidArguments,
+
+            Error::StartScriptManager { source } => source.status_code(),
         }
     }
 

src/frontend/src/expr_factory.rs

@@ -187,7 +187,12 @@ fn columns_to_expr(
         .iter()
         .map(|c| column_def_to_schema(c, c.name.to_string() == time_index).context(ParseSqlSnafu))
         .collect::<Result<Vec<ColumnSchema>>>()?;
+    column_schemas_to_defs(column_schemas)
+}
 
+pub(crate) fn column_schemas_to_defs(
+    column_schemas: Vec<ColumnSchema>,
+) -> Result<Vec<api::v1::ColumnDef>> {
     let column_datatypes = column_schemas
         .iter()
         .map(|c| {

src/frontend/src/instance.rs

@@ -17,6 +17,7 @@ mod grpc;
 mod influxdb;
 mod opentsdb;
 mod prometheus;
+mod script;
 mod standalone;
 
 use std::collections::HashMap;
@@ -40,7 +41,6 @@ use common_telemetry::timer;
 use datafusion::sql::sqlparser::ast::ObjectName;
 use datanode::instance::sql::table_idents_to_full_name;
 use datanode::instance::InstanceRef as DnInstanceRef;
-use datanode::metric;
 use datatypes::schema::Schema;
 use distributed::DistInstance;
 use meta_client::client::{MetaClient, MetaClientBuilder};
@@ -58,26 +58,29 @@ use servers::query_handler::grpc::{GrpcQueryHandler, GrpcQueryHandlerRef};
 use servers::query_handler::sql::SqlQueryHandler;
 use servers::query_handler::{
     InfluxdbLineProtocolHandler, OpentsdbProtocolHandler, PrometheusProtocolHandler, ScriptHandler,
-    ScriptHandlerRef,
 };
 use session::context::QueryContextRef;
 use snafu::prelude::*;
 use sql::dialect::GenericDialect;
 use sql::parser::ParserContext;
 use sql::statements::copy::CopyTable;
+use sql::statements::describe::DescribeTable;
 use sql::statements::statement::Statement;
 use sql::statements::tql::Tql;
 
 use crate::catalog::FrontendCatalogManager;
 use crate::datanode::DatanodeClients;
 use crate::error::{
-    self, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExecuteStatementSnafu, ExternalSnafu,
-    InvalidInsertRequestSnafu, MissingMetasrvOptsSnafu, NotSupportedSnafu, ParseQuerySnafu,
-    ParseSqlSnafu, PlanStatementSnafu, Result, SqlExecInterceptedSnafu,
+    self, CatalogSnafu, DescribeStatementSnafu, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu,
+    ExecuteStatementSnafu, ExternalSnafu, InvalidInsertRequestSnafu, MissingMetasrvOptsSnafu,
+    NotSupportedSnafu, ParseQuerySnafu, ParseSqlSnafu, PlanStatementSnafu, Result,
+    SqlExecInterceptedSnafu, TableNotFoundSnafu,
 };
 use crate::expr_factory::{CreateExprFactoryRef, DefaultCreateExprFactory};
 use crate::frontend::FrontendOptions;
 use crate::instance::standalone::StandaloneGrpcQueryHandler;
+use crate::metric;
+use crate::script::ScriptExecutor;
 use crate::server::{start_server, ServerHandlers, Services};
 
 #[async_trait]
@@ -101,9 +104,7 @@ pub type FrontendInstanceRef = Arc<dyn FrontendInstance>;
 #[derive(Clone)]
 pub struct Instance {
     catalog_manager: CatalogManagerRef,
-
-    /// Script handler is None in distributed mode, only works on standalone mode.
-    script_handler: Option<ScriptHandlerRef>,
+    script_executor: Arc<ScriptExecutor>,
     statement_handler: StatementHandlerRef,
     query_engine: QueryEngineRef,
     grpc_query_handler: GrpcQueryHandlerRef<Error>,
@@ -132,23 +133,29 @@ impl Instance {
         let partition_manager = Arc::new(PartitionRuleManager::new(table_routes));
         let datanode_clients = Arc::new(DatanodeClients::default());
 
-        let catalog_manager = Arc::new(FrontendCatalogManager::new(
-            meta_backend,
-            partition_manager,
-            datanode_clients.clone(),
-        ));
+        let mut catalog_manager =
+            FrontendCatalogManager::new(meta_backend, partition_manager, datanode_clients.clone());
 
-        let dist_instance =
-            DistInstance::new(meta_client, catalog_manager.clone(), datanode_clients);
+        let dist_instance = DistInstance::new(
+            meta_client,
+            Arc::new(catalog_manager.clone()),
+            datanode_clients,
+        );
         let dist_instance = Arc::new(dist_instance);
 
+        catalog_manager.set_dist_instance(dist_instance.clone());
+        let catalog_manager = Arc::new(catalog_manager);
+
         let query_engine =
             QueryEngineFactory::new_with_plugins(catalog_manager.clone(), plugins.clone())
                 .query_engine();
 
+        let script_executor =
+            Arc::new(ScriptExecutor::new(catalog_manager.clone(), query_engine.clone()).await?);
+
         Ok(Instance {
             catalog_manager,
-            script_handler: None,
+            script_executor,
             create_expr_factory: Arc::new(DefaultCreateExprFactory),
             statement_handler: dist_instance.clone(),
             query_engine,
@@ -189,18 +196,22 @@ impl Instance {
         Ok(Arc::new(meta_client))
     }
 
-    pub fn new_standalone(dn_instance: DnInstanceRef) -> Self {
-        Instance {
-            catalog_manager: dn_instance.catalog_manager().clone(),
-            script_handler: None,
+    pub async fn try_new_standalone(dn_instance: DnInstanceRef) -> Result<Self> {
+        let catalog_manager = dn_instance.catalog_manager();
+        let query_engine = dn_instance.query_engine();
+        let script_executor =
+            Arc::new(ScriptExecutor::new(catalog_manager.clone(), query_engine.clone()).await?);
+        Ok(Instance {
+            catalog_manager: catalog_manager.clone(),
+            script_executor,
             create_expr_factory: Arc::new(DefaultCreateExprFactory),
             statement_handler: dn_instance.clone(),
-            query_engine: dn_instance.query_engine(),
+            query_engine,
             grpc_query_handler: StandaloneGrpcQueryHandler::arc(dn_instance.clone()),
             promql_handler: Some(dn_instance.clone()),
             plugins: Default::default(),
             servers: Arc::new(HashMap::new()),
-        }
+        })
     }
 
     pub async fn build_servers(
@@ -215,12 +226,19 @@ impl Instance {
     }
 
     #[cfg(test)]
-    pub(crate) fn new_distributed(dist_instance: Arc<DistInstance>) -> Self {
-        let catalog_manager = dist_instance.catalog_manager();
+    pub(crate) async fn new_distributed(
+        catalog_manager: CatalogManagerRef,
+        dist_instance: Arc<DistInstance>,
+    ) -> Self {
         let query_engine = QueryEngineFactory::new(catalog_manager.clone()).query_engine();
+        let script_executor = Arc::new(
+            ScriptExecutor::new(catalog_manager.clone(), query_engine.clone())
+                .await
+                .unwrap(),
+        );
         Instance {
             catalog_manager,
-            script_handler: None,
+            script_executor,
             statement_handler: dist_instance.clone(),
             query_engine,
             create_expr_factory: Arc::new(DefaultCreateExprFactory),
@@ -235,14 +253,6 @@ impl Instance {
         &self.catalog_manager
     }
 
-    pub fn set_script_handler(&mut self, handler: ScriptHandlerRef) {
-        debug_assert!(
-            self.script_handler.is_none(),
-            "Script handler can be set only once!"
-        );
-        self.script_handler = Some(handler);
-    }
-
     /// Handle batch inserts
     pub async fn handle_inserts(
         &self,
@@ -428,52 +438,87 @@ fn parse_stmt(sql: &str) -> Result<Vec<Statement>> {
 }
 
 impl Instance {
+    async fn plan_exec(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output> {
+        let planner = self.query_engine.planner();
+        let plan = planner
+            .plan(QueryStatement::Sql(stmt), query_ctx.clone())
+            .await
+            .context(PlanStatementSnafu)?;
+        self.query_engine
+            .execute(plan, query_ctx)
+            .await
+            .context(ExecLogicalPlanSnafu)
+    }
+
+    async fn execute_tql(&self, tql: Tql, query_ctx: QueryContextRef) -> Result<Output> {
+        let plan = match tql {
+            Tql::Eval(eval) => {
+                let promql = PromQuery {
+                    start: eval.start,
+                    end: eval.end,
+                    step: eval.step,
+                    query: eval.query,
+                };
+                let stmt = QueryLanguageParser::parse_promql(&promql).context(ParseQuerySnafu)?;
+                self.query_engine
+                    .planner()
+                    .plan(stmt, query_ctx.clone())
+                    .await
+                    .context(PlanStatementSnafu)?
+            }
+            Tql::Explain(_) => unimplemented!(),
+        };
+        self.query_engine
+            .execute(plan, query_ctx)
+            .await
+            .context(ExecLogicalPlanSnafu)
+    }
+
+    async fn describe_table(
+        &self,
+        stmt: DescribeTable,
+        query_ctx: QueryContextRef,
+    ) -> Result<Output> {
+        let (catalog, schema, table) = table_idents_to_full_name(stmt.name(), query_ctx)
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)?;
+
+        let table = self
+            .catalog_manager
+            .table(&catalog, &schema, &table)
+            .await
+            .context(CatalogSnafu)?
+            .with_context(|| TableNotFoundSnafu {
+                table_name: stmt.name().to_string(),
+            })?;
+
+        query::sql::describe_table(table).context(DescribeStatementSnafu)
+    }
+
     async fn query_statement(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output> {
         check_permission(self.plugins.clone(), &stmt, &query_ctx)?;
 
-        let planner = self.query_engine.planner();
-
         match stmt {
-            Statement::Query(_) | Statement::Explain(_) => {
-                let plan = planner
-                    .plan(QueryStatement::Sql(stmt), query_ctx)
-                    .await
-                    .context(PlanStatementSnafu)?;
-                self.query_engine
-                    .execute(&plan)
-                    .await
-                    .context(ExecLogicalPlanSnafu)
+            Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => {
+                self.plan_exec(stmt, query_ctx).await
             }
-            Statement::Tql(tql) => {
-                let plan = match tql {
-                    Tql::Eval(eval) => {
-                        let promql = PromQuery {
-                            start: eval.start,
-                            end: eval.end,
-                            step: eval.step,
-                            query: eval.query,
-                        };
-                        let stmt =
-                            QueryLanguageParser::parse_promql(&promql).context(ParseQuerySnafu)?;
-                        planner
-                            .plan(stmt, query_ctx)
-                            .await
-                            .context(PlanStatementSnafu)?
-                    }
-                    Tql::Explain(_) => unimplemented!(),
-                };
-                self.query_engine
-                    .execute(&plan)
-                    .await
-                    .context(ExecLogicalPlanSnafu)
+
+            // For performance consideration, only "insert with select" is executed by query engine.
+            // Plain insert ("insert with values") is still executed directly in statement.
+            Statement::Insert(ref insert) if insert.is_insert_select() => {
+                self.plan_exec(stmt, query_ctx).await
             }
+
+            Statement::Tql(tql) => self.execute_tql(tql, query_ctx).await,
+
+            Statement::DescribeTable(stmt) => self.describe_table(stmt, query_ctx).await,
+
             Statement::CreateDatabase(_)
+            | Statement::CreateExternalTable(_)
             | Statement::ShowDatabases(_)
             | Statement::CreateTable(_)
             | Statement::ShowTables(_)
-            | Statement::DescribeTable(_)
             | Statement::Insert(_)
-            | Statement::Delete(_)
             | Statement::Alter(_)
             | Statement::DropTable(_)
             | Statement::Copy(_) => self
@@ -583,41 +628,6 @@ impl SqlQueryHandler for Instance {
     }
 }
 
-#[async_trait]
-impl ScriptHandler for Instance {
-    async fn insert_script(
-        &self,
-        schema: &str,
-        name: &str,
-        script: &str,
-    ) -> server_error::Result<()> {
-        if let Some(handler) = &self.script_handler {
-            handler.insert_script(schema, name, script).await
-        } else {
-            server_error::NotSupportedSnafu {
-                feat: "Script execution in Frontend",
-            }
-            .fail()
-        }
-    }
-
-    async fn execute_script(
-        &self,
-        schema: &str,
-        script: &str,
-        params: HashMap<String, String>,
-    ) -> server_error::Result<Output> {
-        if let Some(handler) = &self.script_handler {
-            handler.execute_script(schema, script, params).await
-        } else {
-            server_error::NotSupportedSnafu {
-                feat: "Script execution in Frontend",
-            }
-            .fail()
-        }
-    }
-}
-
 #[async_trait]
 impl PromHandler for Instance {
     async fn do_query(&self, query: &PromQuery) -> server_error::Result<Output> {
@@ -647,12 +657,13 @@ pub fn check_permission(
     }
 
     match stmt {
-        // query,explain and tql will be checked in QueryEngineState
-        Statement::Query(_) | Statement::Explain(_) | Statement::Tql(_) => {}
+        // These are executed by query engine, and will be checked there.
+        Statement::Query(_) | Statement::Explain(_) | Statement::Tql(_) | Statement::Delete(_) => {}
         // database ops won't be checked
         Statement::CreateDatabase(_) | Statement::ShowDatabases(_) | Statement::Use(_) => {}
         // show create table and alter are not supported yet
-        Statement::ShowCreateTable(_) | Statement::Alter(_) => {}
+        Statement::ShowCreateTable(_) | Statement::CreateExternalTable(_) | Statement::Alter(_) => {
+        }
 
         Statement::Insert(insert) => {
             validate_param(insert.table_name(), query_ctx)?;
@@ -673,9 +684,6 @@ pub fn check_permission(
         Statement::DescribeTable(stmt) => {
             validate_param(stmt.name(), query_ctx)?;
         }
-        Statement::Delete(delete) => {
-            validate_param(delete.table_name(), query_ctx)?;
-        }
         Statement::Copy(stmd) => match stmd {
             CopyTable::To(copy_table_to) => validate_param(&copy_table_to.table_name, query_ctx)?,
             CopyTable::From(copy_table_from) => {
@@ -1086,7 +1094,7 @@ mod tests {
                 .plan(stmt.clone(), QueryContext::arc())
                 .await
                 .unwrap();
-            let output = engine.execute(&plan).await.unwrap();
+            let output = engine.execute(plan, QueryContext::arc()).await.unwrap();
             let Output::Stream(stream) = output else { unreachable!() };
             let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
             let actual = recordbatches.pretty_print().unwrap();

src/frontend/src/instance/distributed.rs

@@ -33,6 +33,7 @@ use common_error::prelude::BoxedError;
 use common_query::Output;
 use common_telemetry::{debug, info};
 use datanode::instance::sql::table_idents_to_full_name;
+use datanode::sql::SqlHandler;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::RawSchema;
 use meta_client::client::MetaClient;
@@ -45,7 +46,7 @@ use partition::partition::{PartitionBound, PartitionDef};
 use query::error::QueryExecutionSnafu;
 use query::parser::QueryStatement;
 use query::query_engine::StatementHandler;
-use query::sql::{describe_table, show_databases, show_tables};
+use query::sql::{show_databases, show_tables};
 use session::context::QueryContextRef;
 use snafu::{ensure, OptionExt, ResultExt};
 use sql::ast::Value as SqlValue;
@@ -55,18 +56,18 @@ use sql::statements::statement::Statement;
 use table::metadata::{RawTableInfo, RawTableMeta, TableIdent, TableType};
 use table::requests::TableOptions;
 use table::table::AlterContext;
+use table::TableRef;
 
 use crate::catalog::FrontendCatalogManager;
 use crate::datanode::DatanodeClients;
 use crate::error::{
     self, AlterExprToRequestSnafu, CatalogEntrySerdeSnafu, CatalogSnafu, ColumnDataTypeSnafu,
-    DeserializePartitionSnafu, NotSupportedSnafu, ParseSqlSnafu, PrimaryKeyNotFoundSnafu,
-    RequestDatanodeSnafu, RequestMetaSnafu, Result, SchemaExistsSnafu, StartMetaClientSnafu,
-    TableAlreadyExistSnafu, TableNotFoundSnafu, TableSnafu, ToTableInsertRequestSnafu,
-    UnrecognizedTableOptionSnafu,
+    DeserializePartitionSnafu, InvokeDatanodeSnafu, NotSupportedSnafu, ParseSqlSnafu,
+    PrimaryKeyNotFoundSnafu, RequestDatanodeSnafu, RequestMetaSnafu, Result, SchemaExistsSnafu,
+    StartMetaClientSnafu, TableAlreadyExistSnafu, TableNotFoundSnafu, TableSnafu,
+    ToTableInsertRequestSnafu, UnrecognizedTableOptionSnafu,
 };
 use crate::expr_factory;
-use crate::sql::insert_to_request;
 use crate::table::DistTable;
 
 #[derive(Clone)]
@@ -93,14 +94,14 @@ impl DistInstance {
         &self,
         create_table: &mut CreateTableExpr,
         partitions: Option<Partitions>,
-    ) -> Result<Output> {
+    ) -> Result<TableRef> {
         let table_name = TableName::new(
             &create_table.catalog_name,
             &create_table.schema_name,
             &create_table.table_name,
         );
 
-        if self
+        if let Some(table) = self
             .catalog_manager
             .table(
                 &table_name.catalog_name,
@@ -109,10 +110,9 @@ impl DistInstance {
             )
             .await
             .context(CatalogSnafu)?
-            .is_some()
         {
             return if create_table.create_if_not_exists {
-                Ok(Output::AffectedRows(0))
+                Ok(table)
             } else {
                 TableAlreadyExistSnafu {
                     table: table_name.to_string(),
@@ -153,20 +153,20 @@ impl DistInstance {
 
         create_table.table_id = Some(TableId { id: table_id });
 
-        let table = DistTable::new(
+        let table = Arc::new(DistTable::new(
             table_name.clone(),
             table_info,
             self.catalog_manager.partition_manager(),
             self.catalog_manager.datanode_clients(),
             self.catalog_manager.backend(),
-        );
+        ));
 
         let request = RegisterTableRequest {
             catalog: table_name.catalog_name.clone(),
             schema: table_name.schema_name.clone(),
             table_name: table_name.table_name.clone(),
             table_id,
-            table: Arc::new(table),
+            table: table.clone(),
         };
         ensure!(
             self.catalog_manager
@@ -196,9 +196,7 @@ impl DistInstance {
                 .await
                 .context(RequestDatanodeSnafu)?;
         }
-
-        // Checked in real MySQL, it truly returns "0 rows affected".
-        Ok(Output::AffectedRows(0))
+        Ok(table)
     }
 
     async fn drop_table(&self, table_name: TableName) -> Result<Output> {
@@ -329,7 +327,8 @@ impl DistInstance {
             }
             Statement::CreateTable(stmt) => {
                 let create_expr = &mut expr_factory::create_to_expr(&stmt, query_ctx)?;
-                Ok(self.create_table(create_expr, stmt.partitions).await?)
+                let _ = self.create_table(create_expr, stmt.partitions).await?;
+                Ok(Output::AffectedRows(0))
             }
             Statement::Alter(alter_table) => {
                 let expr = grpc::to_alter_expr(alter_table, query_ctx)?;
@@ -347,20 +346,6 @@ impl DistInstance {
             Statement::ShowTables(stmt) => {
                 show_tables(stmt, self.catalog_manager.clone(), query_ctx)
             }
-            Statement::DescribeTable(stmt) => {
-                let (catalog, schema, table) = table_idents_to_full_name(stmt.name(), query_ctx)
-                    .map_err(BoxedError::new)
-                    .context(error::ExternalSnafu)?;
-                let table = self
-                    .catalog_manager
-                    .table(&catalog, &schema, &table)
-                    .await
-                    .context(CatalogSnafu)?
-                    .with_context(|| TableNotFoundSnafu {
-                        table_name: stmt.name().to_string(),
-                    })?;
-                describe_table(table)
-            }
             Statement::Insert(insert) => {
                 let (catalog, schema, table) =
                     table_idents_to_full_name(insert.table_name(), query_ctx.clone())
@@ -374,7 +359,10 @@ impl DistInstance {
                     .context(CatalogSnafu)?
                     .context(TableNotFoundSnafu { table_name: table })?;
 
-                let insert_request = insert_to_request(&table, *insert, query_ctx)?;
+                let insert_request =
+                    SqlHandler::insert_to_request(self.catalog_manager.clone(), *insert, query_ctx)
+                        .await
+                        .context(InvokeDatanodeSnafu)?;
 
                 return Ok(Output::AffectedRows(
                     table.insert(insert_request).await.context(TableSnafu)?,

src/frontend/src/instance/distributed/grpc.rs

@@ -49,7 +49,8 @@ impl GrpcQueryHandler for DistInstance {
                     DdlExpr::CreateTable(mut expr) => {
                         // TODO(LFC): Support creating distributed table through GRPC interface.
                         // Currently only SQL supports it; how to design the fields in CreateTableExpr?
-                        self.create_table(&mut expr, None).await
+                        let _ = self.create_table(&mut expr, None).await;
+                        Ok(Output::AffectedRows(0))
                     }
                     DdlExpr::Alter(expr) => self.handle_alter_table(expr).await,
                     DdlExpr::DropTable(expr) => {

src/frontend/src/instance/grpc.rs

@@ -383,8 +383,6 @@ CREATE TABLE {table_name} (
         // Wait for previous task finished
         flush_table(frontend, "greptime", "public", table_name, None).await;
 
-        let table_id = 1024;
-
         let table = instance
             .frontend
             .catalog_manager()
@@ -394,7 +392,7 @@ CREATE TABLE {table_name} (
             .unwrap();
         let table = table.as_any().downcast_ref::<DistTable>().unwrap();
 
-        let TableGlobalValue { regions_id_map, .. } = table
+        let tgv = table
             .table_global_value(&TableGlobalKey {
                 catalog_name: "greptime".to_string(),
                 schema_name: "public".to_string(),
@@ -403,7 +401,10 @@ CREATE TABLE {table_name} (
             .await
             .unwrap()
             .unwrap();
-        let region_to_dn_map = regions_id_map
+        let table_id = tgv.table_id();
+
+        let region_to_dn_map = tgv
+            .regions_id_map
             .iter()
             .map(|(k, v)| (v[0], *k))
             .collect::<HashMap<u32, u64>>();
@@ -590,7 +591,7 @@ CREATE TABLE {table_name} (
                 .plan(stmt, QueryContext::arc())
                 .await
                 .unwrap();
-            let output = engine.execute(&plan).await.unwrap();
+            let output = engine.execute(plan, QueryContext::arc()).await.unwrap();
             let Output::Stream(stream) = output else { unreachable!() };
             let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
             let actual = recordbatches.pretty_print().unwrap();

src/frontend/src/lib.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #![feature(assert_matches)]
+#![feature(trait_upcasting)]
 
 pub mod catalog;
 pub mod datanode;
@@ -22,13 +23,14 @@ pub mod frontend;
 pub mod grpc;
 pub mod influxdb;
 pub mod instance;
+pub(crate) mod metric;
 pub mod mysql;
 pub mod opentsdb;
 pub mod postgres;
 pub mod prom;
 pub mod prometheus;
+mod script;
 mod server;
-mod sql;
 mod table;
 #[cfg(test)]
 mod tests;

src/frontend/src/metric.rs

@@ -0,0 +1,17 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) const METRIC_HANDLE_SQL_ELAPSED: &str = "frontend.handle_sql_elapsed";
+pub(crate) const METRIC_HANDLE_SCRIPTS_ELAPSED: &str = "frontend.handle_scripts_elapsed";
+pub(crate) const METRIC_RUN_SCRIPT_ELAPSED: &str = "frontend.run_script_elapsed";

src/frontend/src/server.rs

@@ -22,7 +22,7 @@ use common_telemetry::info;
 use servers::auth::UserProviderRef;
 use servers::error::Error::InternalIo;
 use servers::grpc::GrpcServer;
-use servers::http::HttpServer;
+use servers::http::HttpServerBuilder;
 use servers::mysql::server::{MysqlServer, MysqlSpawnConfig, MysqlSpawnRef};
 use servers::opentsdb::OpentsdbServer;
 use servers::postgres::PostgresServer;
@@ -150,33 +150,33 @@ impl Services {
         if let Some(http_options) = &opts.http_options {
             let http_addr = parse_addr(&http_options.addr)?;
 
-            let mut http_server = HttpServer::new(
-                ServerSqlQueryHandlerAdaptor::arc(instance.clone()),
-                ServerGrpcQueryHandlerAdaptor::arc(instance.clone()),
-                http_options.clone(),
-            );
+            let mut http_server_builder = HttpServerBuilder::new(http_options.clone());
+            http_server_builder
+                .with_sql_handler(ServerSqlQueryHandlerAdaptor::arc(instance.clone()))
+                .with_grpc_handler(ServerGrpcQueryHandlerAdaptor::arc(instance.clone()));
+
             if let Some(user_provider) = user_provider.clone() {
-                http_server.set_user_provider(user_provider);
+                http_server_builder.with_user_provider(user_provider);
             }
 
             if set_opentsdb_handler {
-                http_server.set_opentsdb_handler(instance.clone());
+                http_server_builder.with_opentsdb_handler(instance.clone());
             }
             if matches!(
                 opts.influxdb_options,
                 Some(InfluxdbOptions { enable: true })
             ) {
-                http_server.set_influxdb_handler(instance.clone());
+                http_server_builder.with_influxdb_handler(instance.clone());
             }
 
             if matches!(
                 opts.prometheus_options,
                 Some(PrometheusOptions { enable: true })
             ) {
-                http_server.set_prom_handler(instance.clone());
+                http_server_builder.with_prom_handler(instance.clone());
             }
-            http_server.set_script_handler(instance.clone());
-
+            http_server_builder.with_script_handler(instance.clone());
+            let http_server = http_server_builder.build();
             result.push((Box::new(http_server), http_addr));
         }
 

src/frontend/src/sql.rs

@@ -1,130 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_error::ext::BoxedError;
-use common_error::snafu::ensure;
-use datanode::instance::sql::table_idents_to_full_name;
-use datatypes::data_type::DataType;
-use datatypes::prelude::MutableVector;
-use datatypes::schema::ColumnSchema;
-use session::context::QueryContextRef;
-use snafu::{OptionExt, ResultExt};
-use sql::ast::Value as SqlValue;
-use sql::statements;
-use sql::statements::insert::Insert;
-use table::requests::InsertRequest;
-use table::TableRef;
-
-use crate::error::{self, ExternalSnafu, Result};
-
-const DEFAULT_PLACEHOLDER_VALUE: &str = "default";
-
-// TODO(fys): Extract the common logic in datanode and frontend in the future.
-// This function convert insert statement to an `InsertRequest` to region 0.
-pub(crate) fn insert_to_request(
-    table: &TableRef,
-    stmt: Insert,
-    query_ctx: QueryContextRef,
-) -> Result<InsertRequest> {
-    let columns = stmt.columns();
-    let values = stmt
-        .values_body()
-        .context(error::ParseSqlSnafu)?
-        .context(error::MissingInsertValuesSnafu)?;
-
-    let (catalog_name, schema_name, table_name) =
-        table_idents_to_full_name(stmt.table_name(), query_ctx)
-            .map_err(BoxedError::new)
-            .context(ExternalSnafu)?;
-
-    let schema = table.schema();
-    let columns_num = if columns.is_empty() {
-        schema.column_schemas().len()
-    } else {
-        columns.len()
-    };
-    let rows_num = values.len();
-
-    let mut columns_builders: Vec<(&ColumnSchema, Box<dyn MutableVector>)> =
-        Vec::with_capacity(columns_num);
-
-    if columns.is_empty() {
-        for column_schema in schema.column_schemas() {
-            let data_type = &column_schema.data_type;
-            columns_builders.push((column_schema, data_type.create_mutable_vector(rows_num)));
-        }
-    } else {
-        for column_name in columns {
-            let column_schema = schema.column_schema_by_name(column_name).with_context(|| {
-                error::ColumnNotFoundSnafu {
-                    table_name: &table_name,
-                    column_name: column_name.to_string(),
-                }
-            })?;
-            let data_type = &column_schema.data_type;
-            columns_builders.push((column_schema, data_type.create_mutable_vector(rows_num)));
-        }
-    }
-
-    for row in values {
-        ensure!(
-            row.len() == columns_num,
-            error::ColumnValuesNumberMismatchSnafu {
-                columns: columns_num,
-                values: row.len(),
-            }
-        );
-
-        for (sql_val, (column_schema, builder)) in row.iter().zip(columns_builders.iter_mut()) {
-            add_row_to_vector(column_schema, sql_val, builder)?;
-        }
-    }
-
-    Ok(InsertRequest {
-        catalog_name,
-        schema_name,
-        table_name,
-        columns_values: columns_builders
-            .into_iter()
-            .map(|(cs, mut b)| (cs.name.to_string(), b.to_vector()))
-            .collect(),
-        region_number: 0,
-    })
-}
-
-fn add_row_to_vector(
-    column_schema: &ColumnSchema,
-    sql_val: &SqlValue,
-    builder: &mut Box<dyn MutableVector>,
-) -> Result<()> {
-    let value = if replace_default(sql_val) {
-        column_schema
-            .create_default()
-            .context(error::ColumnDefaultValueSnafu {
-                column: column_schema.name.to_string(),
-            })?
-            .context(error::ColumnNoneDefaultValueSnafu {
-                column: column_schema.name.to_string(),
-            })?
-    } else {
-        statements::sql_value_to_value(&column_schema.name, &column_schema.data_type, sql_val)
-            .context(error::ParseSqlSnafu)?
-    };
-    builder.push_value_ref(value.as_value_ref());
-    Ok(())
-}
-
-fn replace_default(sql_val: &SqlValue) -> bool {
-    matches!(sql_val, SqlValue::Placeholder(s) if s.to_lowercase() == DEFAULT_PLACEHOLDER_VALUE)
-}

src/frontend/src/tests.rs

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod instance_test;
+mod test_util;
+
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
@@ -21,7 +24,9 @@ use client::Client;
 use common_grpc::channel_manager::ChannelManager;
 use common_runtime::Builder as RuntimeBuilder;
 use common_test_util::temp_dir::{create_temp_dir, TempDir};
-use datanode::datanode::{DatanodeOptions, FileConfig, ObjectStoreConfig, WalConfig};
+use datanode::datanode::{
+    DatanodeOptions, FileConfig, ObjectStoreConfig, StorageConfig, WalConfig,
+};
 use datanode::instance::Instance as DatanodeInstance;
 use meta_client::client::MetaClientBuilder;
 use meta_client::rpc::Peer;
@@ -54,6 +59,7 @@ pub(crate) struct MockDistributedInstance {
     pub(crate) frontend: Arc<Instance>,
     pub(crate) dist_instance: Arc<DistInstance>,
     pub(crate) datanodes: HashMap<u64, Arc<DatanodeInstance>>,
+    pub(crate) catalog_manager: Arc<FrontendCatalogManager>,
     _guards: Vec<TestGuard>,
 }
 
@@ -76,11 +82,11 @@ impl MockStandaloneInstance {
 
 pub(crate) async fn create_standalone_instance(test_name: &str) -> MockStandaloneInstance {
     let (opts, guard) = create_tmp_dir_and_datanode_opts(test_name);
-    let datanode_instance = DatanodeInstance::new(&opts).await.unwrap();
-    datanode_instance.start().await.unwrap();
-
-    let frontend_instance = Instance::new_standalone(Arc::new(datanode_instance));
-
+    let dn_instance = Arc::new(DatanodeInstance::new(&opts).await.unwrap());
+    let frontend_instance = Instance::try_new_standalone(dn_instance.clone())
+        .await
+        .unwrap();
+    dn_instance.start().await.unwrap();
     MockStandaloneInstance {
         instance: Arc::new(frontend_instance),
         _guard: guard,
@@ -95,9 +101,12 @@ fn create_tmp_dir_and_datanode_opts(name: &str) -> (DatanodeOptions, TestGuard)
             dir: wal_tmp_dir.path().to_str().unwrap().to_string(),
             ..Default::default()
         },
-        storage: ObjectStoreConfig::File(FileConfig {
-            data_dir: data_tmp_dir.path().to_str().unwrap().to_string(),
-        }),
+        storage: StorageConfig {
+            store: ObjectStoreConfig::File(FileConfig {
+                data_dir: data_tmp_dir.path().to_str().unwrap().to_string(),
+            }),
+            ..Default::default()
+        },
         mode: Mode::Standalone,
         ..Default::default()
     };
@@ -182,9 +191,12 @@ async fn create_distributed_datanode(
             dir: wal_tmp_dir.path().to_str().unwrap().to_string(),
             ..Default::default()
         },
-        storage: ObjectStoreConfig::File(FileConfig {
-            data_dir: data_tmp_dir.path().to_str().unwrap().to_string(),
-        }),
+        storage: StorageConfig {
+            store: ObjectStoreConfig::File(FileConfig {
+                data_dir: data_tmp_dir.path().to_str().unwrap().to_string(),
+            }),
+            ..Default::default()
+        },
         mode: Mode::Distributed,
         ..Default::default()
     };
@@ -259,26 +271,28 @@ pub(crate) async fn create_distributed_instance(test_name: &str) -> MockDistribu
     let partition_manager = Arc::new(PartitionRuleManager::new(Arc::new(TableRoutes::new(
         meta_client.clone(),
     ))));
-    let catalog_manager = Arc::new(FrontendCatalogManager::new(
-        meta_backend,
-        partition_manager,
-        datanode_clients.clone(),
-    ));
+    let mut catalog_manager =
+        FrontendCatalogManager::new(meta_backend, partition_manager, datanode_clients.clone());
 
     wait_datanodes_alive(kv_store).await;
 
     let dist_instance = DistInstance::new(
         meta_client.clone(),
-        catalog_manager,
+        Arc::new(catalog_manager.clone()),
         datanode_clients.clone(),
     );
     let dist_instance = Arc::new(dist_instance);
-    let frontend = Instance::new_distributed(dist_instance.clone());
+
+    catalog_manager.set_dist_instance(dist_instance.clone());
+    let catalog_manager = Arc::new(catalog_manager);
+
+    let frontend = Instance::new_distributed(catalog_manager.clone(), dist_instance.clone()).await;
 
     MockDistributedInstance {
         frontend: Arc::new(frontend),
         dist_instance,
         datanodes: datanode_instances,
+        catalog_manager,
         _guards: test_guards,
     }
 }

src/frontend/src/tests/instance_test.rs

@@ -19,19 +19,17 @@ use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_query::Output;
 use common_recordbatch::util;
 use common_telemetry::logging;
-use datatypes::data_type::ConcreteDataType;
 use datatypes::vectors::{Int64Vector, StringVector, UInt64Vector, VectorRef};
-use query::parser::{QueryLanguageParser, QueryStatement};
+use servers::query_handler::sql::SqlQueryHandler;
 use session::context::QueryContext;
-use snafu::ResultExt;
-use sql::statements::statement::Statement;
 
-use crate::error::{Error, ExecuteLogicalPlanSnafu, PlanStatementSnafu};
-use crate::tests::test_util::{self, check_output_stream, setup_test_instance, MockInstance};
+use crate::error::Error;
+use crate::tests::test_util::check_output_stream;
+use crate::tests::{create_standalone_instance, MockStandaloneInstance};
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_create_database_and_insert_query() {
-    let instance = MockInstance::new("create_database_and_insert_query").await;
+    let instance = create_standalone_instance("create_database_and_insert_query").await;
 
     let output = execute_sql(&instance, "create database test").await;
     assert!(matches!(output, Output::AffectedRows(1)));
@@ -78,7 +76,8 @@ async fn test_create_database_and_insert_query() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_issue477_same_table_name_in_different_databases() {
-    let instance = MockInstance::new("test_issue477_same_table_name_in_different_databases").await;
+    let instance =
+        create_standalone_instance("test_issue477_same_table_name_in_different_databases").await;
 
     // Create database a and b
     let output = execute_sql(&instance, "create database a").await;
@@ -145,7 +144,7 @@ async fn test_issue477_same_table_name_in_different_databases() {
     .await;
 }
 
-async fn assert_query_result(instance: &MockInstance, sql: &str, ts: i64, host: &str) {
+async fn assert_query_result(instance: &MockStandaloneInstance, sql: &str, ts: i64, host: &str) {
     let query_output = execute_sql(instance, sql).await;
     match query_output {
         Output::Stream(s) => {
@@ -167,7 +166,7 @@ async fn assert_query_result(instance: &MockInstance, sql: &str, ts: i64, host:
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_execute_insert() {
-    let instance = setup_test_instance("test_execute_insert").await;
+    let instance = create_standalone_instance("test_execute_insert").await;
 
     // create table
     execute_sql(
@@ -189,7 +188,7 @@ async fn test_execute_insert() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_execute_insert_by_select() {
-    let instance = setup_test_instance("test_execute_insert_by_select").await;
+    let instance = create_standalone_instance("test_execute_insert_by_select").await;
 
     // create table
     execute_sql(
@@ -217,20 +216,20 @@ async fn test_execute_insert_by_select() {
         try_execute_sql(&instance, "insert into demo2(host) select * from demo1")
             .await
             .unwrap_err(),
-        Error::ColumnValuesNumberMismatch { .. }
+        Error::PlanStatement { .. }
     ));
     assert!(matches!(
         try_execute_sql(&instance, "insert into demo2 select cpu,memory from demo1")
             .await
             .unwrap_err(),
-        Error::ColumnValuesNumberMismatch { .. }
+        Error::PlanStatement { .. }
     ));
 
     assert!(matches!(
         try_execute_sql(&instance, "insert into demo2(ts) select memory from demo1")
             .await
             .unwrap_err(),
-        Error::ColumnTypeMismatch { .. }
+        Error::PlanStatement { .. }
     ));
 
     let output = execute_sql(&instance, "insert into demo2 select * from demo1").await;
@@ -250,11 +249,12 @@ async fn test_execute_insert_by_select() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_execute_insert_query_with_i64_timestamp() {
-    let instance = MockInstance::new("insert_query_i64_timestamp").await;
+    let instance = create_standalone_instance("insert_query_i64_timestamp").await;
 
-    test_util::create_test_table(instance.inner(), ConcreteDataType::int64_datatype())
-        .await
-        .unwrap();
+    execute_sql(
+        &instance,
+        "create table demo(host string, cpu double, memory double, ts bigint time index, primary key (host));",
+    ).await;
 
     let output = execute_sql(
         &instance,
@@ -301,7 +301,7 @@ async fn test_execute_insert_query_with_i64_timestamp() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_execute_query() {
-    let instance = MockInstance::new("execute_query").await;
+    let instance = create_standalone_instance("execute_query").await;
 
     let output = execute_sql(&instance, "select sum(number) from numbers limit 20").await;
     match output {
@@ -321,7 +321,7 @@ async fn test_execute_query() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_execute_show_databases_tables() {
-    let instance = MockInstance::new("execute_show_databases_tables").await;
+    let instance = create_standalone_instance("execute_show_databases_tables").await;
 
     let output = execute_sql(&instance, "show databases").await;
     match output {
@@ -363,13 +363,10 @@ async fn test_execute_show_databases_tables() {
         _ => unreachable!(),
     }
 
-    // creat a table
-    test_util::create_test_table(
-        instance.inner(),
-        ConcreteDataType::timestamp_millisecond_datatype(),
-    )
-    .await
-    .unwrap();
+    execute_sql(
+        &instance,
+        "create table demo(host string, cpu double, memory double, ts timestamp time index, primary key (host));",
+    ).await;
 
     let output = execute_sql(&instance, "show tables").await;
     match output {
@@ -400,7 +397,7 @@ async fn test_execute_show_databases_tables() {
 
 #[tokio::test(flavor = "multi_thread")]
 pub async fn test_execute_create() {
-    let instance = MockInstance::new("execute_create").await;
+    let instance = create_standalone_instance("execute_create").await;
 
     let output = execute_sql(
         &instance,
@@ -419,7 +416,7 @@ pub async fn test_execute_create() {
 
 #[tokio::test]
 async fn test_rename_table() {
-    let instance = MockInstance::new("test_rename_table_local").await;
+    let instance = create_standalone_instance("test_rename_table_local").await;
 
     let output = execute_sql(&instance, "create database db").await;
     assert!(matches!(output, Output::AffectedRows(1)));
@@ -475,7 +472,7 @@ async fn test_rename_table() {
 
 #[tokio::test]
 async fn test_create_table_after_rename_table() {
-    let instance = MockInstance::new("test_rename_table_local").await;
+    let instance = create_standalone_instance("test_rename_table_local").await;
 
     let output = execute_sql(&instance, "create database db").await;
     assert!(matches!(output, Output::AffectedRows(1)));
@@ -525,7 +522,7 @@ async fn test_create_table_after_rename_table() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_alter_table() {
-    let instance = setup_test_instance("test_alter_table").await;
+    let instance = create_standalone_instance("test_alter_table").await;
 
     // create table
     execute_sql(
@@ -612,7 +609,7 @@ async fn test_alter_table() {
 }
 
 async fn test_insert_with_default_value_for_type(type_name: &str) {
-    let instance = MockInstance::new("execute_create").await;
+    let instance = create_standalone_instance("execute_create").await;
 
     let create_sql = format!(
         r#"create table test_table(
@@ -663,7 +660,7 @@ async fn test_insert_with_default_value() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_use_database() {
-    let instance = MockInstance::new("test_use_database").await;
+    let instance = create_standalone_instance("test_use_database").await;
 
     let output = execute_sql(&instance, "create database db1").await;
     assert!(matches!(output, Output::AffectedRows(1)));
@@ -722,7 +719,7 @@ async fn test_use_database() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn test_delete() {
-    let instance = MockInstance::new("test_delete").await;
+    let instance = create_standalone_instance("test_delete").await;
 
     let output = execute_sql(
         &instance,
@@ -751,7 +748,7 @@ async fn test_delete() {
 
     let output = execute_sql(
         &instance,
-        "delete from test_table where host = host1 and ts = 1655276557000 ",
+        "delete from test_table where host = 'host1' and ts = 1655276557000 ",
     )
     .await;
     assert!(matches!(output, Output::AffectedRows(1)));
@@ -774,7 +771,7 @@ async fn test_execute_copy_to_s3() {
     logging::init_default_ut_logging();
     if let Ok(bucket) = env::var("GT_S3_BUCKET") {
         if !bucket.is_empty() {
-            let instance = setup_test_instance("test_execute_copy_to_s3").await;
+            let instance = create_standalone_instance("test_execute_copy_to_s3").await;
 
             // setups
             execute_sql(
@@ -813,7 +810,7 @@ async fn test_execute_copy_from_s3() {
     logging::init_default_ut_logging();
     if let Ok(bucket) = env::var("GT_S3_BUCKET") {
         if !bucket.is_empty() {
-            let instance = setup_test_instance("test_execute_copy_from_s3").await;
+            let instance = create_standalone_instance("test_execute_copy_from_s3").await;
 
             // setups
             execute_sql(
@@ -852,7 +849,7 @@ async fn test_execute_copy_from_s3() {
             let tests = [
                 Test {
                     sql: &format!(
-                        "Copy with_filename FROM 's3://{}/{}/export/demo.parquet_1_2'",
+                        "Copy with_filename FROM 's3://{}/{}/export/demo.parquet'",
                         bucket, root
                     ),
                     table_name: "with_filename",
@@ -908,75 +905,26 @@ async fn test_execute_copy_from_s3() {
     }
 }
 
-#[tokio::test(flavor = "multi_thread")]
-async fn test_create_by_procedure() {
-    common_telemetry::init_default_ut_logging();
-
-    let instance = MockInstance::with_procedure_enabled("create_by_procedure").await;
-
-    let output = execute_sql(
-        &instance,
-        r#"create table test_table(
-                            host string,
-                            ts timestamp,
-                            cpu double default 0,
-                            memory double,
-                            TIME INDEX (ts),
-                            PRIMARY KEY(host)
-                        ) engine=mito with(regions=1);"#,
-    )
-    .await;
-    assert!(matches!(output, Output::AffectedRows(0)));
-
-    // Create if not exists
-    let output = execute_sql(
-        &instance,
-        r#"create table if not exists test_table(
-                            host string,
-                            ts timestamp,
-                            cpu double default 0,
-                            memory double,
-                            TIME INDEX (ts),
-                            PRIMARY KEY(host)
-                        ) engine=mito with(regions=1);"#,
-    )
-    .await;
-    assert!(matches!(output, Output::AffectedRows(0)));
-}
-
-async fn execute_sql(instance: &MockInstance, sql: &str) -> Output {
+async fn execute_sql(instance: &MockStandaloneInstance, sql: &str) -> Output {
     execute_sql_in_db(instance, sql, DEFAULT_SCHEMA_NAME).await
 }
 
 async fn try_execute_sql(
-    instance: &MockInstance,
+    instance: &MockStandaloneInstance,
     sql: &str,
 ) -> Result<Output, crate::error::Error> {
     try_execute_sql_in_db(instance, sql, DEFAULT_SCHEMA_NAME).await
 }
 
 async fn try_execute_sql_in_db(
-    instance: &MockInstance,
+    instance: &MockStandaloneInstance,
     sql: &str,
     db: &str,
 ) -> Result<Output, crate::error::Error> {
     let query_ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db));
-
-    let stmt = QueryLanguageParser::parse_sql(sql).unwrap();
-    match stmt {
-        QueryStatement::Sql(Statement::Query(_)) => {
-            let engine = instance.inner().query_engine();
-            let plan = engine
-                .planner()
-                .plan(stmt, query_ctx)
-                .await
-                .context(PlanStatementSnafu)?;
-            engine.execute(&plan).await.context(ExecuteLogicalPlanSnafu)
-        }
-        _ => instance.inner().execute_stmt(stmt, query_ctx).await,
-    }
+    instance.instance.do_query(sql, query_ctx).await.remove(0)
 }
 
-async fn execute_sql_in_db(instance: &MockInstance, sql: &str, db: &str) -> Output {
+async fn execute_sql_in_db(instance: &MockStandaloneInstance, sql: &str, db: &str) -> Output {
     try_execute_sql_in_db(instance, sql, db).await.unwrap()
 }

src/frontend/src/tests/test_util.rs

@@ -0,0 +1,26 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_query::Output;
+use common_recordbatch::util;
+
+pub(crate) async fn check_output_stream(output: Output, expected: String) {
+    let recordbatches = match output {
+        Output::Stream(stream) => util::collect_batches(stream).await.unwrap(),
+        Output::RecordBatches(recordbatches) => recordbatches,
+        _ => unreachable!(),
+    };
+    let pretty_print = recordbatches.pretty_print().unwrap();
+    assert_eq!(pretty_print, expected, "{}", pretty_print);
+}

src/log-store/src/error.rs

@@ -15,16 +15,22 @@
 use std::any::Any;
 
 use common_error::prelude::{ErrorExt, Snafu};
+use common_runtime::error::Error as RuntimeError;
 use snafu::{Backtrace, ErrorCompat};
-use tokio::task::JoinError;
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
-    #[snafu(display("Failed to wait for gc task to stop, source: {}", source))]
-    WaitGcTaskStop {
-        source: JoinError,
-        backtrace: Backtrace,
+    #[snafu(display("Failed to start log store gc task, source: {}", source))]
+    StartGcTask {
+        #[snafu(backtrace)]
+        source: RuntimeError,
+    },
+
+    #[snafu(display("Failed to stop log store gc task, source: {}", source))]
+    StopGcTask {
+        #[snafu(backtrace)]
+        source: RuntimeError,
     },
 
     #[snafu(display("Failed to add entry to LogBatch, source: {}", source))]

src/log-store/src/raft_engine/log_store.rs

@@ -13,25 +13,22 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
-use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
 use async_stream::stream;
+use common_runtime::{RepeatedTask, TaskFunction};
 use common_telemetry::{error, info};
 use raft_engine::{Config, Engine, LogBatch, MessageExt, ReadableSize, RecoveryMode};
-use snafu::{ensure, OptionExt, ResultExt};
+use snafu::{ensure, ResultExt};
 use store_api::logstore::entry::Id;
 use store_api::logstore::entry_stream::SendableEntryStream;
 use store_api::logstore::namespace::Namespace as NamespaceTrait;
 use store_api::logstore::{AppendResponse, LogStore};
-use tokio::sync::Mutex;
-use tokio::task::JoinHandle;
-use tokio_util::sync::CancellationToken;
 
 use crate::config::LogConfig;
 use crate::error::{
     AddEntryLogBatchSnafu, Error, FetchEntrySnafu, IllegalNamespaceSnafu, IllegalStateSnafu,
-    RaftEngineSnafu, WaitGcTaskStopSnafu,
+    RaftEngineSnafu, StartGcTaskSnafu, StopGcTaskSnafu,
 };
 use crate::raft_engine::protos::logstore::{EntryImpl as Entry, NamespaceImpl as Namespace};
 
@@ -41,9 +38,36 @@ const SYSTEM_NAMESPACE: u64 = 0;
 pub struct RaftEngineLogStore {
     config: LogConfig,
     engine: Arc<Engine>,
-    cancel_token: Mutex<Option<CancellationToken>>,
-    gc_task_handle: Mutex<Option<JoinHandle<()>>>,
-    started: AtomicBool,
+    gc_task: RepeatedTask<Error>,
+}
+
+pub struct PurgeExpiredFilesFunction {
+    engine: Arc<Engine>,
+}
+
+#[async_trait::async_trait]
+impl TaskFunction<Error> for PurgeExpiredFilesFunction {
+    fn name(&self) -> &str {
+        "RaftEngineLogStore-gc-task"
+    }
+
+    async fn call(&self) -> Result<(), Error> {
+        match self.engine.purge_expired_files().context(RaftEngineSnafu) {
+            Ok(res) => {
+                // TODO(hl): the retval of purge_expired_files indicates the namespaces need to be compact,
+                // which is useful when monitoring regions failed to flush it's memtable to SSTs.
+                info!(
+                    "Successfully purged logstore files, namespaces need compaction: {:?}",
+                    res
+                );
+            }
+            Err(e) => {
+                error!(e; "Failed to purge files in logstore");
+            }
+        }
+
+        Ok(())
+    }
 }
 
 impl RaftEngineLogStore {
@@ -58,56 +82,31 @@ impl RaftEngineLogStore {
             ..Default::default()
         };
         let engine = Arc::new(Engine::open(raft_engine_config).context(RaftEngineSnafu)?);
+        let gc_task = RepeatedTask::new(
+            config.purge_interval,
+            Arc::new(PurgeExpiredFilesFunction {
+                engine: engine.clone(),
+            }),
+        );
+
         let log_store = Self {
             config,
             engine,
-            cancel_token: Mutex::new(None),
-            gc_task_handle: Mutex::new(None),
-            started: AtomicBool::new(false),
+            gc_task,
         };
         log_store.start().await?;
         Ok(log_store)
     }
 
     pub fn started(&self) -> bool {
-        self.started.load(Ordering::Relaxed)
+        self.gc_task.started()
     }
 
     async fn start(&self) -> Result<(), Error> {
-        let engine_clone = self.engine.clone();
-        let interval = self.config.purge_interval;
-        let token = CancellationToken::new();
-        let child = token.child_token();
-        // TODO(hl): Maybe spawn to a blocking runtime.
-        let handle = common_runtime::spawn_bg(async move {
-            loop {
-                tokio::select! {
-                    _ = tokio::time::sleep(interval) => {}
-                    _ = child.cancelled() => {
-                        info!("LogStore gc task has been cancelled");
-                        return;
-                    }
-                }
-                match engine_clone.purge_expired_files().context(RaftEngineSnafu) {
-                    Ok(res) => {
-                        // TODO(hl): the retval of purge_expired_files indicates the namespaces need to be compact,
-                        // which is useful when monitoring regions failed to flush it's memtable to SSTs.
-                        info!(
-                            "Successfully purged logstore files, namespaces need compaction: {:?}",
-                            res
-                        );
-                    }
-                    Err(e) => {
-                        error!(e; "Failed to purge files in logstore");
-                    }
-                }
-            }
-        });
-        *self.cancel_token.lock().await = Some(token);
-        *self.gc_task_handle.lock().await = Some(handle);
-        self.started.store(true, Ordering::Relaxed);
-        info!("RaftEngineLogStore started with config: {:?}", self.config);
-        Ok(())
+        self.gc_task
+            .start(common_runtime::bg_runtime())
+            .await
+            .context(StartGcTaskSnafu)
     }
 }
 
@@ -115,7 +114,7 @@ impl Debug for RaftEngineLogStore {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("RaftEngineLogsStore")
             .field("config", &self.config)
-            .field("started", &self.started.load(Ordering::Relaxed))
+            .field("started", &self.gc_task.started())
             .finish()
     }
 }
@@ -127,28 +126,7 @@ impl LogStore for RaftEngineLogStore {
     type Entry = Entry;
 
     async fn stop(&self) -> Result<(), Self::Error> {
-        ensure!(
-            self.started
-                .compare_exchange(true, false, Ordering::Relaxed, Ordering::Relaxed)
-                .is_ok(),
-            IllegalStateSnafu
-        );
-        let handle = self
-            .gc_task_handle
-            .lock()
-            .await
-            .take()
-            .context(IllegalStateSnafu)?;
-        let token = self
-            .cancel_token
-            .lock()
-            .await
-            .take()
-            .context(IllegalStateSnafu)?;
-        token.cancel();
-        handle.await.context(WaitGcTaskStopSnafu)?;
-        info!("RaftEngineLogStore stopped");
-        Ok(())
+        self.gc_task.stop().await.context(StopGcTaskSnafu)
     }
 
     /// Append an entry to logstore. Currently of existence of entry's namespace is not checked.

src/meta-client/examples/meta_client.rs

@@ -22,8 +22,8 @@ use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use meta_client::client::MetaClientBuilder;
 use meta_client::rpc::{
-    BatchPutRequest, CompareAndPutRequest, CreateRequest, DeleteRangeRequest, Partition,
-    PutRequest, RangeRequest, TableName,
+    BatchDeleteRequest, BatchGetRequest, BatchPutRequest, CompareAndPutRequest, CreateRequest,
+    DeleteRangeRequest, Partition, PutRequest, RangeRequest, TableName,
 };
 use table::metadata::{RawTableInfo, RawTableMeta, TableIdent, TableType};
 use table::requests::TableOptions;
@@ -146,6 +146,30 @@ async fn run() {
     // get none
     let res = meta_client.range(range).await.unwrap();
     event!(Level::INFO, "get range result: {:#?}", res);
+
+    // batch delete
+    // put two
+    let batch_put = BatchPutRequest::new()
+        .add_kv(b"batch_put1".to_vec(), b"batch_put_v1".to_vec())
+        .add_kv(b"batch_put2".to_vec(), b"batch_put_v2".to_vec())
+        .with_prev_kv();
+    let res = meta_client.batch_put(batch_put).await.unwrap();
+    event!(Level::INFO, "batch put result: {:#?}", res);
+
+    // delete one
+    let batch_delete = BatchDeleteRequest::new()
+        .add_key(b"batch_put1".to_vec())
+        .with_prev_kv();
+    let res = meta_client.batch_delete(batch_delete).await.unwrap();
+    event!(Level::INFO, "batch delete result: {:#?}", res);
+
+    // get other one
+    let batch_get = BatchGetRequest::new()
+        .add_key(b"batch_put1".to_vec())
+        .add_key(b"batch_put2".to_vec());
+
+    let res = meta_client.batch_get(batch_get).await.unwrap();
+    event!(Level::INFO, "batch get result: {:#?}", res);
 }
 
 fn new_table_info() -> RawTableInfo {

src/meta-client/src/client.rs

@@ -32,10 +32,10 @@ use crate::error::Result;
 use crate::rpc::lock::{LockRequest, LockResponse, UnlockRequest};
 use crate::rpc::router::DeleteRequest;
 use crate::rpc::{
-    BatchGetRequest, BatchGetResponse, BatchPutRequest, BatchPutResponse, CompareAndPutRequest,
-    CompareAndPutResponse, CreateRequest, DeleteRangeRequest, DeleteRangeResponse,
-    MoveValueRequest, MoveValueResponse, PutRequest, PutResponse, RangeRequest, RangeResponse,
-    RouteRequest, RouteResponse,
+    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
+    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, CreateRequest,
+    DeleteRangeRequest, DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest,
+    PutResponse, RangeRequest, RangeResponse, RouteRequest, RouteResponse,
 };
 
 pub type Id = (u64, u64);
@@ -256,6 +256,14 @@ impl MetaClient {
         self.store_client()?.batch_put(req.into()).await?.try_into()
     }
 
+    /// BatchDelete atomically deletes the given keys from the key-value store.
+    pub async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
+        self.store_client()?
+            .batch_delete(req.into())
+            .await?
+            .try_into()
+    }
+
     /// CompareAndPut atomically puts the value to the given updated
     /// value if the current value == the expected value.
     pub async fn compare_and_put(

src/meta-client/src/client/store.rs

@@ -17,9 +17,10 @@ use std::sync::Arc;
 
 use api::v1::meta::store_client::StoreClient;
 use api::v1::meta::{
-    BatchGetRequest, BatchGetResponse, BatchPutRequest, BatchPutResponse, CompareAndPutRequest,
-    CompareAndPutResponse, DeleteRangeRequest, DeleteRangeResponse, MoveValueRequest,
-    MoveValueResponse, PutRequest, PutResponse, RangeRequest, RangeResponse,
+    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
+    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
+    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+    RangeRequest, RangeResponse,
 };
 use common_grpc::channel_manager::ChannelManager;
 use snafu::{ensure, OptionExt, ResultExt};
@@ -80,6 +81,11 @@ impl Client {
         inner.batch_put(req).await
     }
 
+    pub async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
+        let inner = self.inner.read().await;
+        inner.batch_delete(req).await
+    }
+
     pub async fn compare_and_put(
         &self,
         req: CompareAndPutRequest,
@@ -169,6 +175,17 @@ impl Inner {
         Ok(res.into_inner())
     }
 
+    async fn batch_delete(&self, mut req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
+        let mut client = self.random_client()?;
+        req.set_header(self.id);
+        let res = client
+            .batch_delete(req)
+            .await
+            .context(error::TonicStatusSnafu)?;
+
+        Ok(res.into_inner())
+    }
+
     async fn compare_and_put(
         &self,
         mut req: CompareAndPutRequest,

src/meta-client/src/rpc.rs

@@ -28,9 +28,10 @@ pub use router::{
 };
 use serde::{Deserialize, Serialize};
 pub use store::{
-    BatchGetRequest, BatchGetResponse, BatchPutRequest, BatchPutResponse, CompareAndPutRequest,
-    CompareAndPutResponse, DeleteRangeRequest, DeleteRangeResponse, MoveValueRequest,
-    MoveValueResponse, PutRequest, PutResponse, RangeRequest, RangeResponse,
+    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
+    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
+    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+    RangeRequest, RangeResponse,
 };
 
 #[derive(Debug, Clone)]

src/meta-client/src/rpc/store.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use api::v1::meta::{
+    BatchDeleteRequest as PbBatchDeleteRequest, BatchDeleteResponse as PbBatchDeleteResponse,
     BatchGetRequest as PbBatchGetRequest, BatchGetResponse as PbBatchGetResponse,
     BatchPutRequest as PbBatchPutRequest, BatchPutResponse as PbBatchPutResponse,
     CompareAndPutRequest as PbCompareAndPutRequest,
@@ -377,6 +378,78 @@ impl BatchPutResponse {
     }
 }
 
+#[derive(Debug, Clone, Default)]
+pub struct BatchDeleteRequest {
+    pub keys: Vec<Vec<u8>>,
+    /// If prev_kv is set, gets the previous key-value pairs before deleting it.
+    /// The previous key-value pairs will be returned in the batch delete response.
+    pub prev_kv: bool,
+}
+
+impl From<BatchDeleteRequest> for PbBatchDeleteRequest {
+    fn from(req: BatchDeleteRequest) -> Self {
+        Self {
+            header: None,
+            keys: req.keys,
+            prev_kv: req.prev_kv,
+        }
+    }
+}
+
+impl BatchDeleteRequest {
+    #[inline]
+    pub fn new() -> Self {
+        Self {
+            keys: vec![],
+            prev_kv: false,
+        }
+    }
+
+    #[inline]
+    pub fn add_key(mut self, key: impl Into<Vec<u8>>) -> Self {
+        self.keys.push(key.into());
+        self
+    }
+
+    /// If prev_kv is set, gets the previous key-value pair before deleting it.
+    /// The previous key-value pair will be returned in the batch delete response.
+    #[inline]
+    pub fn with_prev_kv(mut self) -> Self {
+        self.prev_kv = true;
+        self
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct BatchDeleteResponse(PbBatchDeleteResponse);
+
+impl TryFrom<PbBatchDeleteResponse> for BatchDeleteResponse {
+    type Error = error::Error;
+
+    fn try_from(pb: PbBatchDeleteResponse) -> Result<Self> {
+        util::check_response_header(pb.header.as_ref())?;
+
+        Ok(Self::new(pb))
+    }
+}
+
+impl BatchDeleteResponse {
+    #[inline]
+    pub fn new(res: PbBatchDeleteResponse) -> Self {
+        Self(res)
+    }
+
+    #[inline]
+    pub fn take_header(&mut self) -> Option<ResponseHeader> {
+        self.0.header.take().map(ResponseHeader::new)
+    }
+
+    #[inline]
+    pub fn take_prev_kvs(&mut self) -> Vec<KeyValue> {
+        self.0.prev_kvs.drain(..).map(KeyValue::new).collect()
+    }
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct CompareAndPutRequest {
     /// key is the key, in bytes, to put into the key-value store.
@@ -832,6 +905,39 @@ mod tests {
         assert_eq!(b"v1".to_vec(), kvs[0].value().to_vec());
     }
 
+    #[test]
+    fn test_batch_delete_request_trans() {
+        let req = BatchDeleteRequest::new()
+            .add_key(b"test_key1".to_vec())
+            .add_key(b"test_key2".to_vec())
+            .add_key(b"test_key3".to_vec())
+            .with_prev_kv();
+
+        let into_req: PbBatchDeleteRequest = req.into();
+        assert!(into_req.header.is_none());
+        assert_eq!(&b"test_key1".to_vec(), into_req.keys.get(0).unwrap());
+        assert_eq!(&b"test_key2".to_vec(), into_req.keys.get(1).unwrap());
+        assert_eq!(&b"test_key3".to_vec(), into_req.keys.get(2).unwrap());
+        assert!(into_req.prev_kv);
+    }
+
+    #[test]
+    fn test_batch_delete_response_trans() {
+        let pb_res = PbBatchDeleteResponse {
+            header: None,
+            prev_kvs: vec![PbKeyValue {
+                key: b"k1".to_vec(),
+                value: b"v1".to_vec(),
+            }],
+        };
+
+        let mut res = BatchDeleteResponse::new(pb_res);
+        assert!(res.take_header().is_none());
+        let kvs = res.take_prev_kvs();
+        assert_eq!(b"k1".to_vec(), kvs[0].key().to_vec());
+        assert_eq!(b"v1".to_vec(), kvs[0].value().to_vec());
+    }
+
     #[test]
     fn test_compare_and_put_request_trans() {
         let (key, expect, value) = (

src/meta-srv/Cargo.toml

@@ -10,12 +10,14 @@ mock = []
 [dependencies]
 anymap = "1.0.0-beta.2"
 api = { path = "../api" }
+async-stream.workspace = true
 async-trait = "0.1"
 catalog = { path = "../catalog" }
 common-base = { path = "../common/base" }
 common-catalog = { path = "../common/catalog" }
 common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
+common-procedure = { path = "../common/procedure" }
 common-runtime = { path = "../common/runtime" }
 common-telemetry = { path = "../common/telemetry" }
 common-time = { path = "../common/time" }
@@ -39,6 +41,7 @@ tokio-stream = { version = "0.1", features = ["net"] }
 tonic.workspace = true
 tower = "0.4"
 url = "2.3"
+servers = { path = "../servers" }
 
 [dev-dependencies]
 tracing = "0.1"

src/meta-srv/src/bootstrap.rs

@@ -20,8 +20,12 @@ use api::v1::meta::lock_server::LockServer;
 use api::v1::meta::router_server::RouterServer;
 use api::v1::meta::store_server::StoreServer;
 use etcd_client::Client;
+use servers::http::{HttpServer, HttpServerBuilder};
+use servers::metrics_handler::MetricsHandler;
+use servers::server::Server;
 use snafu::ResultExt;
 use tokio::net::TcpListener;
+use tokio::select;
 use tokio::sync::mpsc::{self, Receiver, Sender};
 use tokio_stream::wrappers::TcpListenerStream;
 use tonic::transport::server::Router;
@@ -44,6 +48,8 @@ use crate::{error, Result};
 pub struct MetaSrvInstance {
     meta_srv: MetaSrv,
 
+    http_srv: Arc<HttpServer>,
+
     opts: MetaSrvOptions,
 
     signal_sender: Option<Sender<()>>,
@@ -52,26 +58,44 @@ pub struct MetaSrvInstance {
 impl MetaSrvInstance {
     pub async fn new(opts: MetaSrvOptions) -> Result<MetaSrvInstance> {
         let meta_srv = build_meta_srv(&opts).await?;
-
+        let http_srv = Arc::new(
+            HttpServerBuilder::new(opts.http_opts.clone())
+                .with_metrics_handler(MetricsHandler)
+                .build(),
+        );
         Ok(MetaSrvInstance {
             meta_srv,
+            http_srv,
             opts,
             signal_sender: None,
         })
     }
 
     pub async fn start(&mut self) -> Result<()> {
-        self.meta_srv.start().await;
+        self.meta_srv.try_start().await?;
+
         let (tx, mut rx) = mpsc::channel::<()>(1);
 
         self.signal_sender = Some(tx);
 
-        bootstrap_meta_srv_with_router(
+        let meta_srv = bootstrap_meta_srv_with_router(
             &self.opts.bind_addr,
             router(self.meta_srv.clone()),
             &mut rx,
-        )
-        .await?;
+        );
+        let addr = self
+            .opts
+            .http_opts
+            .addr
+            .parse()
+            .context(error::ParseAddrSnafu {
+                addr: &self.opts.http_opts.addr,
+            })?;
+        let http_srv = self.http_srv.start(addr);
+        select! {
+            v = meta_srv => v?,
+            v = http_srv => v.map(|_| ()).context(error::StartMetricsExportSnafu)?,
+        }
 
         Ok(())
     }
@@ -85,7 +109,12 @@ impl MetaSrvInstance {
         }
 
         self.meta_srv.shutdown();
-
+        self.http_srv
+            .shutdown()
+            .await
+            .context(error::ShutdownServerSnafu {
+                server: self.http_srv.name(),
+            })?;
         Ok(())
     }
 }
@@ -131,10 +160,7 @@ pub async fn build_meta_srv(opts: &MetaSrvOptions) -> Result<MetaSrv> {
             .context(error::ConnectEtcdSnafu)?;
         (
             EtcdStore::with_etcd_client(etcd_client.clone())?,
-            Some(EtcdElection::with_etcd_client(
-                &opts.server_addr,
-                etcd_client.clone(),
-            )?),
+            Some(EtcdElection::with_etcd_client(&opts.server_addr, etcd_client.clone()).await?),
             Some(EtcdLock::with_etcd_client(etcd_client)?),
         )
     };
@@ -172,7 +198,7 @@ pub async fn build_meta_srv(opts: &MetaSrvOptions) -> Result<MetaSrv> {
 pub async fn make_meta_srv(opts: &MetaSrvOptions) -> Result<MetaSrv> {
     let meta_srv = build_meta_srv(opts).await?;
 
-    meta_srv.start().await;
+    meta_srv.try_start().await?;
 
     Ok(meta_srv)
 }

src/meta-srv/src/cluster.rs

@@ -261,7 +261,7 @@ mod tests {
         let stat_val = StatValue { stats: vec![stat] }.try_into().unwrap();
 
         let kv = KeyValue {
-            key: stat_key.clone().into(),
+            key: stat_key.into(),
             value: stat_val,
         };
 

src/meta-srv/src/election.rs

@@ -14,12 +14,23 @@
 
 pub mod etcd;
 
+use std::sync::Arc;
+
+use etcd_client::LeaderKey;
+use tokio::sync::broadcast::Receiver;
+
 use crate::error::Result;
 
 pub const LEASE_SECS: i64 = 3;
 pub const KEEP_ALIVE_PERIOD_SECS: u64 = LEASE_SECS as u64 * 2 / 3;
 pub const ELECTION_KEY: &str = "__meta_srv_election";
 
+#[derive(Clone)]
+pub enum LeaderChangeMessage {
+    Elected(Arc<LeaderKey>),
+    StepDown(Arc<LeaderKey>),
+}
+
 #[async_trait::async_trait]
 pub trait Election: Send + Sync {
     type Leader;
@@ -46,4 +57,6 @@ pub trait Election: Send + Sync {
     /// Releases election leadership so other campaigners may
     /// acquire leadership on the election.
     async fn resign(&self) -> Result<()>;
+
+    fn subscribe_leader_change(&self) -> Receiver<LeaderChangeMessage>;
 }

src/meta-srv/src/election/etcd.rs

@@ -16,11 +16,16 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
-use common_telemetry::{info, warn};
+use common_telemetry::{error, info, warn};
 use etcd_client::Client;
 use snafu::{OptionExt, ResultExt};
+use tokio::sync::broadcast;
+use tokio::sync::broadcast::error::RecvError;
+use tokio::sync::broadcast::Receiver;
 
-use crate::election::{Election, ELECTION_KEY, KEEP_ALIVE_PERIOD_SECS, LEASE_SECS};
+use crate::election::{
+    Election, LeaderChangeMessage, ELECTION_KEY, KEEP_ALIVE_PERIOD_SECS, LEASE_SECS,
+};
 use crate::error;
 use crate::error::Result;
 use crate::metasrv::{ElectionRef, LeaderValue};
@@ -30,6 +35,7 @@ pub struct EtcdElection {
     client: Client,
     is_leader: AtomicBool,
     infancy: AtomicBool,
+    leader_watcher: broadcast::Sender<LeaderChangeMessage>,
 }
 
 impl EtcdElection {
@@ -42,20 +48,50 @@ impl EtcdElection {
             .await
             .context(error::ConnectEtcdSnafu)?;
 
-        Self::with_etcd_client(leader_value, client)
+        Self::with_etcd_client(leader_value, client).await
     }
 
-    pub fn with_etcd_client<E>(leader_value: E, client: Client) -> Result<ElectionRef>
+    pub async fn with_etcd_client<E>(leader_value: E, client: Client) -> Result<ElectionRef>
     where
         E: AsRef<str>,
     {
-        let leader_value = leader_value.as_ref().into();
+        let leader_value: String = leader_value.as_ref().into();
+
+        let leader_ident = leader_value.clone();
+        let (tx, mut rx) = broadcast::channel(100);
+        common_runtime::spawn_bg(async move {
+            loop {
+                match rx.recv().await {
+                    Ok(msg) => match msg {
+                        LeaderChangeMessage::Elected(key) => {
+                            info!(
+                                "[{leader_ident}] is elected as leader: {:?}, lease: {}",
+                                key.name_str(),
+                                key.lease()
+                            );
+                        }
+                        LeaderChangeMessage::StepDown(key) => {
+                            warn!(
+                                "[{leader_ident}] is stepping down: {:?}, lease: {}",
+                                key.name_str(),
+                                key.lease()
+                            );
+                        }
+                    },
+                    Err(RecvError::Lagged(_)) => {
+                        warn!("Log printing is too slow or leader changed too fast!");
+                    }
+                    Err(RecvError::Closed) => break,
+                }
+            }
+        });
 
         Ok(Arc::new(Self {
             leader_value,
             client,
             is_leader: AtomicBool::new(false),
             infancy: AtomicBool::new(false),
+            leader_watcher: tx,
         }))
     }
 }
@@ -120,18 +156,21 @@ impl Election for EtcdElection {
                             .is_ok()
                         {
                             self.infancy.store(true, Ordering::Relaxed);
-                            info!(
-                                "[{}] becoming leader: {:?}, lease: {}",
-                                &self.leader_value,
-                                leader.name_str(),
-                                leader.lease()
-                            );
+
+                            if let Err(e) = self
+                                .leader_watcher
+                                .send(LeaderChangeMessage::Elected(Arc::new(leader.clone())))
+                            {
+                                error!("Failed to send leader change message, error: {e}");
+                            }
                         }
                     } else {
-                        warn!(
-                            "Failed to keep-alive, lease: {}, will re-initiate election",
-                            leader.lease()
-                        );
+                        if let Err(e) = self
+                            .leader_watcher
+                            .send(LeaderChangeMessage::StepDown(Arc::new(leader.clone())))
+                        {
+                            error!("Failed to send leader change message, error: {e}");
+                        }
                         break;
                     }
                 }
@@ -162,4 +201,8 @@ impl Election for EtcdElection {
     async fn resign(&self) -> Result<()> {
         todo!()
     }
+
+    fn subscribe_leader_change(&self) -> Receiver<LeaderChangeMessage> {
+        self.leader_watcher.subscribe()
+    }
 }

src/meta-srv/src/error.rs

@@ -25,6 +25,13 @@ pub enum Error {
     #[snafu(display("Failed to send shutdown signal"))]
     SendShutdownSignal { source: SendError<()> },
 
+    #[snafu(display("Failed to shutdown {} server, source: {}", server, source))]
+    ShutdownServer {
+        #[snafu(backtrace)]
+        source: servers::error::Error,
+        server: String,
+    },
+
     #[snafu(display("Error stream request next is None"))]
     StreamNone { backtrace: Backtrace },
 
@@ -55,7 +62,16 @@ pub enum Error {
         source: tonic::transport::Error,
         backtrace: Backtrace,
     },
-
+    #[snafu(display("Failed to start gRPC server, source: {}", source))]
+    StartMetricsExport {
+        #[snafu(backtrace)]
+        source: servers::error::Error,
+    },
+    #[snafu(display("Failed to parse address {}, source: {}", addr, source))]
+    ParseAddr {
+        addr: String,
+        source: std::net::AddrParseError,
+    },
     #[snafu(display("Empty table name"))]
     EmptyTableName { backtrace: Backtrace },
 
@@ -274,6 +290,12 @@ pub enum Error {
 
     #[snafu(display("Missing required parameter, param: {:?}", param))]
     MissingRequiredParameter { param: String },
+
+    #[snafu(display("Failed to recover procedure, source: {source}"))]
+    RecoverProcedure {
+        #[snafu(backtrace)]
+        source: common_procedure::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -317,6 +339,7 @@ impl ErrorExt for Error {
             | Error::LockNotConfig { .. }
             | Error::ExceededRetryLimit { .. }
             | Error::SendShutdownSignal { .. }
+            | Error::ParseAddr { .. }
             | Error::StartGrpc { .. } => StatusCode::Internal,
             Error::EmptyKey { .. }
             | Error::MissingRequiredParameter { .. }
@@ -341,6 +364,10 @@ impl ErrorExt for Error {
             Error::TableNotFound { .. } => StatusCode::TableNotFound,
             Error::InvalidCatalogValue { source, .. } => source.status_code(),
             Error::MetaInternal { source } => source.status_code(),
+            Error::RecoverProcedure { source } => source.status_code(),
+            Error::ShutdownServer { source, .. } | Error::StartMetricsExport { source } => {
+                source.status_code()
+            }
         }
     }
 }

src/meta-srv/src/failure_detector.rs

@@ -32,30 +32,27 @@ use std::collections::VecDeque;
 ///
 /// where F is the cumulative distribution function of a normal distribution with mean
 /// and standard deviation estimated from historical heartbeat inter-arrival times.
+#[cfg_attr(test, derive(Clone))]
 pub(crate) struct PhiAccrualFailureDetector {
     /// A low threshold is prone to generate many wrong suspicions but ensures a quick detection
     /// in the event of a real crash. Conversely, a high threshold generates fewer mistakes but
     /// needs more time to detect actual crashes.
-    threshold: f64,
-
-    /// Number of samples to use for calculation of mean and standard deviation of inter-arrival
-    /// times.
-    max_sample_size: u32,
+    threshold: f32,
 
     /// Minimum standard deviation to use for the normal distribution used when calculating phi.
     /// Too low standard deviation might result in too much sensitivity for sudden, but normal,
     /// deviations in heartbeat inter arrival times.
-    min_std_deviation_millis: f64,
+    min_std_deviation_millis: f32,
 
     /// Duration corresponding to number of potentially lost/delayed heartbeats that will be
     /// accepted before considering it to be an anomaly.
     /// This margin is important to be able to survive sudden, occasional, pauses in heartbeat
     /// arrivals, due to for example network drop.
-    acceptable_heartbeat_pause_millis: i64,
+    acceptable_heartbeat_pause_millis: u32,
 
     /// Bootstrap the stats with heartbeats that corresponds to this duration, with a rather high
     /// standard deviation (since environment is unknown in the beginning).
-    first_heartbeat_estimate_millis: i64,
+    first_heartbeat_estimate_millis: u32,
 
     heartbeat_history: HeartbeatHistory,
     last_heartbeat_millis: Option<i64>,
@@ -65,14 +62,12 @@ impl Default for PhiAccrualFailureDetector {
     fn default() -> Self {
         // default configuration is the same as of Akka:
         // https://github.com/akka/akka/blob/main/akka-cluster/src/main/resources/reference.conf#L181
-        let max_sample_size = 1000;
         Self {
-            threshold: 8_f64,
-            max_sample_size,
-            min_std_deviation_millis: 100_f64,
+            threshold: 8_f32,
+            min_std_deviation_millis: 100_f32,
             acceptable_heartbeat_pause_millis: 3000,
             first_heartbeat_estimate_millis: 1000,
-            heartbeat_history: HeartbeatHistory::new(max_sample_size),
+            heartbeat_history: HeartbeatHistory::new(1000),
             last_heartbeat_millis: None,
         }
     }
@@ -95,28 +90,28 @@ impl PhiAccrualFailureDetector {
             // bootstrap with 2 entries with rather high standard deviation
             let std_deviation = self.first_heartbeat_estimate_millis / 4;
             self.heartbeat_history
-                .add(self.first_heartbeat_estimate_millis - std_deviation);
+                .add((self.first_heartbeat_estimate_millis - std_deviation) as _);
             self.heartbeat_history
-                .add(self.first_heartbeat_estimate_millis + std_deviation);
+                .add((self.first_heartbeat_estimate_millis + std_deviation) as _);
         }
         let _ = self.last_heartbeat_millis.insert(ts_millis);
     }
 
     pub(crate) fn is_available(&self, ts_millis: i64) -> bool {
-        self.phi(ts_millis) < self.threshold
+        self.phi(ts_millis) < self.threshold as _
     }
 
     /// The suspicion level of the accrual failure detector.
     ///
     /// If a connection does not have any records in failure detector then it is considered healthy.
-    fn phi(&self, ts_millis: i64) -> f64 {
+    pub(crate) fn phi(&self, ts_millis: i64) -> f64 {
         if let Some(last_heartbeat_millis) = self.last_heartbeat_millis {
             let time_diff = ts_millis - last_heartbeat_millis;
             let mean = self.heartbeat_history.mean();
             let std_deviation = self
                 .heartbeat_history
                 .std_deviation()
-                .max(self.min_std_deviation_millis);
+                .max(self.min_std_deviation_millis as _);
 
             phi(
                 time_diff,
@@ -128,6 +123,16 @@ impl PhiAccrualFailureDetector {
             0.0
         }
     }
+
+    #[cfg(test)]
+    pub(crate) fn threshold(&self) -> f32 {
+        self.threshold
+    }
+
+    #[cfg(test)]
+    pub(crate) fn acceptable_heartbeat_pause_millis(&self) -> u32 {
+        self.acceptable_heartbeat_pause_millis
+    }
 }
 
 /// Calculation of phi, derived from the Cumulative distribution function for
@@ -141,6 +146,8 @@ impl PhiAccrualFailureDetector {
 /// Usually phi = 1 means likeliness that we will make a mistake is about 10%.
 /// The likeliness is about 1% with phi = 2, 0.1% with phi = 3 and so on.
 fn phi(time_diff: i64, mean: f64, std_deviation: f64) -> f64 {
+    assert_ne!(std_deviation, 0.0);
+
     let time_diff = time_diff as f64;
     let y = (time_diff - mean) / std_deviation;
     let e = (-y * (1.5976 + 0.070566 * y * y)).exp();
@@ -155,8 +162,12 @@ fn phi(time_diff: i64, mean: f64, std_deviation: f64) -> f64 {
 /// It is capped by the number of samples specified in `max_sample_size`.
 ///
 /// The stats (mean, variance, std_deviation) are not defined for empty HeartbeatHistory.
+#[derive(Clone)]
 struct HeartbeatHistory {
+    /// Number of samples to use for calculation of mean and standard deviation of inter-arrival
+    /// times.
     max_sample_size: u32,
+
     intervals: VecDeque<i64>,
     interval_sum: i64,
     squared_interval_sum: i64,
@@ -198,7 +209,7 @@ impl HeartbeatHistory {
         let oldest = self
             .intervals
             .pop_front()
-            .expect("intervals must not empty here");
+            .expect("intervals must not be empty here");
         self.interval_sum -= oldest;
         self.squared_interval_sum -= oldest * oldest;
     }
@@ -207,42 +218,9 @@ impl HeartbeatHistory {
 #[cfg(test)]
 mod tests {
     use common_time::util::current_time_millis;
-    use rand::Rng;
 
     use super::*;
 
-    #[test]
-    fn test_heartbeat() {
-        // Generate 2000 heartbeats start from now. Heartbeat interval is one second, plus some
-        // random millis.
-        fn generate_heartbeats() -> Vec<i64> {
-            let mut rng = rand::thread_rng();
-            let start = current_time_millis();
-            (0..2000)
-                .map(|i| start + i * 1000 + rng.gen_range(0..100))
-                .collect::<Vec<i64>>()
-        }
-        let heartbeats = generate_heartbeats();
-
-        let mut fd = PhiAccrualFailureDetector::default();
-        // feed the failure detector with these heartbeats
-        heartbeats.iter().for_each(|x| fd.heartbeat(*x));
-
-        let start = *heartbeats.last().unwrap();
-        // Within the "acceptable_heartbeat_pause_millis" period, phi is zero ...
-        for i in 1..=fd.acceptable_heartbeat_pause_millis / 1000 {
-            let now = start + i * 1000;
-            assert_eq!(fd.phi(now), 0.0);
-        }
-
-        // ... then in less than two seconds, phi is above the threshold.
-        // The same effect can be seen in the diagrams in Akka's document.
-        let now = start + fd.acceptable_heartbeat_pause_millis + 1000;
-        assert!(fd.phi(now) < fd.threshold);
-        let now = start + fd.acceptable_heartbeat_pause_millis + 2000;
-        assert!(fd.phi(now) > fd.threshold);
-    }
-
     #[test]
     fn test_is_available() {
         let ts_millis = current_time_millis();
@@ -254,12 +232,13 @@ mod tests {
 
         fd.heartbeat(ts_millis);
 
+        let acceptable_heartbeat_pause_millis = fd.acceptable_heartbeat_pause_millis as i64;
         // is available when heartbeat
         assert!(fd.is_available(ts_millis));
         // is available before heartbeat timeout
-        assert!(fd.is_available(ts_millis + fd.acceptable_heartbeat_pause_millis / 2));
+        assert!(fd.is_available(ts_millis + acceptable_heartbeat_pause_millis / 2));
         // is not available after heartbeat timeout
-        assert!(!fd.is_available(ts_millis + fd.acceptable_heartbeat_pause_millis * 2));
+        assert!(!fd.is_available(ts_millis + acceptable_heartbeat_pause_millis * 2));
     }
 
     #[test]
@@ -286,14 +265,15 @@ mod tests {
 
         fd.heartbeat(ts_millis);
 
+        let acceptable_heartbeat_pause_millis = fd.acceptable_heartbeat_pause_millis as i64;
         // phi == 0 when heartbeat
         assert_eq!(fd.phi(ts_millis), 0.0);
         // phi < threshold before heartbeat timeout
-        let now = ts_millis + fd.acceptable_heartbeat_pause_millis / 2;
-        assert!(fd.phi(now) < fd.threshold);
+        let now = ts_millis + acceptable_heartbeat_pause_millis / 2;
+        assert!(fd.phi(now) < fd.threshold as _);
         // phi >= threshold after heartbeat timeout
-        let now = ts_millis + fd.acceptable_heartbeat_pause_millis * 2;
-        assert!(fd.phi(now) >= fd.threshold);
+        let now = ts_millis + acceptable_heartbeat_pause_millis * 2;
+        assert!(fd.phi(now) >= fd.threshold as _);
     }
 
     // The following test cases are port from Akka's test:
@@ -349,7 +329,6 @@ mod tests {
     fn test_return_phi_of_0_on_startup_when_no_heartbeats() {
         let fd = PhiAccrualFailureDetector {
             threshold: 8.0,
-            max_sample_size: 1000,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 0,
             first_heartbeat_estimate_millis: 1000,
@@ -364,7 +343,6 @@ mod tests {
     fn test_return_phi_based_on_guess_when_only_one_heartbeat() {
         let mut fd = PhiAccrualFailureDetector {
             threshold: 8.0,
-            max_sample_size: 1000,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 0,
             first_heartbeat_estimate_millis: 1000,
@@ -381,7 +359,6 @@ mod tests {
     fn test_return_phi_using_first_interval_after_second_heartbeat() {
         let mut fd = PhiAccrualFailureDetector {
             threshold: 8.0,
-            max_sample_size: 1000,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 0,
             first_heartbeat_estimate_millis: 1000,
@@ -398,7 +375,6 @@ mod tests {
     fn test_is_available_after_a_series_of_successful_heartbeats() {
         let mut fd = PhiAccrualFailureDetector {
             threshold: 8.0,
-            max_sample_size: 1000,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 0,
             first_heartbeat_estimate_millis: 1000,
@@ -417,7 +393,6 @@ mod tests {
     fn test_is_not_available_if_heartbeat_are_missed() {
         let mut fd = PhiAccrualFailureDetector {
             threshold: 3.0,
-            max_sample_size: 1000,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 0,
             first_heartbeat_estimate_millis: 1000,
@@ -436,7 +411,6 @@ mod tests {
     ) {
         let mut fd = PhiAccrualFailureDetector {
             threshold: 8.0,
-            max_sample_size: 1000,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 3000,
             first_heartbeat_estimate_millis: 1000,
@@ -476,7 +450,6 @@ mod tests {
     fn test_accept_some_configured_missing_heartbeats() {
         let mut fd = PhiAccrualFailureDetector {
             threshold: 8.0,
-            max_sample_size: 1000,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 3000,
             first_heartbeat_estimate_millis: 1000,
@@ -496,7 +469,6 @@ mod tests {
     fn test_fail_after_configured_acceptable_missing_heartbeats() {
         let mut fd = PhiAccrualFailureDetector {
             threshold: 8.0,
-            max_sample_size: 1000,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 3000,
             first_heartbeat_estimate_millis: 1000,
@@ -518,7 +490,6 @@ mod tests {
     fn test_use_max_sample_size_heartbeats() {
         let mut fd = PhiAccrualFailureDetector {
             threshold: 8.0,
-            max_sample_size: 3,
             min_std_deviation_millis: 100.0,
             acceptable_heartbeat_pause_millis: 0,
             first_heartbeat_estimate_millis: 1000,

src/meta-srv/src/handler.rs

@@ -14,6 +14,7 @@
 
 pub use check_leader_handler::CheckLeaderHandler;
 pub use collect_stats_handler::CollectStatsHandler;
+pub use failure_handler::RegionFailureHandler;
 pub use keep_lease_handler::KeepLeaseHandler;
 pub use on_leader_start::OnLeaderStartHandler;
 pub use persist_stats_handler::PersistStatsHandler;
@@ -21,6 +22,7 @@ pub use response_header_handler::ResponseHeaderHandler;
 
 mod check_leader_handler;
 mod collect_stats_handler;
+mod failure_handler;
 mod instruction;
 mod keep_lease_handler;
 pub mod node_stat;
@@ -54,8 +56,8 @@ pub trait HeartbeatHandler: Send + Sync {
 #[derive(Debug, Default)]
 pub struct HeartbeatAccumulator {
     pub header: Option<ResponseHeader>,
-    pub stats: Vec<Stat>,
     pub instructions: Vec<Instruction>,
+    pub stat: Option<Stat>,
 }
 
 impl HeartbeatAccumulator {