fix: missing file engine with default options (#2519)

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

.github/actions/upload-artifacts/action.yml

@@ -37,7 +37,7 @@ inputs:
   upload-retry-timeout:
     description: Timeout for uploading artifacts to S3
     required: false
-    default: "10" # minutes
+    default: "30" # minutes
   working-dir:
     description: Working directory to upload the artifacts
     required: false

.github/workflows/develop.yml

@@ -194,38 +194,37 @@ jobs:
     runs-on: windows-latest-8-cores
     timeout-minutes: 60
     steps:
-      - run: 'echo "temporary disabled"'
-      # - run: git config --global core.autocrlf false
-      # - uses: actions/checkout@v3
-      # - uses: arduino/setup-protoc@v1
-      #   with:
-      #     repo-token: ${{ secrets.GITHUB_TOKEN }}
-      # - name: Install Rust toolchain
-      #   uses: dtolnay/rust-toolchain@master
-      #   with:
-      #     toolchain: ${{ env.RUST_TOOLCHAIN }}
-      #     components: llvm-tools-preview
-      # - name: Rust Cache
-      #   uses: Swatinem/rust-cache@v2
-      # - name: Install Cargo Nextest
-      #   uses: taiki-e/install-action@nextest
-      # - name: Install Python
-      #   uses: actions/setup-python@v4
-      #   with:
-      #     python-version: '3.10'
-      # - name: Install PyArrow Package
-      #   run: pip install pyarrow
-      # - name: Install WSL distribution
-      #   uses: Vampire/setup-wsl@v2
-      #   with:
-      #     distribution: Ubuntu-22.04
-      # - name: Running tests
-      #   run: cargo nextest run -F pyo3_backend,dashboard
-      #   env:
-      #     RUST_BACKTRACE: 1
-      #     CARGO_INCREMENTAL: 0
-      #     GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-      #     GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-      #     GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-      #     GT_S3_REGION: ${{ secrets.S3_REGION }}
-      #     UNITTEST_LOG_DIR: "__unittest_logs"
+      - run: git config --global core.autocrlf false
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
+      - name: Install WSL distribution
+        uses: Vampire/setup-wsl@v2
+        with:
+         distribution: Ubuntu-22.04
+      - name: Running tests
+        run: cargo nextest run -F pyo3_backend,dashboard
+        env:
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          GT_S3_REGION: ${{ secrets.S3_REGION }}
+          UNITTEST_LOG_DIR: "__unittest_logs"

Cargo.lock

@@ -208,6 +208,7 @@ version = "0.4.0-nightly"
 dependencies = [
  "common-base",
  "common-error",
+ "common-macro",
  "common-time",
  "datatypes",
  "greptime-proto",
@@ -690,6 +691,7 @@ dependencies = [
  "api",
  "async-trait",
  "common-error",
+ "common-macro",
  "common-test-util",
  "digest",
  "hex",
@@ -1250,6 +1252,7 @@ dependencies = [
  "common-catalog",
  "common-error",
  "common-grpc",
+ "common-macro",
  "common-meta",
  "common-query",
  "common-recordbatch",
@@ -1266,7 +1269,7 @@ dependencies = [
  "log-store",
  "meta-client",
  "metrics",
- "moka 0.11.3",
+ "moka",
  "object-store",
  "parking_lot 0.12.1",
  "partition",
@@ -1532,6 +1535,7 @@ dependencies = [
  "common-error",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-query",
  "common-recordbatch",
@@ -1544,7 +1548,7 @@ dependencies = [
  "derive_builder 0.12.0",
  "enum_dispatch",
  "futures-util",
- "moka 0.9.9",
+ "moka",
  "parking_lot 0.12.1",
  "prost",
  "rand",
@@ -1600,6 +1604,7 @@ dependencies = [
  "common-catalog",
  "common-config",
  "common-error",
+ "common-macro",
  "common-meta",
  "common-procedure",
  "common-query",
@@ -1612,12 +1617,14 @@ dependencies = [
  "datatypes",
  "either",
  "etcd-client",
+ "file-engine",
  "frontend",
  "futures",
  "lazy_static",
  "meta-client",
  "meta-srv",
  "metrics",
+ "mito2",
  "nu-ansi-term",
  "partition",
  "prost",
@@ -1670,6 +1677,7 @@ dependencies = [
  "bitvec",
  "bytes",
  "common-error",
+ "common-macro",
  "paste",
  "serde",
  "snafu",
@@ -1682,6 +1690,7 @@ version = "0.4.0-nightly"
 dependencies = [
  "chrono",
  "common-error",
+ "common-macro",
  "serde",
  "serde_json",
  "snafu",
@@ -1707,11 +1716,13 @@ dependencies = [
  "async-trait",
  "bytes",
  "common-error",
+ "common-macro",
  "common-runtime",
  "common-test-util",
  "datafusion",
  "derive_builder 0.12.0",
  "futures",
+ "lazy_static",
  "object-store",
  "orc-rust",
  "paste",
@@ -1784,6 +1795,7 @@ dependencies = [
  "backtrace",
  "common-base",
  "common-error",
+ "common-macro",
  "common-recordbatch",
  "common-runtime",
  "common-telemetry",
@@ -1812,6 +1824,7 @@ dependencies = [
  "common-base",
  "common-catalog",
  "common-error",
+ "common-macro",
  "common-query",
  "common-telemetry",
  "common-time",
@@ -1835,6 +1848,7 @@ dependencies = [
  "snafu",
  "static_assertions",
  "syn 1.0.109",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -1842,6 +1856,7 @@ name = "common-mem-prof"
 version = "0.4.0-nightly"
 dependencies = [
  "common-error",
+ "common-macro",
  "snafu",
  "tempfile",
  "tikv-jemalloc-ctl",
@@ -1861,6 +1876,7 @@ dependencies = [
  "common-catalog",
  "common-error",
  "common-grpc-expr",
+ "common-macro",
  "common-procedure",
  "common-recordbatch",
  "common-runtime",
@@ -1892,6 +1908,7 @@ dependencies = [
  "async-trait",
  "backon",
  "common-error",
+ "common-macro",
  "common-runtime",
  "common-telemetry",
  "common-test-util",
@@ -1923,6 +1940,7 @@ dependencies = [
  "async-trait",
  "common-base",
  "common-error",
+ "common-macro",
  "common-recordbatch",
  "common-time",
  "datafusion",
@@ -1942,6 +1960,7 @@ name = "common-recordbatch"
 version = "0.4.0-nightly"
 dependencies = [
  "common-error",
+ "common-macro",
  "datafusion",
  "datafusion-common",
  "datatypes",
@@ -1959,6 +1978,7 @@ version = "0.4.0-nightly"
 dependencies = [
  "async-trait",
  "common-error",
+ "common-macro",
  "common-telemetry",
  "metrics",
  "once_cell",
@@ -2013,6 +2033,7 @@ dependencies = [
  "chrono",
  "chrono-tz 0.8.3",
  "common-error",
+ "common-macro",
  "rand",
  "serde",
  "serde_json",
@@ -2681,6 +2702,7 @@ dependencies = [
  "common-greptimedb-telemetry",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-procedure",
  "common-query",
@@ -2738,6 +2760,7 @@ dependencies = [
  "arrow-schema",
  "common-base",
  "common-error",
+ "common-macro",
  "common-telemetry",
  "common-time",
  "datafusion-common",
@@ -3200,6 +3223,7 @@ dependencies = [
  "common-catalog",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-procedure",
  "common-procedure-test",
  "common-query",
@@ -3322,6 +3346,7 @@ dependencies = [
  "common-function",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-procedure",
  "common-query",
@@ -3344,7 +3369,7 @@ dependencies = [
  "meta-client",
  "meta-srv",
  "metrics",
- "moka 0.9.9",
+ "moka",
  "object-store",
  "openmetrics-parser",
  "opentelemetry-proto",
@@ -4185,7 +4210,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=9d3f28d07d29607d0e3c1823f4a4d2bc229d05b9#9d3f28d07d29607d0e3c1823f4a4d2bc229d05b9"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=693128abe9adc70ba636010a172c9da55b206bba#693128abe9adc70ba636010a172c9da55b206bba"
 dependencies = [
  "prost",
  "serde",
@@ -5027,6 +5052,7 @@ dependencies = [
  "common-base",
  "common-config",
  "common-error",
+ "common-macro",
  "common-meta",
  "common-runtime",
  "common-telemetry",
@@ -5302,6 +5328,7 @@ dependencies = [
  "chrono",
  "common-error",
  "common-grpc",
+ "common-macro",
  "common-meta",
  "common-telemetry",
  "datatypes",
@@ -5338,6 +5365,7 @@ dependencies = [
  "common-greptimedb-telemetry",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-procedure",
  "common-procedure-test",
@@ -5529,6 +5557,7 @@ dependencies = [
  "common-catalog",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-procedure",
  "common-procedure-test",
  "common-query",
@@ -5548,6 +5577,7 @@ dependencies = [
  "log-store",
  "memcomparable",
  "metrics",
+ "moka",
  "object-store",
  "parquet",
  "paste",
@@ -5566,32 +5596,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "moka"
-version = "0.9.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b28455ac4363046076054a7e9cfbd7f168019c29dba32a625f59fc0aeffaaea4"
-dependencies = [
- "async-io",
- "async-lock",
- "crossbeam-channel",
- "crossbeam-epoch",
- "crossbeam-utils",
- "futures-util",
- "num_cpus",
- "once_cell",
- "parking_lot 0.12.1",
- "quanta 0.11.1",
- "rustc_version",
- "scheduled-thread-pool",
- "skeptic",
- "smallvec",
- "tagptr",
- "thiserror",
- "triomphe",
- "uuid",
-]
-
 [[package]]
 name = "moka"
 version = "0.11.3"
@@ -6052,8 +6056,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
 
 [[package]]
 name = "opendal"
-version = "0.39.0"
-source = "git+https://github.com/apache/incubator-opendal.git?rev=7d5524f35f29f7eda8131e8b0873590b7cbe34ab#7d5524f35f29f7eda8131e8b0873590b7cbe34ab"
+version = "0.40.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddba7299bab261d3ae2f37617fb7f45b19ed872752bb4e22cf93a69d979366c5"
 dependencies = [
  "anyhow",
  "async-compat",
@@ -6237,6 +6242,7 @@ dependencies = [
  "common-datasource",
  "common-error",
  "common-grpc-expr",
+ "common-macro",
  "common-meta",
  "common-query",
  "common-recordbatch",
@@ -6499,6 +6505,7 @@ dependencies = [
  "async-trait",
  "common-catalog",
  "common-error",
+ "common-macro",
  "common-meta",
  "common-query",
  "common-telemetry",
@@ -6507,7 +6514,7 @@ dependencies = [
  "datafusion-expr",
  "datatypes",
  "meta-client",
- "moka 0.9.9",
+ "moka",
  "serde",
  "serde_json",
  "snafu",
@@ -8622,6 +8629,7 @@ dependencies = [
  "common-catalog",
  "common-error",
  "common-function",
+ "common-macro",
  "common-query",
  "common-recordbatch",
  "common-runtime",
@@ -8912,6 +8920,7 @@ dependencies = [
  "common-error",
  "common-grpc",
  "common-grpc-expr",
+ "common-macro",
  "common-mem-prof",
  "common-meta",
  "common-query",
@@ -9268,6 +9277,7 @@ dependencies = [
  "common-catalog",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-query",
  "common-time",
  "datafusion-sql",
@@ -9532,6 +9542,7 @@ dependencies = [
  "common-config",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-query",
  "common-recordbatch",
  "common-runtime",
@@ -9579,6 +9590,7 @@ dependencies = [
  "bytes",
  "common-base",
  "common-error",
+ "common-macro",
  "common-query",
  "common-recordbatch",
  "common-time",
@@ -9714,6 +9726,7 @@ dependencies = [
  "catalog",
  "common-catalog",
  "common-error",
+ "common-macro",
  "common-telemetry",
  "datafusion",
  "datafusion-common",
@@ -9872,6 +9885,7 @@ dependencies = [
  "common-catalog",
  "common-datasource",
  "common-error",
+ "common-macro",
  "common-procedure",
  "common-query",
  "common-recordbatch",

Cargo.toml

@@ -78,10 +78,11 @@ datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git
 derive_builder = "0.12"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "9d3f28d07d29607d0e3c1823f4a4d2bc229d05b9" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "693128abe9adc70ba636010a172c9da55b206bba" }
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
+moka = { version = "0.11" }
 once_cell = "1.18"
 opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
 parquet = "43.0"

README.md

@@ -27,6 +27,14 @@
     <a href="https://greptime.com/slack"><img src="https://img.shields.io/badge/slack-GreptimeDB-0abd59?logo=slack" alt="slack" /></a>
 </p>
 
+## Upcoming Event
+Come and meet us in **KubeCon + CloudNativeCon North America 2023!**
+<p align="center">
+  <picture>
+    <img alt="KubeCon + CloudNativeCon North Logo" src="./docs/banner/KCCNC_NA_2023_1000x200_Email Banner.png" width="800px">
+  </picture>
+</p>
+
 ## What is GreptimeDB
 
 GreptimeDB is an open-source time-series database with a special focus on

config/datanode.example.toml

@@ -13,17 +13,19 @@ rpc_runtime_size = 8
 require_lease_before_startup = false
 
 [heartbeat]
-# Interval for sending heartbeat messages to the Metasrv in milliseconds, 5000 by default.
-interval_millis = 5000
+# Interval for sending heartbeat messages to the Metasrv in milliseconds, 3000 by default.
+interval_millis = 3000
 
 # Metasrv client options.
 [meta_client]
 # Metasrv address list.
 metasrv_addrs = ["127.0.0.1:3002"]
+# Heartbeat timeout in milliseconds, 500 by default.
+heartbeat_timeout_millis = 500
 # Operation timeout in milliseconds, 3000 by default.
 timeout_millis = 3000
 # Connect server timeout in milliseconds, 5000 by default.
-connect_timeout_millis = 5000
+connect_timeout_millis = 1000
 # `TCP_NODELAY` option for accepted connections, true by default.
 tcp_nodelay = true
 

config/frontend.example.toml

@@ -8,7 +8,7 @@ interval_millis = 5000
 retry_interval_millis = 5000
 
 # HTTP server options, see `standalone.example.toml`.
-[http_options]
+[http]
 addr = "127.0.0.1:4000"
 timeout = "30s"
 body_limit = "64MB"
@@ -62,7 +62,7 @@ metasrv_addrs = ["127.0.0.1:3002"]
 timeout_millis = 3000
 # DDL timeouts options.
 ddl_timeout_millis = 10000
-connect_timeout_millis = 5000
+connect_timeout_millis = 1000
 tcp_nodelay = true
 
 # Log options, see `standalone.example.toml`

src/api/Cargo.toml

@@ -7,6 +7,7 @@ license.workspace = true
 [dependencies]
 common-base = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-time = { workspace = true }
 datatypes = { workspace = true }
 greptime-proto.workspace = true

src/api/src/error.rs

@@ -16,14 +16,16 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
 use snafu::Location;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unknown proto column datatype: {}", datatype))]
     UnknownColumnDataType { datatype: i32, location: Location },
@@ -34,22 +36,14 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display(
-        "Failed to convert column default constraint, column: {}, source: {}",
-        column,
-        source
-    ))]
+    #[snafu(display("Failed to convert column default constraint, column: {}", column))]
     ConvertColumnDefaultConstraint {
         column: String,
         location: Location,
         source: datatypes::error::Error,
     },
 
-    #[snafu(display(
-        "Invalid column default constraint, column: {}, source: {}",
-        column,
-        source
-    ))]
+    #[snafu(display("Invalid column default constraint, column: {}", column))]
     InvalidColumnDefaultConstraint {
         column: String,
         location: Location,

src/auth/Cargo.toml

@@ -14,6 +14,7 @@ testing = []
 api.workspace = true
 async-trait.workspace = true
 common-error.workspace = true
+common-macro.workspace = true
 digest = "0.10"
 hex = { version = "0.4" }
 secrecy = { version = "0.8", features = ["serde", "alloc"] }

src/auth/src/error.rs

@@ -14,10 +14,12 @@
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid config value: {}, {}", value, msg))]
     InvalidConfig { value: String, msg: String },
@@ -28,13 +30,14 @@ pub enum Error {
     #[snafu(display("Internal state error: {}", msg))]
     InternalState { msg: String },
 
-    #[snafu(display("IO error, source: {}", source))]
+    #[snafu(display("IO error"))]
     Io {
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 
-    #[snafu(display("Auth failed, source: {}", source))]
+    #[snafu(display("Auth failed"))]
     AuthBackend {
         location: Location,
         source: BoxedError,

src/catalog/Cargo.toml

@@ -16,6 +16,7 @@ async-trait = "0.1"
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
@@ -30,7 +31,7 @@ futures-util.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 metrics.workspace = true
-moka = { version = "0.11", features = ["future"] }
+moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 partition.workspace = true
 regex.workspace = true

src/catalog/src/error.rs

@@ -17,53 +17,48 @@ use std::fmt::Debug;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
 use table::metadata::TableId;
 use tokio::task::JoinError;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to list catalogs, source: {}", source))]
+    #[snafu(display("Failed to list catalogs"))]
     ListCatalogs {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to list {}'s schemas, source: {}", catalog, source))]
+    #[snafu(display("Failed to list {}'s schemas", catalog))]
     ListSchemas {
         location: Location,
         catalog: String,
         source: BoxedError,
     },
 
-    #[snafu(display(
-        "Failed to re-compile script due to internal error, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to re-compile script due to internal error"))]
     CompileScriptInternal {
         location: Location,
         source: BoxedError,
     },
-    #[snafu(display("Failed to open system catalog table, source: {}", source))]
+    #[snafu(display("Failed to open system catalog table"))]
     OpenSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to create system catalog table, source: {}", source))]
+    #[snafu(display("Failed to create system catalog table"))]
     CreateSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display(
-        "Failed to create table, table info: {}, source: {}",
-        table_info,
-        source
-    ))]
+    #[snafu(display("Failed to create table, table info: {}", table_info))]
     CreateTable {
         table_info: String,
         location: Location,
@@ -97,13 +92,14 @@ pub enum Error {
     #[snafu(display("Catalog value is not present"))]
     EmptyValue { location: Location },
 
-    #[snafu(display("Failed to deserialize value, source: {}", source))]
+    #[snafu(display("Failed to deserialize value"))]
     ValueDeserialize {
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
+    #[snafu(display("Table engine not found: {}", engine_name))]
     TableEngineNotFound {
         engine_name: String,
         location: Location,
@@ -141,15 +137,18 @@ pub enum Error {
     #[snafu(display("Operation {} not supported", op))]
     NotSupported { op: String, location: Location },
 
-    #[snafu(display("Failed to open table {table_id}, source: {source}, at {location}"))]
+    #[snafu(display("Failed to open table {table_id}"))]
     OpenTable {
         table_id: TableId,
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to open table in parallel, source: {}", source))]
-    ParallelOpenTable { source: JoinError },
+    #[snafu(display("Failed to open table in parallel"))]
+    ParallelOpenTable {
+        #[snafu(source)]
+        error: JoinError,
+    },
 
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
     TableNotFound {
@@ -163,58 +162,52 @@ pub enum Error {
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to create recordbatch, source: {}", source))]
+    #[snafu(display("Failed to create recordbatch"))]
     CreateRecordBatch {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display(
-        "Failed to insert table creation record to system catalog, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to insert table creation record to system catalog"))]
     InsertCatalogRecord {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to scan system catalog table, source: {}", source))]
+    #[snafu(display("Failed to scan system catalog table"))]
     SystemCatalogTableScan {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("{source}"))]
+    #[snafu(display(""))]
     Internal {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display(
-        "Failed to upgrade weak catalog manager reference. location: {}",
-        location
-    ))]
+    #[snafu(display("Failed to upgrade weak catalog manager reference"))]
     UpgradeWeakCatalogManagerRef { location: Location },
 
-    #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
+    #[snafu(display("Failed to execute system catalog table scan"))]
     SystemCatalogTableScanExec {
         location: Location,
         source: common_query::error::Error,
     },
 
-    #[snafu(display("Cannot parse catalog value, source: {}", source))]
+    #[snafu(display("Cannot parse catalog value"))]
     InvalidCatalogValue {
         location: Location,
         source: common_catalog::error::Error,
     },
 
-    #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
+    #[snafu(display("Failed to perform metasrv operation"))]
     MetaSrv {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Invalid table info in catalog, source: {}", source))]
+    #[snafu(display("Invalid table info in catalog"))]
     InvalidTableInfoInCatalog {
         location: Location,
         source: datatypes::error::Error,
@@ -223,14 +216,14 @@ pub enum Error {
     #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
     QueryAccessDenied { catalog: String, schema: String },
 
-    #[snafu(display("{}: {}", msg, source))]
+    #[snafu(display(""))]
     Datafusion {
-        msg: String,
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
-    #[snafu(display("Table schema mismatch, source: {}", source))]
+    #[snafu(display("Table schema mismatch"))]
     TableSchemaMismatch {
         location: Location,
         source: table::error::Error,
@@ -239,7 +232,7 @@ pub enum Error {
     #[snafu(display("A generic error has occurred, msg: {}", msg))]
     Generic { msg: String, location: Location },
 
-    #[snafu(display("Table metadata manager error: {}", source))]
+    #[snafu(display("Table metadata manager error"))]
     TableMetadataManager {
         source: common_meta::error::Error,
         location: Location,

src/catalog/src/kvbackend.rs

@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
-
 pub use client::{CachedMetaKvBackend, MetaKvBackend};
 
 mod client;
@@ -22,18 +20,3 @@ mod manager;
 #[cfg(feature = "testing")]
 pub mod mock;
 pub use manager::KvBackendCatalogManager;
-
-/// KvBackend cache invalidator
-#[async_trait::async_trait]
-pub trait KvCacheInvalidator: Send + Sync {
-    async fn invalidate_key(&self, key: &[u8]);
-}
-
-pub type KvCacheInvalidatorRef = Arc<dyn KvCacheInvalidator>;
-
-pub struct DummyKvCacheInvalidator;
-
-#[async_trait::async_trait]
-impl KvCacheInvalidator for DummyKvCacheInvalidator {
-    async fn invalidate_key(&self, _key: &[u8]) {}
-}

src/catalog/src/kvbackend/client.rs

@@ -18,6 +18,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use common_error::ext::BoxedError;
+use common_meta::cache_invalidator::KvCacheInvalidator;
 use common_meta::error::Error::{CacheNotGet, GetKvCache};
 use common_meta::error::{CacheNotGetSnafu, Error, ExternalSnafu, Result};
 use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
@@ -28,12 +29,11 @@ use common_meta::rpc::store::{
     RangeRequest, RangeResponse,
 };
 use common_meta::rpc::KeyValue;
-use common_telemetry::timer;
+use common_telemetry::{debug, timer};
 use meta_client::client::MetaClient;
 use moka::future::{Cache, CacheBuilder};
 use snafu::{OptionExt, ResultExt};
 
-use super::KvCacheInvalidator;
 use crate::metrics::{METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET};
 
 const CACHE_MAX_CAPACITY: u64 = 10000;
@@ -197,7 +197,8 @@ impl KvBackend for CachedMetaKvBackend {
 #[async_trait::async_trait]
 impl KvCacheInvalidator for CachedMetaKvBackend {
     async fn invalidate_key(&self, key: &[u8]) {
-        self.cache.invalidate(key).await
+        self.cache.invalidate(key).await;
+        debug!("invalidated cache key: {}", String::from_utf8_lossy(key));
     }
 }
 

src/catalog/src/kvbackend/manager.rs

@@ -18,18 +18,17 @@ use std::sync::{Arc, Weak};
 
 use common_catalog::consts::{DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, NUMBERS_TABLE_ID};
 use common_error::ext::BoxedError;
-use common_meta::cache_invalidator::{CacheInvalidator, Context};
+use common_meta::cache_invalidator::{
+    CacheInvalidator, Context, KvCacheInvalidatorRef, TableMetadataCacheInvalidator,
+};
 use common_meta::datanode_manager::DatanodeManagerRef;
 use common_meta::error::Result as MetaResult;
 use common_meta::key::catalog_name::CatalogNameKey;
 use common_meta::key::schema_name::SchemaNameKey;
-use common_meta::key::table_info::TableInfoKey;
 use common_meta::key::table_name::TableNameKey;
-use common_meta::key::table_route::TableRouteKey;
-use common_meta::key::{TableMetaKey, TableMetadataManager, TableMetadataManagerRef};
+use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
 use common_meta::table_name::TableName;
-use common_telemetry::debug;
 use futures_util::TryStreamExt;
 use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
 use snafu::prelude::*;
@@ -43,7 +42,6 @@ use crate::error::{
     TableMetadataManagerSnafu,
 };
 use crate::information_schema::{InformationSchemaProvider, COLUMNS, TABLES};
-use crate::kvbackend::KvCacheInvalidatorRef;
 use crate::CatalogManager;
 
 /// Access all existing catalog, schema and tables.
@@ -56,7 +54,7 @@ pub struct KvBackendCatalogManager {
     // TODO(LFC): Maybe use a real implementation for Standalone mode.
     // Now we use `NoopKvCacheInvalidator` for Standalone mode. In Standalone mode, the KV backend
     // is implemented by RaftEngine. Maybe we need a cache for it?
-    backend_cache_invalidator: KvCacheInvalidatorRef,
+    table_metadata_cache_invalidator: TableMetadataCacheInvalidator,
     partition_manager: PartitionRuleManagerRef,
     table_metadata_manager: TableMetadataManagerRef,
     datanode_manager: DatanodeManagerRef,
@@ -66,40 +64,16 @@ pub struct KvBackendCatalogManager {
 
 #[async_trait::async_trait]
 impl CacheInvalidator for KvBackendCatalogManager {
-    async fn invalidate_table_name(&self, _ctx: &Context, table_name: TableName) -> MetaResult<()> {
-        let key: TableNameKey = (&table_name).into();
-
-        self.backend_cache_invalidator
-            .invalidate_key(&key.as_raw_key())
-            .await;
-        debug!(
-            "invalidated cache key: {}",
-            String::from_utf8_lossy(&key.as_raw_key())
-        );
-
-        Ok(())
+    async fn invalidate_table_name(&self, ctx: &Context, table_name: TableName) -> MetaResult<()> {
+        self.table_metadata_cache_invalidator
+            .invalidate_table_name(ctx, table_name)
+            .await
     }
 
-    async fn invalidate_table_id(&self, _ctx: &Context, table_id: TableId) -> MetaResult<()> {
-        let key = TableInfoKey::new(table_id);
-        self.backend_cache_invalidator
-            .invalidate_key(&key.as_raw_key())
-            .await;
-        debug!(
-            "invalidated cache key: {}",
-            String::from_utf8_lossy(&key.as_raw_key())
-        );
-
-        let key = &TableRouteKey { table_id };
-        self.backend_cache_invalidator
-            .invalidate_key(&key.as_raw_key())
-            .await;
-        debug!(
-            "invalidated cache key: {}",
-            String::from_utf8_lossy(&key.as_raw_key())
-        );
-
-        Ok(())
+    async fn invalidate_table_id(&self, ctx: &Context, table_id: TableId) -> MetaResult<()> {
+        self.table_metadata_cache_invalidator
+            .invalidate_table_id(ctx, table_id)
+            .await
     }
 }
 
@@ -110,9 +84,11 @@ impl KvBackendCatalogManager {
         datanode_manager: DatanodeManagerRef,
     ) -> Arc<Self> {
         Arc::new_cyclic(|me| Self {
-            backend_cache_invalidator,
             partition_manager: Arc::new(PartitionRuleManager::new(backend.clone())),
             table_metadata_manager: Arc::new(TableMetadataManager::new(backend)),
+            table_metadata_cache_invalidator: TableMetadataCacheInvalidator::new(
+                backend_cache_invalidator.clone(),
+            ),
             datanode_manager,
             system_catalog: SystemCatalog {
                 catalog_manager: me.clone(),
@@ -133,9 +109,9 @@ impl KvBackendCatalogManager {
     }
 
     pub async fn invalidate_schema(&self, catalog: &str, schema: &str) {
-        let key = SchemaNameKey::new(catalog, schema).as_raw_key();
-
-        self.backend_cache_invalidator.invalidate_key(&key).await;
+        self.table_metadata_cache_invalidator
+            .invalidate_schema(catalog, schema)
+            .await
     }
 }
 

src/catalog/src/lib.rs

@@ -20,10 +20,8 @@ use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use api::v1::meta::RegionStat;
-use common_telemetry::warn;
 use futures::future::BoxFuture;
-use table::metadata::{TableId, TableType};
+use table::metadata::TableId;
 use table::requests::CreateTableRequest;
 use table::TableRef;
 
@@ -124,66 +122,3 @@ pub struct RegisterSchemaRequest {
     pub catalog: String,
     pub schema: String,
 }
-
-/// The stat of regions in the datanode node.
-/// The number of regions can be got from len of vec.
-///
-/// Ignores any errors occurred during iterating regions. The intention of this method is to
-/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
-/// "try our best" job.
-pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
-    let mut region_number: u64 = 0;
-    let mut region_stats = Vec::new();
-
-    let Ok(catalog_names) = catalog_manager.catalog_names().await else {
-        return (region_number, region_stats);
-    };
-    for catalog_name in catalog_names {
-        let Ok(schema_names) = catalog_manager.schema_names(&catalog_name).await else {
-            continue;
-        };
-        for schema_name in schema_names {
-            let Ok(table_names) = catalog_manager
-                .table_names(&catalog_name, &schema_name)
-                .await
-            else {
-                continue;
-            };
-            for table_name in table_names {
-                let Ok(Some(table)) = catalog_manager
-                    .table(&catalog_name, &schema_name, &table_name)
-                    .await
-                else {
-                    continue;
-                };
-
-                if table.table_type() != TableType::Base {
-                    continue;
-                }
-
-                let table_info = table.table_info();
-                let region_numbers = &table_info.meta.region_numbers;
-                region_number += region_numbers.len() as u64;
-
-                let engine = &table_info.meta.engine;
-
-                match table.region_stats() {
-                    Ok(stats) => {
-                        let stats = stats.into_iter().map(|stat| RegionStat {
-                            region_id: stat.region_id,
-                            approximate_bytes: stat.disk_usage_bytes as i64,
-                            engine: engine.clone(),
-                            ..Default::default()
-                        });
-
-                        region_stats.extend(stats);
-                    }
-                    Err(e) => {
-                        warn!("Failed to get region status, err: {:?}", e);
-                    }
-                };
-            }
-        }
-    }
-    (region_number, region_stats)
-}

src/client/Cargo.toml

@@ -16,6 +16,7 @@ common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
@@ -26,7 +27,7 @@ datatypes = { workspace = true }
 derive_builder.workspace = true
 enum_dispatch = "0.3"
 futures-util.workspace = true
-moka = { version = "0.9", features = ["future"] }
+moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 prost.workspace = true
 rand.workspace = true

src/client/examples/stream_ingest.rs

@@ -42,14 +42,14 @@ async fn run() {
         .insert(vec![to_insert_request(weather_records_1())])
         .await
     {
-        error!("Error: {e}");
+        error!("Error: {e:?}");
     }
 
     if let Err(e) = stream_inserter
         .insert(vec![to_insert_request(weather_records_2())])
         .await
     {
-        error!("Error: {e}");
+        error!("Error: {e:?}");
     }
 
     let result = stream_inserter.finish().await;
@@ -59,7 +59,7 @@ async fn run() {
             info!("Rows written: {rows}");
         }
         Err(e) => {
-            error!("Error: {e}");
+            error!("Error: {e:?}");
         }
     };
 }

src/client/src/client.rs

@@ -138,8 +138,12 @@ impl Client {
         Ok((addr, channel))
     }
 
-    fn max_grpc_message_size(&self) -> usize {
-        self.inner.channel_manager.config().max_message_size
+    fn max_grpc_recv_message_size(&self) -> usize {
+        self.inner.channel_manager.config().max_recv_message_size
+    }
+
+    fn max_grpc_send_message_size(&self) -> usize {
+        self.inner.channel_manager.config().max_send_message_size
     }
 
     pub(crate) fn make_flight_client(&self) -> Result<FlightClient> {
@@ -147,7 +151,8 @@ impl Client {
         Ok(FlightClient {
             addr,
             client: FlightServiceClient::new(channel)
-                .max_decoding_message_size(self.max_grpc_message_size()),
+                .max_decoding_message_size(self.max_grpc_recv_message_size())
+                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
@@ -155,13 +160,16 @@ impl Client {
         let (_, channel) = self.find_channel()?;
         Ok(DatabaseClient {
             inner: GreptimeDatabaseClient::new(channel)
-                .max_decoding_message_size(self.max_grpc_message_size()),
+                .max_decoding_message_size(self.max_grpc_recv_message_size())
+                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
     pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
         let (_, channel) = self.find_channel()?;
-        Ok(PbRegionClient::new(channel).max_decoding_message_size(self.max_grpc_message_size()))
+        Ok(PbRegionClient::new(channel)
+            .max_decoding_message_size(self.max_grpc_recv_message_size())
+            .max_encoding_message_size(self.max_grpc_send_message_size()))
     }
 
     pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {

src/client/src/database.rs

@@ -276,7 +276,7 @@ impl Database {
                 source: BoxedError::new(ServerSnafu { code, msg }.build()),
             };
             logging::error!(
-                "Failed to do Flight get, addr: {}, code: {}, source: {}",
+                "Failed to do Flight get, addr: {}, code: {}, source: {:?}",
                 client.addr(),
                 tonic_code,
                 error

src/client/src/error.rs

@@ -17,39 +17,37 @@ use std::any::Any;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tonic::{Code, Status};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Illegal Flight messages, reason: {}", reason))]
     IllegalFlightMessages { reason: String, location: Location },
 
-    #[snafu(display("Failed to do Flight get, code: {}, source: {}", tonic_code, source))]
+    #[snafu(display("Failed to do Flight get, code: {}", tonic_code))]
     FlightGet {
         addr: String,
         tonic_code: Code,
         source: BoxedError,
     },
 
-    #[snafu(display(
-        "Failure occurs during handling request, location: {}, source: {}",
-        location,
-        source
-    ))]
+    #[snafu(display("Failure occurs during handling request"))]
     HandleRequest {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to convert FlightData, source: {}", source))]
+    #[snafu(display("Failed to convert FlightData"))]
     ConvertFlightData {
         location: Location,
         source: common_grpc::Error,
     },
 
-    #[snafu(display("Column datatype error, source: {}", source))]
+    #[snafu(display("Column datatype error"))]
     ColumnDataType {
         location: Location,
         source: api::error::Error,
@@ -61,18 +59,14 @@ pub enum Error {
     #[snafu(display("Missing required field in protobuf, field: {}", field))]
     MissingField { field: String, location: Location },
 
-    #[snafu(display(
-        "Failed to create gRPC channel, peer address: {}, source: {}",
-        addr,
-        source
-    ))]
+    #[snafu(display("Failed to create gRPC channel, peer address: {}", addr))]
     CreateChannel {
         addr: String,
         location: Location,
         source: common_grpc::error::Error,
     },
 
-    #[snafu(display("Failed to request RegionServer, code: {}, source: {}", code, source))]
+    #[snafu(display("Failed to request RegionServer, code: {}", code))]
     RegionServer { code: Code, source: BoxedError },
 
     // Server error carried in Tonic Status's metadata.

src/cmd/Cargo.toml

@@ -26,6 +26,7 @@ common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-config = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-procedure = { workspace = true }
 common-query = { workspace = true }
@@ -38,12 +39,14 @@ datanode = { workspace = true }
 datatypes = { workspace = true }
 either = "1.8"
 etcd-client.workspace = true
+file-engine = { workspace = true }
 frontend = { workspace = true }
 futures.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 meta-srv = { workspace = true }
 metrics.workspace = true
+mito2 = { workspace = true }
 nu-ansi-term = "0.46"
 partition = { workspace = true }
 prost.workspace = true

src/cmd/src/cli/repl.rs

@@ -35,7 +35,7 @@ use query::QueryEngine;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 use session::context::QueryContext;
-use snafu::{ErrorCompat, ResultExt};
+use snafu::ResultExt;
 use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 
 use crate::cli::cmd::ReplCommand;
@@ -148,7 +148,7 @@ impl Repl {
             .await
             .map_err(|e| {
                 let status_code = e.status_code();
-                let root_cause = e.iter_chain().last().unwrap();
+                let root_cause = e.output_msg();
                 println!("Error: {}({status_code}), {root_cause}", status_code as u32)
             })
             .is_ok()
@@ -261,6 +261,7 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
     let state = Arc::new(QueryEngineState::new(
         catalog_list,
         None,
+        None,
         false,
         plugins.clone(),
     ));

src/cmd/src/datanode.rs

@@ -255,6 +255,7 @@ mod tests {
             connect_timeout_millis,
             tcp_nodelay,
             ddl_timeout_millis,
+            ..
         } = options.meta_client.unwrap();
 
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);

src/cmd/src/error.rs

@@ -16,62 +16,64 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use config::ConfigError;
 use rustyline::error::ReadlineError;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to create default catalog and schema, source: {}", source))]
+    #[snafu(display("Failed to create default catalog and schema"))]
     InitMetadata {
         location: Location,
         source: common_meta::error::Error,
     },
 
-    #[snafu(display("Failed to iter stream, source: {}", source))]
+    #[snafu(display("Failed to iter stream"))]
     IterStream {
         location: Location,
         source: common_meta::error::Error,
     },
 
-    #[snafu(display("Failed to start datanode, source: {}", source))]
+    #[snafu(display("Failed to start datanode"))]
     StartDatanode {
         location: Location,
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown datanode, source: {}", source))]
+    #[snafu(display("Failed to shutdown datanode"))]
     ShutdownDatanode {
         location: Location,
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Failed to start frontend, source: {}", source))]
+    #[snafu(display("Failed to start frontend"))]
     StartFrontend {
         location: Location,
         source: frontend::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown frontend, source: {}", source))]
+    #[snafu(display("Failed to shutdown frontend"))]
     ShutdownFrontend {
         location: Location,
         source: frontend::error::Error,
     },
 
-    #[snafu(display("Failed to build meta server, source: {}", source))]
+    #[snafu(display("Failed to build meta server"))]
     BuildMetaServer {
         location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to start meta server, source: {}", source))]
+    #[snafu(display("Failed to start meta server"))]
     StartMetaServer {
         location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown meta server, source: {}", source))]
+    #[snafu(display("Failed to shutdown meta server"))]
     ShutdownMetaServer {
         location: Location,
         source: meta_srv::error::Error,
@@ -83,13 +85,13 @@ pub enum Error {
     #[snafu(display("Illegal config: {}", msg))]
     IllegalConfig { msg: String, location: Location },
 
-    #[snafu(display("Illegal auth config: {}", source))]
+    #[snafu(display("Illegal auth config"))]
     IllegalAuthConfig {
         location: Location,
         source: auth::error::Error,
     },
 
-    #[snafu(display("Unsupported selector type, {} source: {}", selector_type, source))]
+    #[snafu(display("Unsupported selector type: {}", selector_type))]
     UnsupportedSelectorType {
         selector_type: String,
         location: Location,
@@ -99,78 +101,89 @@ pub enum Error {
     #[snafu(display("Invalid REPL command: {reason}"))]
     InvalidReplCommand { reason: String },
 
-    #[snafu(display("Cannot create REPL: {}", source))]
+    #[snafu(display("Cannot create REPL"))]
     ReplCreation {
-        source: ReadlineError,
+        #[snafu(source)]
+        error: ReadlineError,
         location: Location,
     },
 
-    #[snafu(display("Error reading command: {}", source))]
+    #[snafu(display("Error reading command"))]
     Readline {
-        source: ReadlineError,
+        #[snafu(source)]
+        error: ReadlineError,
         location: Location,
     },
 
-    #[snafu(display("Failed to request database, sql: {sql}, source: {source}"))]
+    #[snafu(display("Failed to request database, sql: {sql}"))]
     RequestDatabase {
         sql: String,
         location: Location,
         source: client::Error,
     },
 
-    #[snafu(display("Failed to collect RecordBatches, source: {source}"))]
+    #[snafu(display("Failed to collect RecordBatches"))]
     CollectRecordBatches {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to pretty print Recordbatches, source: {source}"))]
+    #[snafu(display("Failed to pretty print Recordbatches"))]
     PrettyPrintRecordBatches {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to start Meta client, source: {}", source))]
+    #[snafu(display("Failed to start Meta client"))]
     StartMetaClient {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Failed to parse SQL: {}, source: {}", sql, source))]
+    #[snafu(display("Failed to parse SQL: {}", sql))]
     ParseSql {
         sql: String,
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to plan statement, source: {}", source))]
+    #[snafu(display("Failed to plan statement"))]
     PlanStatement {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to encode logical plan in substrait, source: {}", source))]
+    #[snafu(display("Failed to encode logical plan in substrait"))]
     SubstraitEncodeLogicalPlan {
         location: Location,
         source: substrait::error::Error,
     },
 
-    #[snafu(display("Failed to load layered config, source: {}", source))]
+    #[snafu(display("Failed to load layered config"))]
     LoadLayeredConfig {
-        source: ConfigError,
+        #[snafu(source)]
+        error: ConfigError,
         location: Location,
     },
 
-    #[snafu(display("Failed to start catalog manager, source: {}", source))]
+    #[snafu(display("Failed to start catalog manager"))]
     StartCatalogManager {
         location: Location,
         source: catalog::error::Error,
     },
 
-    #[snafu(display("Failed to connect to Etcd at {etcd_addr}, source: {}", source))]
+    #[snafu(display("Failed to connect to Etcd at {etcd_addr}"))]
     ConnectEtcd {
         etcd_addr: String,
-        source: etcd_client::Error,
+        #[snafu(source)]
+        error: etcd_client::Error,
+        location: Location,
+    },
+
+    #[snafu(display("Failed to serde json"))]
+    SerdeJson {
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 }
@@ -208,6 +221,8 @@ impl ErrorExt for Error {
             }
             Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
             Error::StartCatalogManager { source, .. } => source.status_code(),
+
+            Error::SerdeJson { .. } => StatusCode::Unexpected,
         }
     }
 

src/cmd/src/frontend.rs

@@ -383,7 +383,7 @@ mod tests {
                     Some("11"),
                 ),
                 (
-                    // http_options.addr = 127.0.0.1:24000
+                    // http.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
                         "http".to_uppercase(),

src/cmd/src/options.rs

@@ -21,7 +21,7 @@ use meta_srv::metasrv::MetaSrvOptions;
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 
-use crate::error::{LoadLayeredConfigSnafu, Result};
+use crate::error::{LoadLayeredConfigSnafu, Result, SerdeJsonSnafu};
 
 pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
@@ -29,11 +29,11 @@ pub const ENV_LIST_SEP: &str = ",";
 /// Options mixed up from datanode, frontend and metasrv.
 pub struct MixOptions {
     pub data_home: String,
-    pub procedure_cfg: ProcedureConfig,
-    pub kv_store_cfg: KvStoreConfig,
-    pub fe_opts: FrontendOptions,
-    pub dn_opts: DatanodeOptions,
-    pub logging_opts: LoggingOptions,
+    pub procedure: ProcedureConfig,
+    pub kv_store: KvStoreConfig,
+    pub frontend: FrontendOptions,
+    pub datanode: DatanodeOptions,
+    pub logging: LoggingOptions,
 }
 
 pub enum Options {
@@ -56,7 +56,7 @@ impl Options {
             Options::Datanode(opts) => &opts.logging,
             Options::Frontend(opts) => &opts.logging,
             Options::Metasrv(opts) => &opts.logging,
-            Options::Standalone(opts) => &opts.logging_opts,
+            Options::Standalone(opts) => &opts.logging,
             Options::Cli(opts) => opts,
         }
     }
@@ -94,9 +94,16 @@ impl Options {
                 .ignore_empty(true)
         };
 
+        // Workaround: Replacement for `Config::try_from(&default_opts)` due to
+        // `ConfigSerializer` cannot handle the case of an empty struct contained
+        // within an iterative structure.
+        // See: https://github.com/mehcode/config-rs/issues/461
+        let json_str = serde_json::to_string(&default_opts).context(SerdeJsonSnafu)?;
+        let default_config = File::from_str(&json_str, FileFormat::Json);
+
         // Add default values and environment variables as the sources of the configuration.
         let mut layered_config = Config::builder()
-            .add_source(Config::try_from(&default_opts).context(LoadLayeredConfigSnafu)?)
+            .add_source(default_config)
             .add_source(env_source);
 
         // Add config file as the source of the configuration if it is specified.

src/cmd/src/standalone.rs

@@ -14,23 +14,26 @@
 
 use std::sync::Arc;
 
-use catalog::kvbackend::{DummyKvCacheInvalidator, KvBackendCatalogManager};
+use catalog::kvbackend::KvBackendCatalogManager;
 use catalog::CatalogManagerRef;
 use clap::Parser;
 use common_base::Plugins;
 use common_config::{kv_store_dir, KvStoreConfig, WalConfig};
+use common_meta::cache_invalidator::DummyKvCacheInvalidator;
 use common_meta::kv_backend::KvBackendRef;
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
-use datanode::config::{DatanodeOptions, ProcedureConfig, StorageConfig};
+use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use datanode::region_server::RegionServer;
+use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
+use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -84,18 +87,20 @@ impl SubCommand {
 pub struct StandaloneOptions {
     pub mode: Mode,
     pub enable_telemetry: bool,
-    pub http_options: HttpOptions,
-    pub grpc_options: GrpcOptions,
-    pub mysql_options: MysqlOptions,
-    pub postgres_options: PostgresOptions,
-    pub opentsdb_options: OpentsdbOptions,
-    pub influxdb_options: InfluxdbOptions,
-    pub prom_store_options: PromStoreOptions,
+    pub http: HttpOptions,
+    pub grpc: GrpcOptions,
+    pub mysql: MysqlOptions,
+    pub postgres: PostgresOptions,
+    pub opentsdb: OpentsdbOptions,
+    pub influxdb: InfluxdbOptions,
+    pub prom_store: PromStoreOptions,
     pub wal: WalConfig,
     pub storage: StorageConfig,
     pub kv_store: KvStoreConfig,
     pub procedure: ProcedureConfig,
     pub logging: LoggingOptions,
+    /// Options for different store engines.
+    pub region_engine: Vec<RegionEngineConfig>,
 }
 
 impl Default for StandaloneOptions {
@@ -103,18 +108,22 @@ impl Default for StandaloneOptions {
         Self {
             mode: Mode::Standalone,
             enable_telemetry: true,
-            http_options: HttpOptions::default(),
-            grpc_options: GrpcOptions::default(),
-            mysql_options: MysqlOptions::default(),
-            postgres_options: PostgresOptions::default(),
-            opentsdb_options: OpentsdbOptions::default(),
-            influxdb_options: InfluxdbOptions::default(),
-            prom_store_options: PromStoreOptions::default(),
+            http: HttpOptions::default(),
+            grpc: GrpcOptions::default(),
+            mysql: MysqlOptions::default(),
+            postgres: PostgresOptions::default(),
+            opentsdb: OpentsdbOptions::default(),
+            influxdb: InfluxdbOptions::default(),
+            prom_store: PromStoreOptions::default(),
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
             kv_store: KvStoreConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
+            region_engine: vec![
+                RegionEngineConfig::Mito(MitoConfig::default()),
+                RegionEngineConfig::File(FileEngineConfig::default()),
+            ],
         }
     }
 }
@@ -123,13 +132,13 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
-            http: self.http_options,
-            grpc: self.grpc_options,
-            mysql: self.mysql_options,
-            postgres: self.postgres_options,
-            opentsdb: self.opentsdb_options,
-            influxdb: self.influxdb_options,
-            prom_store: self.prom_store_options,
+            http: self.http,
+            grpc: self.grpc,
+            mysql: self.mysql,
+            postgres: self.postgres,
+            opentsdb: self.opentsdb,
+            influxdb: self.influxdb,
+            prom_store: self.prom_store,
             meta_client: None,
             logging: self.logging,
             ..Default::default()
@@ -142,6 +151,7 @@ impl StandaloneOptions {
             enable_telemetry: self.enable_telemetry,
             wal: self.wal,
             storage: self.storage,
+            region_engine: self.region_engine,
             ..Default::default()
         }
     }
@@ -231,7 +241,7 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            opts.http_options.addr = addr.clone()
+            opts.http.addr = addr.clone()
         }
 
         if let Some(addr) = &self.rpc_addr {
@@ -245,42 +255,42 @@ impl StartCommand {
                 }
                 .fail();
             }
-            opts.grpc_options.addr = addr.clone()
+            opts.grpc.addr = addr.clone()
         }
 
         if let Some(addr) = &self.mysql_addr {
-            opts.mysql_options.enable = true;
-            opts.mysql_options.addr = addr.clone();
-            opts.mysql_options.tls = tls_opts.clone();
+            opts.mysql.enable = true;
+            opts.mysql.addr = addr.clone();
+            opts.mysql.tls = tls_opts.clone();
         }
 
         if let Some(addr) = &self.postgres_addr {
-            opts.postgres_options.enable = true;
-            opts.postgres_options.addr = addr.clone();
-            opts.postgres_options.tls = tls_opts;
+            opts.postgres.enable = true;
+            opts.postgres.addr = addr.clone();
+            opts.postgres.tls = tls_opts;
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            opts.opentsdb_options.enable = true;
-            opts.opentsdb_options.addr = addr.clone();
+            opts.opentsdb.enable = true;
+            opts.opentsdb.addr = addr.clone();
         }
 
         if self.influxdb_enable {
-            opts.influxdb_options.enable = self.influxdb_enable;
+            opts.influxdb.enable = self.influxdb_enable;
         }
-        let kv_store_cfg = opts.kv_store.clone();
-        let procedure_cfg = opts.procedure.clone();
-        let fe_opts = opts.clone().frontend_options();
-        let logging_opts = opts.logging.clone();
-        let dn_opts = opts.datanode_options();
+        let kv_store = opts.kv_store.clone();
+        let procedure = opts.procedure.clone();
+        let frontend = opts.clone().frontend_options();
+        let logging = opts.logging.clone();
+        let datanode = opts.datanode_options();
 
         Ok(Options::Standalone(Box::new(MixOptions {
-            procedure_cfg,
-            kv_store_cfg,
-            data_home: dn_opts.storage.data_home.to_string(),
-            fe_opts,
-            dn_opts,
-            logging_opts,
+            procedure,
+            kv_store,
+            data_home: datanode.storage.data_home.to_string(),
+            frontend,
+            datanode,
+            logging,
         })))
     }
 
@@ -289,8 +299,8 @@ impl StartCommand {
     #[allow(clippy::diverging_sub_expression)]
     async fn build(self, opts: MixOptions) -> Result<Instance> {
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
-        let fe_opts = opts.fe_opts;
-        let dn_opts = opts.dn_opts;
+        let fe_opts = opts.frontend;
+        let dn_opts = opts.datanode;
 
         info!("Standalone start command: {:#?}", self);
         info!(
@@ -299,13 +309,10 @@ impl StartCommand {
         );
 
         let kv_dir = kv_store_dir(&opts.data_home);
-        let (kv_store, procedure_manager) = FeInstance::try_build_standalone_components(
-            kv_dir,
-            opts.kv_store_cfg,
-            opts.procedure_cfg,
-        )
-        .await
-        .context(StartFrontendSnafu)?;
+        let (kv_store, procedure_manager) =
+            FeInstance::try_build_standalone_components(kv_dir, opts.kv_store, opts.procedure)
+                .await
+                .context(StartFrontendSnafu)?;
 
         let datanode =
             DatanodeBuilder::new(dn_opts.clone(), Some(kv_store.clone()), plugins.clone())
@@ -435,9 +442,9 @@ mod tests {
             checkpoint_margin = 9
             gc_duration = '7s'
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
-            timeout = "30s"
+            timeout = "33s"
             body_limit = "128MB"
 
             [logging]
@@ -455,12 +462,12 @@ mod tests {
         else {
             unreachable!()
         };
-        let fe_opts = options.fe_opts;
-        let dn_opts = options.dn_opts;
-        let logging_opts = options.logging_opts;
+        let fe_opts = options.frontend;
+        let dn_opts = options.datanode;
+        let logging_opts = options.logging;
         assert_eq!(Mode::Standalone, fe_opts.mode);
         assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
-        assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
+        assert_eq!(Duration::from_secs(33), fe_opts.http.timeout);
         assert_eq!(ReadableSize::mb(128), fe_opts.http.body_limit);
         assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.addr);
         assert!(fe_opts.mysql.enable);
@@ -502,8 +509,8 @@ mod tests {
             unreachable!()
         };
 
-        assert_eq!("/tmp/greptimedb/test/logs", opts.logging_opts.dir);
-        assert_eq!("debug", opts.logging_opts.level.unwrap());
+        assert_eq!("/tmp/greptimedb/test/logs", opts.logging.dir);
+        assert_eq!("debug", opts.logging.level.unwrap());
     }
 
     #[test]
@@ -547,7 +554,7 @@ mod tests {
                     // http.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http_options".to_uppercase(),
+                        "http".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -572,18 +579,39 @@ mod tests {
                 };
 
                 // Should be read from env, env > default values.
-                assert_eq!(opts.logging_opts.dir, "/other/log/dir");
+                assert_eq!(opts.logging.dir, "/other/log/dir");
 
                 // Should be read from config file, config file > env > default values.
-                assert_eq!(opts.logging_opts.level.as_ref().unwrap(), "debug");
+                assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.fe_opts.http.addr, "127.0.0.1:14000");
-                assert_eq!(ReadableSize::mb(64), opts.fe_opts.http.body_limit);
+                assert_eq!(opts.frontend.http.addr, "127.0.0.1:14000");
+                assert_eq!(ReadableSize::mb(64), opts.frontend.http.body_limit);
 
                 // Should be default value.
-                assert_eq!(opts.fe_opts.grpc.addr, GrpcOptions::default().addr);
+                assert_eq!(opts.frontend.grpc.addr, GrpcOptions::default().addr);
             },
         );
     }
+
+    #[test]
+    fn test_load_default_standalone_options() {
+        let options: StandaloneOptions =
+            Options::load_layered_options(None, "GREPTIMEDB_FRONTEND", None).unwrap();
+        let default_options = StandaloneOptions::default();
+        assert_eq!(options.mode, default_options.mode);
+        assert_eq!(options.enable_telemetry, default_options.enable_telemetry);
+        assert_eq!(options.http, default_options.http);
+        assert_eq!(options.grpc, default_options.grpc);
+        assert_eq!(options.mysql, default_options.mysql);
+        assert_eq!(options.postgres, default_options.postgres);
+        assert_eq!(options.opentsdb, default_options.opentsdb);
+        assert_eq!(options.influxdb, default_options.influxdb);
+        assert_eq!(options.prom_store, default_options.prom_store);
+        assert_eq!(options.wal, default_options.wal);
+        assert_eq!(options.kv_store, default_options.kv_store);
+        assert_eq!(options.procedure, default_options.procedure);
+        assert_eq!(options.logging, default_options.logging);
+        assert_eq!(options.region_engine, default_options.region_engine);
+    }
 }

src/common/base/Cargo.toml

@@ -9,6 +9,7 @@ anymap = "1.0.0-beta.2"
 bitvec = "1.0"
 bytes = { version = "1.1", features = ["serde"] }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 paste = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 snafu.workspace = true

src/common/base/src/buffer.rs

@@ -17,11 +17,13 @@ use std::io::{Read, Write};
 
 use bytes::{Buf, BufMut, BytesMut};
 use common_error::ext::ErrorExt;
+use common_macro::stack_trace_debug;
 use paste::paste;
 use snafu::{ensure, Location, ResultExt, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display(
         "Destination buffer overflow, src_len: {}, dst_len: {}",
@@ -37,9 +39,10 @@ pub enum Error {
     #[snafu(display("Buffer underflow"))]
     Underflow { location: Location },
 
-    #[snafu(display("IO operation reach EOF, source: {}", source))]
+    #[snafu(display("IO operation reach EOF"))]
     Eof {
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 }

src/common/catalog/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 common-error = { workspace = true }
+common-macro = { workspace = true }
 serde.workspace = true
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }

src/common/catalog/src/error.rs

@@ -16,10 +16,12 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid full table name: {}", table_name))]
     InvalidFullTableName {

src/common/catalog/src/lib.rs

@@ -13,9 +13,6 @@
 // limitations under the License.
 
 use consts::DEFAULT_CATALOG_NAME;
-use snafu::ensure;
-
-use crate::error::Result;
 
 pub mod consts;
 pub mod error;
@@ -26,17 +23,6 @@ pub fn format_full_table_name(catalog: &str, schema: &str, table: &str) -> Strin
     format!("{catalog}.{schema}.{table}")
 }
 
-pub fn parse_full_table_name(table_name: &str) -> Result<(&str, &str, &str)> {
-    let result = table_name.split('.').collect::<Vec<_>>();
-
-    ensure!(
-        result.len() == 3,
-        error::InvalidFullTableNameSnafu { table_name }
-    );
-
-    Ok((result[0], result[1], result[2]))
-}
-
 /// Build db name from catalog and schema string
 pub fn build_db_string(catalog: &str, schema: &str) -> String {
     if catalog == DEFAULT_CATALOG_NAME {

src/common/config/src/lib.rs

@@ -17,7 +17,7 @@ use std::time::Duration;
 use common_base::readable_size::ReadableSize;
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct WalConfig {
     // wal file size in bytes
@@ -49,7 +49,7 @@ pub fn kv_store_dir(store_dir: &str) -> String {
     format!("{store_dir}/kv")
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct KvStoreConfig {
     // Kv file size in bytes

src/common/datasource/Cargo.toml

@@ -18,10 +18,12 @@ async-compression = { version = "0.3", features = [
 async-trait.workspace = true
 bytes = "1.1"
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-runtime = { workspace = true }
 datafusion.workspace = true
 derive_builder.workspace = true
 futures.workspace = true
+lazy_static.workspace = true
 object-store = { workspace = true }
 orc-rust = "0.2"
 paste = "1.0"

src/common/datasource/src/error.rs

@@ -17,12 +17,14 @@ use std::any::Any;
 use arrow_schema::ArrowError;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::parquet::errors::ParquetError;
 use snafu::{Location, Snafu};
 use url::ParseError;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unsupported compression type: {}", compression_type))]
     UnsupportedCompressionType {
@@ -30,10 +32,11 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Unsupported backend protocol: {}", protocol))]
+    #[snafu(display("Unsupported backend protocol: {}, url: {}", protocol, url))]
     UnsupportedBackendProtocol {
         protocol: String,
         location: Location,
+        url: String,
     },
 
     #[snafu(display("Unsupported format protocol: {}", format))]
@@ -42,95 +45,109 @@ pub enum Error {
     #[snafu(display("empty host: {}", url))]
     EmptyHostPath { url: String, location: Location },
 
-    #[snafu(display("Invalid url: {}, error :{}", url, source))]
+    #[snafu(display("Invalid url: {}", url))]
     InvalidUrl {
         url: String,
-        source: ParseError,
+        #[snafu(source)]
+        error: ParseError,
         location: Location,
     },
 
-    #[snafu(display("Failed to build backend, source: {}", source))]
+    #[snafu(display("Failed to build backend"))]
     BuildBackend {
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to build orc reader, source: {}", source))]
+    #[snafu(display("Failed to build orc reader"))]
     OrcReader {
         location: Location,
-        source: orc_rust::error::Error,
+        #[snafu(source)]
+        error: orc_rust::error::Error,
     },
 
-    #[snafu(display("Failed to read object from path: {}, source: {}", path, source))]
+    #[snafu(display("Failed to read object from path: {}", path))]
     ReadObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
-    #[snafu(display("Failed to write object to path: {}, source: {}", path, source))]
+    #[snafu(display("Failed to write object to path: {}", path))]
     WriteObject {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
-    #[snafu(display("Failed to write: {}", source))]
+    #[snafu(display("Failed to write"))]
     AsyncWrite {
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to write record batch: {}", source))]
+    #[snafu(display("Failed to write record batch"))]
     WriteRecordBatch {
         location: Location,
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
     },
 
-    #[snafu(display("Failed to encode record batch: {}", source))]
+    #[snafu(display("Failed to encode record batch"))]
     EncodeRecordBatch {
         location: Location,
-        source: ParquetError,
+        #[snafu(source)]
+        error: ParquetError,
     },
 
-    #[snafu(display("Failed to read record batch: {}", source))]
+    #[snafu(display("Failed to read record batch"))]
     ReadRecordBatch {
         location: Location,
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
     },
 
-    #[snafu(display("Failed to read parquet source: {}", source))]
+    #[snafu(display("Failed to read parquet"))]
     ReadParquetSnafu {
         location: Location,
-        source: datafusion::parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: datafusion::parquet::errors::ParquetError,
     },
 
-    #[snafu(display("Failed to convert parquet to schema: {}", source))]
+    #[snafu(display("Failed to convert parquet to schema"))]
     ParquetToSchema {
         location: Location,
-        source: datafusion::parquet::errors::ParquetError,
+        #[snafu(source)]
+        error: datafusion::parquet::errors::ParquetError,
     },
 
-    #[snafu(display("Failed to infer schema from file, source: {}", source))]
+    #[snafu(display("Failed to infer schema from file"))]
     InferSchema {
         location: Location,
-        source: arrow_schema::ArrowError,
+        #[snafu(source)]
+        error: arrow_schema::ArrowError,
     },
 
-    #[snafu(display("Failed to list object in path: {}, source: {}", path, source))]
+    #[snafu(display("Failed to list object in path: {}", path))]
     ListObjects {
         path: String,
         location: Location,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
     #[snafu(display("Invalid connection: {}", msg))]
     InvalidConnection { msg: String, location: Location },
 
-    #[snafu(display("Failed to join handle: {}", source))]
+    #[snafu(display("Failed to join handle"))]
     JoinHandle {
         location: Location,
-        source: tokio::task::JoinError,
+        #[snafu(source)]
+        error: tokio::task::JoinError,
     },
 
     #[snafu(display("Failed to parse format {} with value: {}", key, value))]
@@ -140,9 +157,10 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to merge schema: {}", source))]
+    #[snafu(display("Failed to merge schema"))]
     MergeSchema {
-        source: arrow_schema::ArrowError,
+        #[snafu(source)]
+        error: arrow_schema::ArrowError,
         location: Location,
     },
 

src/common/datasource/src/lister.rs

@@ -27,7 +27,7 @@ pub enum Source {
 pub struct Lister {
     object_store: ObjectStore,
     source: Source,
-    path: String,
+    root: String,
     regex: Option<Regex>,
 }
 
@@ -35,13 +35,13 @@ impl Lister {
     pub fn new(
         object_store: ObjectStore,
         source: Source,
-        path: String,
+        root: String,
         regex: Option<Regex>,
     ) -> Self {
         Lister {
             object_store,
             source,
-            path,
+            root,
             regex,
         }
     }
@@ -51,9 +51,9 @@ impl Lister {
             Source::Dir => {
                 let streamer = self
                     .object_store
-                    .lister_with(&self.path)
+                    .lister_with("/")
                     .await
-                    .context(error::ListObjectsSnafu { path: &self.path })?;
+                    .context(error::ListObjectsSnafu { path: &self.root })?;
 
                 streamer
                     .try_filter(|f| {
@@ -66,22 +66,21 @@ impl Lister {
                     })
                     .try_collect::<Vec<_>>()
                     .await
-                    .context(error::ListObjectsSnafu { path: &self.path })
+                    .context(error::ListObjectsSnafu { path: &self.root })
             }
             Source::Filename(filename) => {
                 // make sure this file exists
-                let file_full_path = format!("{}{}", self.path, filename);
-                let _ = self.object_store.stat(&file_full_path).await.context(
+                let _ = self.object_store.stat(filename).await.with_context(|_| {
                     error::ListObjectsSnafu {
-                        path: &file_full_path,
-                    },
-                )?;
+                        path: format!("{}{}", &self.root, filename),
+                    }
+                })?;
 
                 Ok(self
                     .object_store
-                    .list_with(&self.path)
+                    .list_with("/")
                     .await
-                    .context(error::ListObjectsSnafu { path: &self.path })?
+                    .context(error::ListObjectsSnafu { path: &self.root })?
                     .into_iter()
                     .find(|f| f.name() == filename)
                     .map(|f| vec![f])

src/common/datasource/src/object_store.rs

@@ -16,19 +16,29 @@ pub mod fs;
 pub mod s3;
 use std::collections::HashMap;
 
+use lazy_static::lazy_static;
 use object_store::ObjectStore;
+use regex::Regex;
 use snafu::{OptionExt, ResultExt};
 use url::{ParseError, Url};
 
 use self::fs::build_fs_backend;
 use self::s3::build_s3_backend;
 use crate::error::{self, Result};
+use crate::util::find_dir_and_filename;
 
 pub const FS_SCHEMA: &str = "FS";
 pub const S3_SCHEMA: &str = "S3";
 
 /// Returns `(schema, Option<host>, path)`
 pub fn parse_url(url: &str) -> Result<(String, Option<String>, String)> {
+    #[cfg(windows)]
+    {
+        // On Windows, the url may start with `C:/`.
+        if let Some(_) = handle_windows_path(url) {
+            return Ok((FS_SCHEMA.to_string(), None, url.to_string()));
+        }
+    }
     let parsed_url = Url::parse(url);
     match parsed_url {
         Ok(url) => Ok((
@@ -44,17 +54,47 @@ pub fn parse_url(url: &str) -> Result<(String, Option<String>, String)> {
 }
 
 pub fn build_backend(url: &str, connection: &HashMap<String, String>) -> Result<ObjectStore> {
-    let (schema, host, _path) = parse_url(url)?;
+    let (schema, host, path) = parse_url(url)?;
+    let (root, _) = find_dir_and_filename(&path);
 
     match schema.to_uppercase().as_str() {
         S3_SCHEMA => {
             let host = host.context(error::EmptyHostPathSnafu {
                 url: url.to_string(),
             })?;
-            Ok(build_s3_backend(&host, "/", connection)?)
+            Ok(build_s3_backend(&host, &root, connection)?)
         }
-        FS_SCHEMA => Ok(build_fs_backend("/")?),
+        FS_SCHEMA => Ok(build_fs_backend(&root)?),
 
-        _ => error::UnsupportedBackendProtocolSnafu { protocol: schema }.fail(),
+        _ => error::UnsupportedBackendProtocolSnafu {
+            protocol: schema,
+            url,
+        }
+        .fail(),
+    }
+}
+
+lazy_static! {
+    static ref DISK_SYMBOL_PATTERN: Regex = Regex::new("^([A-Za-z]:/)").unwrap();
+}
+
+pub fn handle_windows_path(url: &str) -> Option<String> {
+    DISK_SYMBOL_PATTERN
+        .captures(url)
+        .map(|captures| captures[0].to_string())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::handle_windows_path;
+
+    #[test]
+    fn test_handle_windows_path() {
+        assert_eq!(
+            handle_windows_path("C:/to/path/file"),
+            Some("C:/".to_string())
+        );
+        assert_eq!(handle_windows_path("https://google.com"), None);
+        assert_eq!(handle_windows_path("s3://bucket/path/to"), None);
     }
 }

src/common/error/src/ext.rs

@@ -13,11 +13,12 @@
 // limitations under the License.
 
 use std::any::Any;
+use std::sync::Arc;
 
 use crate::status_code::StatusCode;
 
 /// Extension to [`Error`](std::error::Error) in std.
-pub trait ErrorExt: std::error::Error {
+pub trait ErrorExt: StackError {
     /// Map this error to [StatusCode].
     fn status_code(&self) -> StatusCode {
         StatusCode::Unknown
@@ -33,6 +34,63 @@ pub trait ErrorExt: std::error::Error {
     /// Returns the error as [Any](std::any::Any) so that it can be
     /// downcast to a specific implementation.
     fn as_any(&self) -> &dyn Any;
+
+    fn output_msg(&self) -> String
+    where
+        Self: Sized,
+    {
+        let error = self.last();
+        if let Some(external_error) = error.source() {
+            let external_root = external_error.sources().last().unwrap();
+
+            if error.to_string().is_empty() {
+                format!("{external_root}")
+            } else {
+                format!("{error}: {external_root}")
+            }
+        } else {
+            format!("{error}")
+        }
+    }
+}
+
+pub trait StackError: std::error::Error {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>);
+
+    fn next(&self) -> Option<&dyn StackError>;
+
+    fn last(&self) -> &dyn StackError
+    where
+        Self: Sized,
+    {
+        let Some(mut result) = self.next() else {
+            return self;
+        };
+        while let Some(err) = result.next() {
+            result = err;
+        }
+        result
+    }
+}
+
+impl<T: ?Sized + StackError> StackError for Arc<T> {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+        self.as_ref().debug_fmt(layer, buf)
+    }
+
+    fn next(&self) -> Option<&dyn StackError> {
+        self.as_ref().next()
+    }
+}
+
+impl<T: StackError> StackError for Box<T> {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+        self.as_ref().debug_fmt(layer, buf)
+    }
+
+    fn next(&self) -> Option<&dyn StackError> {
+        self.as_ref().next()
+    }
 }
 
 /// An opaque boxed error based on errors that implement [ErrorExt] trait.
@@ -90,6 +148,16 @@ impl crate::snafu::ErrorCompat for BoxedError {
     }
 }
 
+impl StackError for BoxedError {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+        self.inner.debug_fmt(layer, buf)
+    }
+
+    fn next(&self) -> Option<&dyn StackError> {
+        self.inner.next()
+    }
+}
+
 /// Error type with plain error message
 #[derive(Debug)]
 pub struct PlainError {
@@ -128,3 +196,13 @@ impl crate::ext::ErrorExt for PlainError {
         self as _
     }
 }
+
+impl StackError for PlainError {
+    fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+        buf.push(format!("{}: {}", layer, self.msg))
+    }
+
+    fn next(&self) -> Option<&dyn StackError> {
+        None
+    }
+}

src/common/error/src/format.rs

@@ -50,6 +50,7 @@ mod tests {
     use snafu::{GenerateImplicitData, Location};
 
     use super::*;
+    use crate::ext::StackError;
 
     #[derive(Debug, Snafu)]
     #[snafu(display("This is a leaf error"))]
@@ -65,6 +66,14 @@ mod tests {
         }
     }
 
+    impl StackError for Leaf {
+        fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
+
+        fn next(&self) -> Option<&dyn StackError> {
+            None
+        }
+    }
+
     #[derive(Debug, Snafu)]
     #[snafu(display("This is a leaf with location"))]
     struct LeafWithLocation {
@@ -81,6 +90,14 @@ mod tests {
         }
     }
 
+    impl StackError for LeafWithLocation {
+        fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
+
+        fn next(&self) -> Option<&dyn StackError> {
+            None
+        }
+    }
+
     #[derive(Debug, Snafu)]
     #[snafu(display("Internal error"))]
     struct Internal {
@@ -99,6 +116,17 @@ mod tests {
         }
     }
 
+    impl StackError for Internal {
+        fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+            buf.push(format!("{}: Internal error, at {}", layer, self.location));
+            self.source.debug_fmt(layer + 1, buf);
+        }
+
+        fn next(&self) -> Option<&dyn StackError> {
+            Some(&self.source)
+        }
+    }
+
     #[test]
     fn test_debug_format() {
         let err = Leaf;

src/common/error/src/lib.rs

@@ -11,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![feature(error_iter)]
 
 pub mod ext;
 pub mod format;

src/common/error/src/mock.rs

@@ -19,7 +19,7 @@ use std::fmt;
 
 use snafu::Location;
 
-use crate::ext::ErrorExt;
+use crate::ext::{ErrorExt, StackError};
 use crate::status_code::StatusCode;
 
 /// A mock error mainly for test.
@@ -69,3 +69,11 @@ impl ErrorExt for MockError {
         self
     }
 }
+
+impl StackError for MockError {
+    fn debug_fmt(&self, _: usize, _: &mut Vec<String>) {}
+
+    fn next(&self) -> Option<&dyn StackError> {
+        None
+    }
+}

src/common/function/src/scalars/timestamp.rs

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use std::sync::Arc;
+mod greatest;
 mod to_unixtime;
 
+use greatest::GreatestFunction;
 use to_unixtime::ToUnixtimeFunction;
 
 use crate::scalars::function_registry::FunctionRegistry;
@@ -23,5 +25,6 @@ pub(crate) struct TimestampFunction;
 impl TimestampFunction {
     pub fn register(registry: &FunctionRegistry) {
         registry.register(Arc::new(ToUnixtimeFunction));
+        registry.register(Arc::new(GreatestFunction));
     }
 }

src/common/function/src/scalars/timestamp/greatest.rs

@@ -0,0 +1,175 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{self};
+
+use common_query::error::{
+    self, ArrowComputeSnafu, InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu,
+};
+use common_query::prelude::{Signature, Volatility};
+use datatypes::arrow::array::AsArray;
+use datatypes::arrow::compute::cast;
+use datatypes::arrow::compute::kernels::comparison::gt_dyn;
+use datatypes::arrow::compute::kernels::zip;
+use datatypes::arrow::datatypes::{DataType as ArrowDataType, Date32Type};
+use datatypes::prelude::ConcreteDataType;
+use datatypes::vectors::{Helper, VectorRef};
+use snafu::{ensure, ResultExt};
+
+use crate::scalars::function::{Function, FunctionContext};
+
+#[derive(Clone, Debug, Default)]
+pub struct GreatestFunction;
+
+const NAME: &str = "greatest";
+
+impl Function for GreatestFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::date_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::uniform(
+            2,
+            vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::date_datatype(),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+        match columns[0].data_type() {
+            ConcreteDataType::String(_) => {
+                let column1 = cast(&columns[0].to_arrow_array(), &ArrowDataType::Date32)
+                    .context(ArrowComputeSnafu)?;
+                let column1 = column1.as_primitive::<Date32Type>();
+                let column2 = cast(&columns[1].to_arrow_array(), &ArrowDataType::Date32)
+                    .context(ArrowComputeSnafu)?;
+                let column2 = column2.as_primitive::<Date32Type>();
+                let boolean_array = gt_dyn(&column1, &column2).context(ArrowComputeSnafu)?;
+                let result =
+                    zip::zip(&boolean_array, &column1, &column2).context(ArrowComputeSnafu)?;
+                Ok(Helper::try_into_vector(&result).context(error::FromArrowArraySnafu)?)
+            }
+            ConcreteDataType::Date(_) => {
+                let column1 = columns[0].to_arrow_array();
+                let column1 = column1.as_primitive::<Date32Type>();
+                let column2 = columns[1].to_arrow_array();
+                let column2 = column2.as_primitive::<Date32Type>();
+                let boolean_array = gt_dyn(&column1, &column2).context(ArrowComputeSnafu)?;
+                let result =
+                    zip::zip(&boolean_array, &column1, &column2).context(ArrowComputeSnafu)?;
+                Ok(Helper::try_into_vector(&result).context(error::FromArrowArraySnafu)?)
+            }
+            _ => UnsupportedInputDataTypeSnafu {
+                function: NAME,
+                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+            }
+            .fail(),
+        }
+    }
+}
+
+impl fmt::Display for GreatestFunction {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "GREATEST")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::str::FromStr;
+    use std::sync::Arc;
+
+    use common_time::Date;
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::types::DateType;
+    use datatypes::value::Value;
+    use datatypes::vectors::{DateVector, StringVector, Vector};
+
+    use super::GreatestFunction;
+    use crate::scalars::function::FunctionContext;
+    use crate::scalars::Function;
+
+    #[test]
+    fn test_greatest_takes_string_vector() {
+        let function = GreatestFunction;
+        assert_eq!(
+            function.return_type(&[]).unwrap(),
+            ConcreteDataType::Date(DateType)
+        );
+        let columns = vec![
+            Arc::new(StringVector::from(vec![
+                "1970-01-01".to_string(),
+                "2012-12-23".to_string(),
+            ])) as _,
+            Arc::new(StringVector::from(vec![
+                "2001-02-01".to_string(),
+                "1999-01-01".to_string(),
+            ])) as _,
+        ];
+
+        let result = function.eval(FunctionContext::default(), &columns).unwrap();
+        let result = result.as_any().downcast_ref::<DateVector>().unwrap();
+        assert_eq!(result.len(), 2);
+        assert_eq!(
+            result.get(0),
+            Value::Date(Date::from_str("2001-02-01").unwrap())
+        );
+        assert_eq!(
+            result.get(1),
+            Value::Date(Date::from_str("2012-12-23").unwrap())
+        );
+    }
+
+    #[test]
+    fn test_greatest_takes_date_vector() {
+        let function = GreatestFunction;
+        assert_eq!(
+            function.return_type(&[]).unwrap(),
+            ConcreteDataType::Date(DateType)
+        );
+        let columns = vec![
+            Arc::new(DateVector::from_slice(vec![-1, 2])) as _,
+            Arc::new(DateVector::from_slice(vec![0, 1])) as _,
+        ];
+
+        let result = function.eval(FunctionContext::default(), &columns).unwrap();
+        let result = result.as_any().downcast_ref::<DateVector>().unwrap();
+        assert_eq!(result.len(), 2);
+        assert_eq!(
+            result.get(0),
+            Value::Date(Date::from_str("1970-01-01").unwrap())
+        );
+        assert_eq!(
+            result.get(1),
+            Value::Date(Date::from_str("1970-01-03").unwrap())
+        );
+    }
+}

src/common/grpc-expr/Cargo.toml

@@ -10,6 +10,7 @@ async-trait.workspace = true
 common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-query = { workspace = true }
 common-telemetry = { workspace = true }
 common-time = { workspace = true }

src/common/grpc-expr/src/error.rs

@@ -16,15 +16,17 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Illegal delete request, reason: {reason}"))]
     IllegalDeleteRequest { reason: String, location: Location },
 
-    #[snafu(display("Column datatype error, source: {}", source))]
+    #[snafu(display("Column datatype error"))]
     ColumnDataType {
         location: Location,
         source: api::error::Error,
@@ -49,7 +51,7 @@ pub enum Error {
 
     #[snafu(display("Invalid column proto: {}", err_msg))]
     InvalidColumnProto { err_msg: String, location: Location },
-    #[snafu(display("Failed to create vector, source: {}", source))]
+    #[snafu(display("Failed to create vector"))]
     CreateVector {
         location: Location,
         source: datatypes::error::Error,
@@ -58,11 +60,7 @@ pub enum Error {
     #[snafu(display("Missing required field in protobuf, field: {}", field))]
     MissingField { field: String, location: Location },
 
-    #[snafu(display(
-        "Invalid column proto definition, column: {}, source: {}",
-        column,
-        source
-    ))]
+    #[snafu(display("Invalid column proto definition, column: {}", column))]
     InvalidColumnDef {
         column: String,
         location: Location,

src/common/grpc/Cargo.toml

@@ -11,6 +11,7 @@ async-trait = "0.1"
 backtrace = "0.3"
 common-base = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-recordbatch = { workspace = true }
 common-runtime = { workspace = true }
 common-telemetry = { workspace = true }

src/common/grpc/src/channel_manager.rs

@@ -30,8 +30,9 @@ use crate::error::{CreateChannelSnafu, InvalidConfigFilePathSnafu, InvalidTlsCon
 
 const RECYCLE_CHANNEL_INTERVAL_SECS: u64 = 60;
 pub const DEFAULT_GRPC_REQUEST_TIMEOUT_SECS: u64 = 10;
-pub const DEFAULT_GRPC_CONNECT_TIMEOUT_SECS: u64 = 10;
-pub const DEFAULT_MAX_GRPC_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
+pub const DEFAULT_GRPC_CONNECT_TIMEOUT_SECS: u64 = 1;
+pub const DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
+pub const DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
 
 lazy_static! {
     static ref ID: AtomicU64 = AtomicU64::new(0);
@@ -248,9 +249,10 @@ pub struct ChannelConfig {
     pub tcp_keepalive: Option<Duration>,
     pub tcp_nodelay: bool,
     pub client_tls: Option<ClientTlsOption>,
-    // Max gRPC message size
-    // TODO(dennis): make it configurable
-    pub max_message_size: usize,
+    // Max gRPC receiving(decoding) message size
+    pub max_recv_message_size: usize,
+    // Max gRPC sending(encoding) message size
+    pub max_send_message_size: usize,
 }
 
 impl Default for ChannelConfig {
@@ -269,7 +271,8 @@ impl Default for ChannelConfig {
             tcp_keepalive: None,
             tcp_nodelay: true,
             client_tls: None,
-            max_message_size: DEFAULT_MAX_GRPC_MESSAGE_SIZE,
+            max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+            max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
         }
     }
 }
@@ -534,7 +537,8 @@ mod tests {
                 tcp_keepalive: None,
                 tcp_nodelay: true,
                 client_tls: None,
-                max_message_size: DEFAULT_MAX_GRPC_MESSAGE_SIZE,
+                max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+                max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
             },
             default_cfg
         );
@@ -577,7 +581,8 @@ mod tests {
                     client_cert_path: "some_cert_path".to_string(),
                     client_key_path: "some_key_path".to_string(),
                 }),
-                max_message_size: DEFAULT_MAX_GRPC_MESSAGE_SIZE,
+                max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+                max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
             },
             cfg
         );

src/common/grpc/src/error.rs

@@ -17,19 +17,22 @@ use std::io;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid client tls config, {}", msg))]
     InvalidTlsConfig { msg: String },
 
-    #[snafu(display("Invalid config file path, {}", source))]
+    #[snafu(display("Invalid config file path"))]
     InvalidConfigFilePath {
-        source: io::Error,
+        #[snafu(source)]
+        error: io::Error,
         location: Location,
     },
 
@@ -46,13 +49,14 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to create gRPC channel, source: {}", source))]
+    #[snafu(display("Failed to create gRPC channel"))]
     CreateChannel {
-        source: tonic::transport::Error,
+        #[snafu(source)]
+        error: tonic::transport::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to create RecordBatch, source: {}", source))]
+    #[snafu(display("Failed to create RecordBatch"))]
     CreateRecordBatch {
         location: Location,
         source: common_recordbatch::error::Error,
@@ -61,16 +65,17 @@ pub enum Error {
     #[snafu(display("Failed to convert Arrow type: {}", from))]
     Conversion { from: String, location: Location },
 
-    #[snafu(display("Failed to decode FlightData, source: {}", source))]
+    #[snafu(display("Failed to decode FlightData"))]
     DecodeFlightData {
-        source: api::DecodeError,
+        #[snafu(source)]
+        error: api::DecodeError,
         location: Location,
     },
 
     #[snafu(display("Invalid FlightData, reason: {}", reason))]
     InvalidFlightData { reason: String, location: Location },
 
-    #[snafu(display("Failed to convert Arrow Schema, source: {}", source))]
+    #[snafu(display("Failed to convert Arrow Schema"))]
     ConvertArrowSchema {
         location: Location,
         source: datatypes::error::Error,

src/common/macro/Cargo.toml

@@ -13,6 +13,15 @@ common-telemetry = { workspace = true }
 proc-macro2 = "1.0.66"
 quote = "1.0"
 syn = "1.0"
+syn2 = { version = "2.0", package = "syn", features = [
+    "derive",
+    "parsing",
+    "printing",
+    "clone-impls",
+    "proc-macro",
+    "extra-traits",
+    "full",
+] }
 
 [dev-dependencies]
 arc-swap = "1.0"

src/common/macro/src/lib.rs

@@ -15,6 +15,7 @@
 mod aggr_func;
 mod print_caller;
 mod range_fn;
+mod stack_trace_debug;
 
 use aggr_func::{impl_aggr_func_type_store, impl_as_aggr_func_creator};
 use print_caller::process_print_caller;
@@ -87,3 +88,23 @@ pub fn range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
 pub fn print_caller(args: TokenStream, input: TokenStream) -> TokenStream {
     process_print_caller(args, input)
 }
+
+/// Attribute macro to derive [std::fmt::Debug] for the annotated `Error` type.
+///
+/// The generated `Debug` implementation will print the error in a stack trace style. E.g.:
+/// ```plaintext
+/// 0: Foo error, at src/common/catalog/src/error.rs:80:10
+/// 1: Bar error, at src/common/function/src/error.rs:90:10
+/// 2: Root cause, invalid table name, at src/common/catalog/src/error.rs:100:10
+/// ```
+///
+/// Notes on using this macro:
+/// - `#[snafu(display)]` must present on each enum variants,
+///   and should not include `location` and `source`.
+/// - Only our internal error can be named `source`.
+///   All external error should be `error` with an `#[snafu(source)]` annotation.
+/// - `common_error` crate must be accessible.
+#[proc_macro_attribute]
+pub fn stack_trace_debug(args: TokenStream, input: TokenStream) -> TokenStream {
+    stack_trace_debug::stack_trace_style_impl(args.into(), input.into()).into()
+}

src/common/macro/src/stack_trace_debug.rs

@@ -0,0 +1,278 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! implement `::common_error::ext::StackError`
+
+use proc_macro2::{Span, TokenStream as TokenStream2};
+use quote::{quote, quote_spanned};
+use syn2::spanned::Spanned;
+use syn2::{parenthesized, Attribute, Ident, ItemEnum, Variant};
+
+pub fn stack_trace_style_impl(args: TokenStream2, input: TokenStream2) -> TokenStream2 {
+    let input_cloned: TokenStream2 = input.clone();
+
+    let error_enum_definition: ItemEnum = syn2::parse2(input_cloned).unwrap();
+    let enum_name = error_enum_definition.ident;
+
+    let mut variants = vec![];
+
+    for error_variant in error_enum_definition.variants {
+        let variant = ErrorVariant::from_enum_variant(error_variant);
+        variants.push(variant);
+    }
+
+    let debug_fmt_fn = build_debug_fmt_impl(enum_name.clone(), variants.clone());
+    let next_fn = build_next_impl(enum_name.clone(), variants);
+    let debug_impl = build_debug_impl(enum_name.clone());
+
+    quote! {
+        #args
+        #input
+
+        impl ::common_error::ext::StackError for #enum_name {
+            #debug_fmt_fn
+            #next_fn
+        }
+
+        #debug_impl
+    }
+}
+
+/// Generate `debug_fmt` fn.
+///
+/// The generated fn will be like:
+/// ```rust, ignore
+/// fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>);
+/// ```
+fn build_debug_fmt_impl(enum_name: Ident, variants: Vec<ErrorVariant>) -> TokenStream2 {
+    let match_arms = variants
+        .iter()
+        .map(|v| v.to_debug_match_arm())
+        .collect::<Vec<_>>();
+
+    quote! {
+        fn debug_fmt(&self, layer: usize, buf: &mut Vec<String>) {
+            use #enum_name::*;
+            match self {
+                #(#match_arms)*
+            }
+        }
+    }
+}
+
+/// Generate `next` fn.
+///
+/// The generated fn will be like:
+/// ```rust, ignore
+/// fn next(&self) -> Option<&dyn ::common_error::ext::StackError>;
+/// ```
+fn build_next_impl(enum_name: Ident, variants: Vec<ErrorVariant>) -> TokenStream2 {
+    let match_arms = variants
+        .iter()
+        .map(|v| v.to_next_match_arm())
+        .collect::<Vec<_>>();
+
+    quote! {
+        fn next(&self) -> Option<&dyn ::common_error::ext::StackError> {
+            use #enum_name::*;
+            match self {
+                #(#match_arms)*
+            }
+        }
+    }
+}
+
+/// Implement [std::fmt::Debug] via `debug_fmt`
+fn build_debug_impl(enum_name: Ident) -> TokenStream2 {
+    quote! {
+        impl std::fmt::Debug for #enum_name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                use ::common_error::ext::StackError;
+                let mut buf = vec![];
+                self.debug_fmt(0, &mut buf);
+                write!(f, "{}", buf.join("\n"))
+            }
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct ErrorVariant {
+    name: Ident,
+    fields: Vec<Ident>,
+    has_location: bool,
+    has_source: bool,
+    has_external_cause: bool,
+    display: TokenStream2,
+    span: Span,
+    cfg_attr: Option<Attribute>,
+}
+
+impl ErrorVariant {
+    /// Construct self from [Variant]
+    fn from_enum_variant(variant: Variant) -> Self {
+        let span = variant.span();
+        let mut has_location = false;
+        let mut has_source = false;
+        let mut has_external_cause = false;
+
+        for field in &variant.fields {
+            if let Some(ident) = &field.ident {
+                if ident == "location" {
+                    has_location = true;
+                } else if ident == "source" {
+                    has_source = true;
+                } else if ident == "error" {
+                    has_external_cause = true;
+                }
+            }
+        }
+
+        let mut display = None;
+        let mut cfg_attr = None;
+        for attr in variant.attrs {
+            if attr.path().is_ident("snafu") {
+                attr.parse_nested_meta(|meta| {
+                    if meta.path.is_ident("display") {
+                        let content;
+                        parenthesized!(content in meta.input);
+                        let display_ts: TokenStream2 = content.parse()?;
+                        display = Some(display_ts);
+                        Ok(())
+                    } else {
+                        Err(meta.error("unrecognized repr"))
+                    }
+                })
+                .expect("Each error should contains a display attribute");
+            }
+
+            if attr.path().is_ident("cfg") {
+                cfg_attr = Some(attr);
+            }
+        }
+
+        let field_ident = variant
+            .fields
+            .iter()
+            .map(|f| f.ident.clone().unwrap_or_else(|| Ident::new("_", f.span())))
+            .collect();
+
+        Self {
+            name: variant.ident,
+            fields: field_ident,
+            has_location,
+            has_source,
+            has_external_cause,
+            display: display.unwrap(),
+            span,
+            cfg_attr,
+        }
+    }
+
+    /// Convert self into an match arm that will be used in [build_debug_impl].
+    ///
+    /// The generated match arm will be like:
+    /// ```rust, ignore
+    ///     ErrorKindWithSource { source, .. } => {
+    ///         debug_fmt(source, layer + 1, buf);
+    ///     },
+    ///     ErrorKindWithoutSource { .. } => {
+    ///        buf.push(format!("{layer}: {}, at {}", format!(#display), location)));
+    ///     }
+    /// ```
+    ///
+    /// The generated code assumes fn `debug_fmt`, var `layer`, var `buf` are in scope.
+    fn to_debug_match_arm(&self) -> TokenStream2 {
+        let name = &self.name;
+        let fields = &self.fields;
+        let display = &self.display;
+        let cfg = if let Some(cfg) = &self.cfg_attr {
+            quote_spanned!(cfg.span() => #cfg)
+        } else {
+            quote! {}
+        };
+
+        match (self.has_location, self.has_source, self.has_external_cause) {
+            (true, true, _) => quote_spanned! {
+               self.span => #cfg #[allow(unused_variables)] #name { #(#fields),*, } => {
+                    buf.push(format!("{layer}: {}, at {}", format!(#display), location));
+                    source.debug_fmt(layer + 1, buf);
+                },
+            },
+            (true, false, true) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}, at {}", format!(#display), location));
+                    buf.push(format!("{}: {:?}", layer + 1, error));
+                },
+            },
+            (true, false, false) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}, at {}", format!(#display), location));
+                },
+            },
+            (false, true, _) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}", format!(#display)));
+                    source.debug_fmt(layer + 1, buf);
+                },
+            },
+            (false, false, true) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}", format!(#display)));
+                    buf.push(format!("{}: {:?}", layer + 1, error));
+                },
+            },
+            (false, false, false) => quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    buf.push(format!("{layer}: {}", format!(#display)));
+                },
+            },
+        }
+    }
+
+    /// Convert self into an match arm that will be used in [build_next_impl].
+    ///
+    /// The generated match arm will be like:
+    /// ```rust, ignore
+    ///     ErrorKindWithSource { source, .. } => {
+    ///         Some(source)
+    ///     },
+    ///     ErrorKindWithoutSource { .. } => {
+    ///        None
+    ///     }
+    /// ```
+    fn to_next_match_arm(&self) -> TokenStream2 {
+        let name = &self.name;
+        let fields = &self.fields;
+        let cfg = if let Some(cfg) = &self.cfg_attr {
+            quote_spanned!(cfg.span() => #cfg)
+        } else {
+            quote! {}
+        };
+
+        if self.has_source {
+            quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => {
+                    Some(source)
+                },
+            }
+        } else {
+            quote_spanned! {
+                self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } =>{
+                    None
+                }
+            }
+        }
+    }
+}

src/common/mem-prof/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 common-error = { workspace = true }
+common-macro = { workspace = true }
 snafu.workspace = true
 tempfile = "3.4"
 tokio.workspace = true

src/common/mem-prof/src/error.rs

@@ -16,14 +16,16 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::Snafu;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("{source}"))]
+    #[snafu(display("Internal error"))]
     Internal { source: BoxedError },
 
     #[snafu(display("Memory profiling is not supported"))]

src/common/mem-prof/src/jemalloc/error.rs

@@ -17,13 +17,18 @@ use std::path::PathBuf;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to read OPT_PROF, source: {}", source))]
-    ReadOptProf { source: tikv_jemalloc_ctl::Error },
+    #[snafu(display("Failed to read OPT_PROF"))]
+    ReadOptProf {
+        #[snafu(source)]
+        error: tikv_jemalloc_ctl::Error,
+    },
 
     #[snafu(display("Memory profiling is not enabled"))]
     ProfilingNotEnabled,
@@ -31,20 +36,18 @@ pub enum Error {
     #[snafu(display("Failed to build temp file from given path: {:?}", path))]
     BuildTempPath { path: PathBuf, location: Location },
 
-    #[snafu(display("Failed to open temp file: {}, source: {}", path, source))]
+    #[snafu(display("Failed to open temp file: {}", path))]
     OpenTempFile {
         path: String,
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
     },
 
-    #[snafu(display(
-        "Failed to dump profiling data to temp file: {:?}, source: {}",
-        path,
-        source
-    ))]
+    #[snafu(display("Failed to dump profiling data to temp file: {:?}", path))]
     DumpProfileData {
         path: PathBuf,
-        source: tikv_jemalloc_ctl::Error,
+        #[snafu(source)]
+        error: tikv_jemalloc_ctl::Error,
     },
 }
 

src/common/meta/Cargo.toml

@@ -15,6 +15,7 @@ async-trait.workspace = true
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc-expr.workspace = true
+common-macro = { workspace = true }
 common-procedure = { workspace = true }
 common-recordbatch = { workspace = true }
 common-runtime = { workspace = true }

src/common/meta/src/cache_invalidator.rs

@@ -17,8 +17,28 @@ use std::sync::Arc;
 use table::metadata::TableId;
 
 use crate::error::Result;
+use crate::key::schema_name::SchemaNameKey;
+use crate::key::table_info::TableInfoKey;
+use crate::key::table_name::TableNameKey;
+use crate::key::table_route::TableRouteKey;
+use crate::key::TableMetaKey;
 use crate::table_name::TableName;
 
+/// KvBackend cache invalidator
+#[async_trait::async_trait]
+pub trait KvCacheInvalidator: Send + Sync {
+    async fn invalidate_key(&self, key: &[u8]);
+}
+
+pub type KvCacheInvalidatorRef = Arc<dyn KvCacheInvalidator>;
+
+pub struct DummyKvCacheInvalidator;
+
+#[async_trait::async_trait]
+impl KvCacheInvalidator for DummyKvCacheInvalidator {
+    async fn invalidate_key(&self, _key: &[u8]) {}
+}
+
 /// Places context of invalidating cache. e.g., span id, trace id etc.
 #[derive(Default)]
 pub struct Context {
@@ -47,3 +67,38 @@ impl CacheInvalidator for DummyCacheInvalidator {
         Ok(())
     }
 }
+
+#[derive(Clone)]
+pub struct TableMetadataCacheInvalidator(KvCacheInvalidatorRef);
+
+impl TableMetadataCacheInvalidator {
+    pub fn new(kv_cache_invalidator: KvCacheInvalidatorRef) -> Self {
+        Self(kv_cache_invalidator)
+    }
+
+    pub async fn invalidate_schema(&self, catalog: &str, schema: &str) {
+        let key = SchemaNameKey::new(catalog, schema).as_raw_key();
+        self.0.invalidate_key(&key).await;
+    }
+}
+
+#[async_trait::async_trait]
+impl CacheInvalidator for TableMetadataCacheInvalidator {
+    async fn invalidate_table_name(&self, _ctx: &Context, table_name: TableName) -> Result<()> {
+        let key: TableNameKey = (&table_name).into();
+
+        self.0.invalidate_key(&key.as_raw_key()).await;
+
+        Ok(())
+    }
+
+    async fn invalidate_table_id(&self, _ctx: &Context, table_id: TableId) -> Result<()> {
+        let key = TableInfoKey::new(table_id);
+        self.0.invalidate_key(&key.as_raw_key()).await;
+
+        let key = &TableRouteKey { table_id };
+        self.0.invalidate_key(&key.as_raw_key()).await;
+
+        Ok(())
+    }
+}

src/common/meta/src/ddl.rs

@@ -28,6 +28,7 @@ use crate::rpc::router::RegionRoute;
 pub mod alter_table;
 pub mod create_table;
 pub mod drop_table;
+pub mod truncate_table;
 pub mod utils;
 
 #[derive(Debug, Default)]

src/common/meta/src/ddl/create_table.rs

@@ -161,7 +161,6 @@ impl CreateTableProcedure {
             engine: create_table_expr.engine.to_string(),
             column_defs,
             primary_key,
-            create_if_not_exists: true,
             path: String::new(),
             options: create_table_expr.table_options.clone(),
         })
@@ -200,8 +199,8 @@ impl CreateTableProcedure {
             for request in requests {
                 let request = RegionRequest {
                     header: Some(RegionRequestHeader {
-                        trace_id: 0,
-                        span_id: 0,
+                        trace_id: common_telemetry::trace_id().unwrap_or_default(),
+                        ..Default::default()
                     }),
                     body: Some(request),
                 };

src/common/meta/src/ddl/drop_table.rs

@@ -156,8 +156,8 @@ impl DropTableProcedure {
 
                 let request = RegionRequest {
                     header: Some(RegionRequestHeader {
-                        trace_id: 0,
-                        span_id: 0,
+                        trace_id: common_telemetry::trace_id().unwrap_or_default(),
+                        ..Default::default()
                     }),
                     body: Some(region_request::Body::Drop(PbDropRegionRequest {
                         region_id: region_id.as_u64(),

src/common/meta/src/ddl/truncate_table.rs

@@ -0,0 +1,234 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::region::{
+    region_request, RegionRequest, RegionRequestHeader, TruncateRequest as PbTruncateRegionRequest,
+};
+use async_trait::async_trait;
+use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
+use common_procedure::{
+    Context as ProcedureContext, LockKey, Procedure, Result as ProcedureResult, Status,
+};
+use common_telemetry::debug;
+use futures::future::join_all;
+use serde::{Deserialize, Serialize};
+use snafu::{ensure, ResultExt};
+use store_api::storage::RegionId;
+use strum::AsRefStr;
+use table::engine::TableReference;
+use table::metadata::{RawTableInfo, TableId};
+
+use super::utils::handle_retry_error;
+use crate::ddl::utils::handle_operate_region_error;
+use crate::ddl::DdlContext;
+use crate::error::{Result, TableNotFoundSnafu};
+use crate::key::table_info::TableInfoValue;
+use crate::key::table_name::TableNameKey;
+use crate::metrics;
+use crate::rpc::ddl::TruncateTableTask;
+use crate::rpc::router::{find_leader_regions, find_leaders, RegionRoute};
+use crate::table_name::TableName;
+
+pub struct TruncateTableProcedure {
+    context: DdlContext,
+    data: TruncateTableData,
+}
+
+#[async_trait]
+impl Procedure for TruncateTableProcedure {
+    fn type_name(&self) -> &str {
+        Self::TYPE_NAME
+    }
+
+    async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
+        let state = &self.data.state;
+
+        let _timer = common_telemetry::timer!(
+            metrics::METRIC_META_PROCEDURE_TRUNCATE_TABLE,
+            &[("step", state.as_ref().to_string())]
+        );
+
+        match self.data.state {
+            TruncateTableState::Prepare => self.on_prepare().await,
+            TruncateTableState::DatanodeTruncateRegions => {
+                self.on_datanode_truncate_regions().await
+            }
+        }
+        .map_err(handle_retry_error)
+    }
+
+    fn dump(&self) -> ProcedureResult<String> {
+        serde_json::to_string(&self.data).context(ToJsonSnafu)
+    }
+
+    fn lock_key(&self) -> LockKey {
+        let table_ref = &self.data.table_ref();
+        let key = common_catalog::format_full_table_name(
+            table_ref.catalog,
+            table_ref.schema,
+            table_ref.table,
+        );
+
+        LockKey::single(key)
+    }
+}
+
+impl TruncateTableProcedure {
+    pub(crate) const TYPE_NAME: &'static str = "metasrv-procedure::TruncateTable";
+
+    pub(crate) fn new(
+        cluster_id: u64,
+        task: TruncateTableTask,
+        table_info_value: TableInfoValue,
+        region_routes: Vec<RegionRoute>,
+        context: DdlContext,
+    ) -> Self {
+        Self {
+            context,
+            data: TruncateTableData::new(cluster_id, task, table_info_value, region_routes),
+        }
+    }
+
+    pub(crate) fn from_json(json: &str, context: DdlContext) -> ProcedureResult<Self> {
+        let data = serde_json::from_str(json).context(FromJsonSnafu)?;
+        Ok(Self { context, data })
+    }
+
+    // Checks whether the table exists.
+    async fn on_prepare(&mut self) -> Result<Status> {
+        let table_ref = &self.data.table_ref();
+
+        let manager = &self.context.table_metadata_manager;
+
+        let exist = manager
+            .table_name_manager()
+            .exists(TableNameKey::new(
+                table_ref.catalog,
+                table_ref.schema,
+                table_ref.table,
+            ))
+            .await?;
+
+        ensure!(
+            exist,
+            TableNotFoundSnafu {
+                table_name: table_ref.to_string()
+            }
+        );
+
+        self.data.state = TruncateTableState::DatanodeTruncateRegions;
+
+        Ok(Status::executing(true))
+    }
+
+    async fn on_datanode_truncate_regions(&mut self) -> Result<Status> {
+        let table_id = self.data.table_id();
+
+        let region_routes = &self.data.region_routes;
+        let leaders = find_leaders(region_routes);
+        let mut truncate_region_tasks = Vec::with_capacity(leaders.len());
+
+        for datanode in leaders {
+            let requester = self.context.datanode_manager.datanode(&datanode).await;
+            let regions = find_leader_regions(region_routes, &datanode);
+
+            for region in regions {
+                let region_id = RegionId::new(table_id, region);
+                debug!(
+                    "Truncating table {} region {} on Datanode {:?}",
+                    self.data.table_ref(),
+                    region_id,
+                    datanode
+                );
+
+                let request = RegionRequest {
+                    header: Some(RegionRequestHeader {
+                        trace_id: common_telemetry::trace_id().unwrap_or_default(),
+                        ..Default::default()
+                    }),
+                    body: Some(region_request::Body::Truncate(PbTruncateRegionRequest {
+                        region_id: region_id.as_u64(),
+                    })),
+                };
+
+                let datanode = datanode.clone();
+                let requester = requester.clone();
+
+                truncate_region_tasks.push(async move {
+                    if let Err(err) = requester.handle(request).await {
+                        return Err(handle_operate_region_error(datanode)(err));
+                    }
+                    Ok(())
+                });
+            }
+        }
+
+        join_all(truncate_region_tasks)
+            .await
+            .into_iter()
+            .collect::<Result<Vec<_>>>()?;
+
+        Ok(Status::Done)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TruncateTableData {
+    state: TruncateTableState,
+    cluster_id: u64,
+    task: TruncateTableTask,
+    table_info_value: TableInfoValue,
+    region_routes: Vec<RegionRoute>,
+}
+
+impl TruncateTableData {
+    pub fn new(
+        cluster_id: u64,
+        task: TruncateTableTask,
+        table_info_value: TableInfoValue,
+        region_routes: Vec<RegionRoute>,
+    ) -> Self {
+        Self {
+            state: TruncateTableState::Prepare,
+            cluster_id,
+            task,
+            table_info_value,
+            region_routes,
+        }
+    }
+
+    pub fn table_ref(&self) -> TableReference {
+        self.task.table_ref()
+    }
+
+    pub fn table_name(&self) -> TableName {
+        self.task.table_name()
+    }
+
+    fn table_info(&self) -> &RawTableInfo {
+        &self.table_info_value.table_info
+    }
+
+    fn table_id(&self) -> TableId {
+        self.table_info().ident.table_id
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, AsRefStr)]
+enum TruncateTableState {
+    /// Prepares to truncate the table
+    Prepare,
+    /// Truncates regions on Datanode
+    DatanodeTruncateRegions,
+}

src/common/meta/src/ddl_manager.rs

@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use common_procedure::{watcher, ProcedureId, ProcedureManagerRef, ProcedureWithId};
-use common_telemetry::{error, info};
+use common_telemetry::info;
 use snafu::{OptionExt, ResultExt};
 
 use crate::cache_invalidator::CacheInvalidatorRef;
@@ -23,13 +23,14 @@ use crate::datanode_manager::DatanodeManagerRef;
 use crate::ddl::alter_table::AlterTableProcedure;
 use crate::ddl::create_table::CreateTableProcedure;
 use crate::ddl::drop_table::DropTableProcedure;
+use crate::ddl::truncate_table::TruncateTableProcedure;
 use crate::ddl::{
     DdlContext, DdlTaskExecutor, ExecutorContext, TableMetadataAllocatorContext,
     TableMetadataAllocatorRef,
 };
 use crate::error::{
     self, RegisterProcedureLoaderSnafu, Result, SubmitProcedureSnafu, TableNotFoundSnafu,
-    UnsupportedSnafu, WaitProcedureSnafu,
+    WaitProcedureSnafu,
 };
 use crate::key::table_info::TableInfoValue;
 use crate::key::table_name::TableNameKey;
@@ -122,6 +123,20 @@ impl DdlManager {
             )
             .context(RegisterProcedureLoaderSnafu {
                 type_name: AlterTableProcedure::TYPE_NAME,
+            })?;
+
+        let context = self.create_context();
+
+        self.procedure_manager
+            .register_loader(
+                TruncateTableProcedure::TYPE_NAME,
+                Box::new(move |json| {
+                    let context = context.clone();
+                    TruncateTableProcedure::from_json(json, context).map(|p| Box::new(p) as _)
+                }),
+            )
+            .context(RegisterProcedureLoaderSnafu {
+                type_name: TruncateTableProcedure::TYPE_NAME,
             })
     }
 
@@ -183,15 +198,21 @@ impl DdlManager {
         &self,
         cluster_id: u64,
         truncate_table_task: TruncateTableTask,
+        table_info_value: TableInfoValue,
         region_routes: Vec<RegionRoute>,
     ) -> Result<ProcedureId> {
-        error!("Truncate table procedure is not supported, cluster_id = {}, truncate_table_task = {:?}, region_routes = {:?}",
-            cluster_id, truncate_table_task, region_routes);
+        let context = self.create_context();
+        let procedure = TruncateTableProcedure::new(
+            cluster_id,
+            truncate_table_task,
+            table_info_value,
+            region_routes,
+            context,
+        );
 
-        UnsupportedSnafu {
-            operation: "TRUNCATE TABLE",
-        }
-        .fail()
+        let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
+
+        self.submit_procedure(procedure_with_id).await
     }
 
     async fn submit_procedure(&self, procedure_with_id: ProcedureWithId) -> Result<ProcedureId> {
@@ -216,32 +237,34 @@ async fn handle_truncate_table_task(
     cluster_id: u64,
     truncate_table_task: TruncateTableTask,
 ) -> Result<SubmitDdlTaskResponse> {
-    let truncate_table = &truncate_table_task.truncate_table;
-    let table_id = truncate_table
-        .table_id
-        .as_ref()
-        .context(error::UnexpectedSnafu {
-            err_msg: "expected table id ",
-        })?
-        .id;
-
+    let table_id = truncate_table_task.table_id;
+    let table_metadata_manager = &ddl_manager.table_metadata_manager();
     let table_ref = truncate_table_task.table_ref();
 
-    let table_route_value = ddl_manager
-        .table_metadata_manager()
-        .table_route_manager()
-        .get(table_id)
-        .await?
-        .with_context(|| error::TableRouteNotFoundSnafu {
-            table_name: table_ref.to_string(),
-        })?;
+    let (table_info_value, table_route_value) =
+        table_metadata_manager.get_full_table_info(table_id).await?;
+
+    let table_info_value = table_info_value.with_context(|| error::TableInfoNotFoundSnafu {
+        table_name: table_ref.to_string(),
+    })?;
+
+    let table_route_value = table_route_value.with_context(|| error::TableRouteNotFoundSnafu {
+        table_name: table_ref.to_string(),
+    })?;
 
     let table_route = table_route_value.region_routes;
 
     let id = ddl_manager
-        .submit_truncate_table_task(cluster_id, truncate_table_task, table_route)
+        .submit_truncate_table_task(
+            cluster_id,
+            truncate_table_task,
+            table_info_value,
+            table_route,
+        )
         .await?;
 
+    info!("Table: {table_id} is truncated via procedure_id {id:?}");
+
     Ok(SubmitDdlTaskResponse {
         key: id.to_string().into(),
         ..Default::default()

src/common/meta/src/distributed_time_constants.rs

@@ -29,3 +29,9 @@ pub const REGION_LEASE_SECS: u64 =
 /// When creating table or region failover, a target node needs to be selected.
 /// If the node's lease has expired, the `Selector` will not select it.
 pub const DATANODE_LEASE_SECS: u64 = REGION_LEASE_SECS;
+
+/// The lease seconds of metasrv leader.
+pub const META_LEASE_SECS: u64 = 3;
+
+// In a lease, there are two opportunities for renewal.
+pub const META_KEEP_ALIVE_INTERVAL_SECS: u64 = META_LEASE_SECS / 2;

src/common/meta/src/error.rs

@@ -16,6 +16,7 @@ use std::str::Utf8Error;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use serde_json::error::Error as JsonError;
 use snafu::{Location, Snafu};
 use store_api::storage::RegionNumber;
@@ -23,8 +24,9 @@ use table::metadata::TableId;
 
 use crate::peer::Peer;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Failed to get sequence: {}", err_msg))]
     NextSequence { err_msg: String, location: Location },
@@ -46,56 +48,49 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display(
-        "Failed to register procedure loader, type name: {}, source: {}",
-        type_name,
-        source
-    ))]
+    #[snafu(display("Failed to register procedure loader, type name: {}", type_name))]
     RegisterProcedureLoader {
         type_name: String,
         location: Location,
         source: common_procedure::error::Error,
     },
 
-    #[snafu(display("Failed to submit procedure, source: {source}"))]
+    #[snafu(display("Failed to submit procedure"))]
     SubmitProcedure {
         location: Location,
         source: common_procedure::Error,
     },
 
-    #[snafu(display("Unsupported operation {}, location: {}", operation, location))]
+    #[snafu(display("Unsupported operation {}", operation))]
     Unsupported {
         operation: String,
         location: Location,
     },
 
-    #[snafu(display("Failed to wait procedure done, source: {source}"))]
+    #[snafu(display("Failed to wait procedure done"))]
     WaitProcedure {
         location: Location,
         source: common_procedure::Error,
     },
 
-    #[snafu(display("Failed to convert RawTableInfo into TableInfo: {}", source))]
+    #[snafu(display("Failed to convert RawTableInfo into TableInfo"))]
     ConvertRawTableInfo {
         location: Location,
         source: datatypes::Error,
     },
 
-    #[snafu(display("Primary key '{key}' not found when creating region request, at {location}"))]
+    #[snafu(display("Primary key '{key}' not found when creating region request"))]
     PrimaryKeyNotFound { key: String, location: Location },
 
-    #[snafu(display(
-        "Failed to build table meta for table: {}, source: {}",
-        table_name,
-        source
-    ))]
+    #[snafu(display("Failed to build table meta for table: {}", table_name))]
     BuildTableMeta {
         table_name: String,
-        source: table::metadata::TableMetaBuilderError,
+        #[snafu(source)]
+        error: table::metadata::TableMetaBuilderError,
         location: Location,
     },
 
-    #[snafu(display("Table occurs error, source: {}", source))]
+    #[snafu(display("Table occurs error"))]
     Table {
         location: Location,
         source: table::error::Error,
@@ -107,22 +102,25 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to decode protobuf, source: {}", source))]
+    #[snafu(display("Failed to decode protobuf"))]
     DecodeProto {
         location: Location,
-        source: prost::DecodeError,
+        #[snafu(source)]
+        error: prost::DecodeError,
     },
 
-    #[snafu(display("Failed to encode object into json, source: {}", source))]
+    #[snafu(display("Failed to encode object into json"))]
     EncodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
-    #[snafu(display("Failed to decode object from json, source: {}", source))]
+    #[snafu(display("Failed to decode object from json"))]
     DecodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Payload not exist"))]
@@ -131,9 +129,10 @@ pub enum Error {
     #[snafu(display("Failed to send message: {err_msg}"))]
     SendMessage { err_msg: String, location: Location },
 
-    #[snafu(display("Failed to serde json, source: {}", source))]
+    #[snafu(display("Failed to serde json"))]
     SerdeJson {
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
@@ -154,13 +153,13 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to convert alter table request, source: {source}, at {location}"))]
+    #[snafu(display("Failed to convert alter table request"))]
     ConvertAlterTableRequest {
         source: common_grpc_expr::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Invalid protobuf message: {err_msg}, at {location}"))]
+    #[snafu(display("Invalid protobuf message: {err_msg}"))]
     InvalidProtoMsg { err_msg: String, location: Location },
 
     #[snafu(display("Unexpected: {err_msg}"))]
@@ -182,10 +181,11 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to convert raw key to str, source: {}", source))]
+    #[snafu(display("Failed to convert raw key to str"))]
     ConvertRawKey {
         location: Location,
-        source: Utf8Error,
+        #[snafu(source)]
+        error: Utf8Error,
     },
 
     #[snafu(display("Table nod found, table: {}", table_name))]
@@ -222,29 +222,29 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Invalid catalog value, source: {}", source))]
+    #[snafu(display("Invalid catalog value"))]
     InvalidCatalogValue {
         source: common_catalog::error::Error,
         location: Location,
     },
 
-    #[snafu(display("{}", source))]
+    #[snafu(display("External error"))]
     External {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Invalid heartbeat response, location: {}", location))]
+    #[snafu(display("Invalid heartbeat response"))]
     InvalidHeartbeatResponse { location: Location },
 
-    #[snafu(display("Failed to operate on datanode: {}, source: {}", peer, source))]
+    #[snafu(display("Failed to operate on datanode: {}", peer))]
     OperateDatanode {
         location: Location,
         peer: Peer,
         source: BoxedError,
     },
 
-    #[snafu(display("Retry later, source: {}", source))]
+    #[snafu(display("Retry later"))]
     RetryLater { source: BoxedError },
 }
 

src/common/meta/src/key.rs

@@ -80,7 +80,7 @@ use crate::DatanodeId;
 
 pub const REMOVED_PREFIX: &str = "__removed";
 
-const NAME_PATTERN: &str = "[a-zA-Z_:-][a-zA-Z0-9_:-]*";
+const NAME_PATTERN: &str = r"[a-zA-Z_:-][a-zA-Z0-9_:\-\.]*";
 
 const DATANODE_TABLE_KEY_PREFIX: &str = "__dn_table";
 const TABLE_INFO_KEY_PREFIX: &str = "__table_info";

src/common/meta/src/key/table_name.rs

@@ -268,6 +268,8 @@ mod tests {
         test_ok("my_table");
         test_ok("cpu:metrics");
         test_ok(":cpu:metrics");
+        test_ok("sys.cpu.system");
+        test_ok("foo-bar");
     }
 
     #[test]

src/common/meta/src/metrics.rs

@@ -19,3 +19,4 @@ pub(crate) const METRIC_META_CREATE_SCHEMA: &str = "meta.create_schema";
 pub(crate) const METRIC_META_PROCEDURE_CREATE_TABLE: &str = "meta.procedure.create_table";
 pub(crate) const METRIC_META_PROCEDURE_DROP_TABLE: &str = "meta.procedure.drop_table";
 pub(crate) const METRIC_META_PROCEDURE_ALTER_TABLE: &str = "meta.procedure.alter_table";
+pub(crate) const METRIC_META_PROCEDURE_TRUNCATE_TABLE: &str = "meta.procedure.truncate_table";

src/common/meta/src/rpc/ddl.rs

@@ -65,8 +65,18 @@ impl DdlTask {
         DdlTask::AlterTable(AlterTableTask { alter_table })
     }
 
-    pub fn new_truncate_table(truncate_table: TruncateTableExpr) -> Self {
-        DdlTask::TruncateTable(TruncateTableTask { truncate_table })
+    pub fn new_truncate_table(
+        catalog: String,
+        schema: String,
+        table: String,
+        table_id: TableId,
+    ) -> Self {
+        DdlTask::TruncateTable(TruncateTableTask {
+            catalog,
+            schema,
+            table,
+            table_id,
+        })
     }
 }
 
@@ -112,7 +122,12 @@ impl TryFrom<SubmitDdlTaskRequest> for PbSubmitDdlTaskRequest {
                 alter_table: Some(task.alter_table),
             }),
             DdlTask::TruncateTable(task) => Task::TruncateTableTask(PbTruncateTableTask {
-                truncate_table: Some(task.truncate_table),
+                truncate_table: Some(TruncateTableExpr {
+                    catalog_name: task.catalog,
+                    schema_name: task.schema,
+                    table_name: task.table,
+                    table_id: Some(api::v1::TableId { id: task.table_id }),
+                }),
             }),
         };
 
@@ -358,27 +373,28 @@ impl<'de> Deserialize<'de> for AlterTableTask {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct TruncateTableTask {
-    pub truncate_table: TruncateTableExpr,
+    pub catalog: String,
+    pub schema: String,
+    pub table: String,
+    pub table_id: TableId,
 }
 
 impl TruncateTableTask {
     pub fn table_ref(&self) -> TableReference {
         TableReference {
-            catalog: &self.truncate_table.catalog_name,
-            schema: &self.truncate_table.schema_name,
-            table: &self.truncate_table.table_name,
+            catalog: &self.catalog,
+            schema: &self.schema,
+            table: &self.table,
         }
     }
 
     pub fn table_name(&self) -> TableName {
-        let table = &self.truncate_table;
-
         TableName {
-            catalog_name: table.catalog_name.to_string(),
-            schema_name: table.schema_name.to_string(),
-            table_name: table.table_name.to_string(),
+            catalog_name: self.catalog.to_string(),
+            schema_name: self.schema.to_string(),
+            table_name: self.table.to_string(),
         }
     }
 }
@@ -388,39 +404,20 @@ impl TryFrom<PbTruncateTableTask> for TruncateTableTask {
 
     fn try_from(pb: PbTruncateTableTask) -> Result<Self> {
         let truncate_table = pb.truncate_table.context(error::InvalidProtoMsgSnafu {
-            err_msg: "expected truncate_table",
+            err_msg: "expected drop table",
         })?;
 
-        Ok(TruncateTableTask { truncate_table })
-    }
-}
-
-impl Serialize for TruncateTableTask {
-    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        let pb = PbTruncateTableTask {
-            truncate_table: Some(self.truncate_table.clone()),
-        };
-        let buf = pb.encode_to_vec();
-        serializer.serialize_bytes(&buf)
-    }
-}
-
-impl<'de> Deserialize<'de> for TruncateTableTask {
-    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        let buf = Vec::<u8>::deserialize(deserializer)?;
-        let task: PbTruncateTableTask = PbTruncateTableTask::decode(&*buf)
-            .map_err(|err| serde::de::Error::custom(err.to_string()))?;
-
-        let task = TruncateTableTask::try_from(task)
-            .map_err(|err| serde::de::Error::custom(err.to_string()))?;
-
-        Ok(task)
+        Ok(Self {
+            catalog: truncate_table.catalog_name,
+            schema: truncate_table.schema_name,
+            table: truncate_table.table_name,
+            table_id: truncate_table
+                .table_id
+                .context(error::InvalidProtoMsgSnafu {
+                    err_msg: "expected table_id",
+                })?
+                .id,
+        })
     }
 }
 

src/common/procedure/Cargo.toml

@@ -9,6 +9,7 @@ async-stream.workspace = true
 async-trait.workspace = true
 backon = "0.4"
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-runtime = { workspace = true }
 common-telemetry = { workspace = true }
 futures.workspace = true

src/common/procedure/src/error.rs

@@ -18,26 +18,26 @@ use std::sync::Arc;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
 use crate::procedure::ProcedureId;
 
 /// Procedure error.
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display(
-        "Failed to execute procedure due to external error, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to execute procedure due to external error"))]
     External { source: BoxedError },
 
     #[snafu(display("Loader {} is already registered", name))]
     LoaderConflict { name: String, location: Location },
 
-    #[snafu(display("Failed to serialize to json, source: {}", source))]
+    #[snafu(display("Failed to serialize to json"))]
     ToJson {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
@@ -47,83 +47,85 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to put state, key: '{key}', source: {source}"))]
+    #[snafu(display("Failed to put state, key: '{key}'"))]
     PutState {
         key: String,
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to delete {}, source: {}", key, source))]
+    #[snafu(display("Failed to delete {}", key))]
     DeleteState {
         key: String,
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
     },
 
-    #[snafu(display("Failed to delete keys: '{keys}', source: {source}"))]
+    #[snafu(display("Failed to delete keys: '{keys}'"))]
     DeleteStates {
         keys: String,
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to list state, path: '{path}', source: {source}"))]
+    #[snafu(display("Failed to list state, path: '{path}'"))]
     ListState {
         path: String,
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to deserialize from json, source: {}", source))]
+    #[snafu(display("Failed to deserialize from json"))]
     FromJson {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
-    #[snafu(display("Procedure exec failed, source: {}", source))]
+    #[snafu(display("Procedure exec failed"))]
     RetryLater { source: BoxedError },
 
     #[snafu(display("Procedure panics, procedure_id: {}", procedure_id))]
     ProcedurePanic { procedure_id: ProcedureId },
 
-    #[snafu(display("Failed to wait watcher, source: {}", source))]
+    #[snafu(display("Failed to wait watcher"))]
     WaitWatcher {
-        source: tokio::sync::watch::error::RecvError,
+        #[snafu(source)]
+        error: tokio::sync::watch::error::RecvError,
         location: Location,
     },
 
-    #[snafu(display("Failed to execute procedure, source: {}", source))]
+    #[snafu(display("Failed to execute procedure"))]
     ProcedureExec {
         source: Arc<Error>,
         location: Location,
     },
 
-    #[snafu(display(
-        "Procedure retry exceeded max times, procedure_id: {}, source:{}",
-        procedure_id,
-        source
-    ))]
+    #[snafu(display("Procedure retry exceeded max times, procedure_id: {}", procedure_id))]
     RetryTimesExceeded {
         source: Arc<Error>,
         procedure_id: ProcedureId,
     },
 
-    #[snafu(display("Corrupted data, error: {source}"))]
-    CorruptedData { source: FromUtf8Error },
+    #[snafu(display("Corrupted data, error: "))]
+    CorruptedData {
+        #[snafu(source)]
+        error: FromUtf8Error,
+    },
 
-    #[snafu(display("Failed to start the remove_outdated_meta method, error: {}", source))]
+    #[snafu(display("Failed to start the remove_outdated_meta method, error"))]
     StartRemoveOutdatedMetaTask {
         source: common_runtime::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to stop the remove_outdated_meta method, error: {}", source))]
+    #[snafu(display("Failed to stop the remove_outdated_meta method, error"))]
     StopRemoveOutdatedMetaTask {
         source: common_runtime::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Subprocedure {} failed, source: {}", subprocedure_id, source))]
+    #[snafu(display("Subprocedure {} failed", subprocedure_id))]
     SubprocedureFailed {
         subprocedure_id: ProcedureId,
         source: Arc<Error>,

src/common/procedure/src/local.rs

@@ -229,7 +229,7 @@ impl ManagerContext {
         let procedure = loader(&message.data)
             .map_err(|e| {
                 logging::error!(
-                    "Failed to load procedure data, key: {}, source: {}",
+                    "Failed to load procedure data, key: {}, source: {:?}",
                     procedure_id,
                     e
                 );

src/common/procedure/src/local/runner.rs

@@ -453,7 +453,7 @@ mod tests {
     use std::sync::Arc;
 
     use async_trait::async_trait;
-    use common_error::ext::PlainError;
+    use common_error::ext::{ErrorExt, PlainError};
     use common_error::mock::MockError;
     use common_error::status_code::StatusCode;
     use common_test_util::temp_dir::create_temp_dir;
@@ -942,7 +942,7 @@ mod tests {
 
         // Run the runner and execute the procedure.
         runner.run().await;
-        let err = meta.state().error().unwrap().to_string();
+        let err = meta.state().error().unwrap().output_msg();
         assert!(err.contains("subprocedure failed"), "{err}");
     }
 }

src/common/procedure/src/store.rs

@@ -224,7 +224,7 @@ impl ProcedureStore {
         serde_json::from_slice(value)
             .map_err(|e| {
                 // `e` doesn't impl ErrorExt so we print it as normal error.
-                logging::error!("Failed to parse value, key: {:?}, source: {}", key, e);
+                logging::error!("Failed to parse value, key: {:?}, source: {:?}", key, e);
                 e
             })
             .ok()

src/common/query/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 api = { workspace = true }
 async-trait.workspace = true
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-recordbatch = { workspace = true }
 common-time = { workspace = true }
 datafusion-common.workspace = true

src/common/query/src/error.rs

@@ -17,6 +17,7 @@ use std::any::Any;
 use arrow::error::ArrowError;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_recordbatch::error::Error as RecordbatchError;
 use datafusion_common::DataFusionError;
 use datatypes::arrow;
@@ -26,28 +27,27 @@ use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
 use statrs::StatsError;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Fail to execute Python UDF, source: {}", msg))]
+    #[snafu(display("Failed to execute Python UDF: {}", msg))]
     PyUdf {
         // TODO(discord9): find a way that prevent circle depend(query<-script<-query) and can use script's error type
         msg: String,
         location: Location,
     },
 
-    #[snafu(display(
-        "Fail to create temporary recordbatch when eval Python UDF, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to create temporary recordbatch when eval Python UDF"))]
     UdfTempRecordBatch {
         location: Location,
         source: RecordbatchError,
     },
 
-    #[snafu(display("Fail to execute function, source: {}", source))]
+    #[snafu(display("Failed to execute function"))]
     ExecuteFunction {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
@@ -58,25 +58,26 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Fail to generate function, source: {}", source))]
+    #[snafu(display("Failed to generate function"))]
     GenerateFunction {
-        source: StatsError,
+        #[snafu(source)]
+        error: StatsError,
         location: Location,
     },
 
-    #[snafu(display("Fail to cast scalar value into vector: {}", source))]
+    #[snafu(display("Failed to cast scalar value into vector"))]
     FromScalarValue {
         location: Location,
         source: DataTypeError,
     },
 
-    #[snafu(display("Fail to cast arrow array into vector: {}", source))]
+    #[snafu(display("Failed to cast arrow array into vector"))]
     FromArrowArray {
         location: Location,
         source: DataTypeError,
     },
 
-    #[snafu(display("Fail to cast arrow array into vector: {:?}, {}", data_type, source))]
+    #[snafu(display("Failed to cast arrow array into vector: {:?}", data_type))]
     IntoVector {
         location: Location,
         source: DataTypeError,
@@ -110,56 +111,53 @@ pub enum Error {
     #[snafu(display("Not expected to run ExecutionPlan more than once"))]
     ExecuteRepeatedly { location: Location },
 
-    #[snafu(display("General DataFusion error, source: {}", source))]
+    #[snafu(display("General DataFusion error"))]
     GeneralDataFusion {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
-    #[snafu(display(
-        "Failed to convert DataFusion's recordbatch stream, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to convert DataFusion's recordbatch stream"))]
     ConvertDfRecordBatchStream {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to convert arrow schema, source: {}", source))]
+    #[snafu(display("Failed to convert arrow schema"))]
     ConvertArrowSchema {
         location: Location,
         source: DataTypeError,
     },
 
-    #[snafu(display("Failed to execute physical plan, source: {}", source))]
+    #[snafu(display("Failed to execute physical plan"))]
     ExecutePhysicalPlan {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to cast array to {:?}, source: {}", typ, source))]
+    #[snafu(display("Failed to cast array to {:?}", typ))]
     TypeCast {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         typ: arrow::datatypes::DataType,
         location: Location,
     },
 
-    #[snafu(display(
-        "Failed to perform compute operation on arrow arrays, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to perform compute operation on arrow arrays"))]
     ArrowCompute {
-        source: ArrowError,
+        #[snafu(source)]
+        error: ArrowError,
         location: Location,
     },
 
-    #[snafu(display("Query engine fail to cast value: {}", source))]
+    #[snafu(display("Query engine fail to cast value"))]
     ToScalarValue {
         location: Location,
         source: DataTypeError,
     },
 
-    #[snafu(display("Failed to get scalar vector, {}", source))]
+    #[snafu(display("Failed to get scalar vector"))]
     GetScalarVector {
         location: Location,
         source: DataTypeError,

src/common/recordbatch/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 common-error = { workspace = true }
+common-macro = { workspace = true }
 datafusion-common.workspace = true
 datafusion.workspace = true
 datatypes = { workspace = true }

src/common/recordbatch/src/error.rs

@@ -17,27 +17,30 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Fail to create datafusion record batch, source: {}", source))]
+    #[snafu(display("Fail to create datafusion record batch"))]
     NewDfRecordBatch {
-        source: datatypes::arrow::error::ArrowError,
+        #[snafu(source)]
+        error: datatypes::arrow::error::ArrowError,
         location: Location,
     },
 
-    #[snafu(display("Data types error, source: {}", source))]
+    #[snafu(display("Data types error"))]
     DataTypes {
         location: Location,
         source: datatypes::error::Error,
     },
 
-    #[snafu(display("External error, location: {}, source: {}", location, source))]
+    #[snafu(display("External error"))]
     External {
         location: Location,
         source: BoxedError,
@@ -46,38 +49,41 @@ pub enum Error {
     #[snafu(display("Failed to create RecordBatches, reason: {}", reason))]
     CreateRecordBatches { reason: String, location: Location },
 
-    #[snafu(display("Failed to convert Arrow schema, source: {}", source))]
+    #[snafu(display("Failed to convert Arrow schema"))]
     SchemaConversion {
         source: datatypes::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to poll stream, source: {}", source))]
+    #[snafu(display(""))]
     PollStream {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 
-    #[snafu(display("Fail to format record batch, source: {}", source))]
+    #[snafu(display("Fail to format record batch"))]
     Format {
-        source: datatypes::arrow::error::ArrowError,
+        #[snafu(source)]
+        error: datatypes::arrow::error::ArrowError,
         location: Location,
     },
 
-    #[snafu(display("Failed to init Recordbatch stream, source: {}", source))]
+    #[snafu(display("Failed to init Recordbatch stream"))]
     InitRecordbatchStream {
-        source: datafusion_common::DataFusionError,
+        #[snafu(source)]
+        error: datafusion_common::DataFusionError,
         location: Location,
     },
 
     #[snafu(display(
-        "Failed to project Arrow RecordBatch with schema {:?} and projection {:?}, source: {}",
+        "Failed to project Arrow RecordBatch with schema {:?} and projection {:?}",
         schema,
         projection,
-        source
     ))]
     ProjectArrowRecordBatch {
-        source: datatypes::arrow::error::ArrowError,
+        #[snafu(source)]
+        error: datatypes::arrow::error::ArrowError,
         location: Location,
         schema: datatypes::schema::SchemaRef,
         projection: Vec<usize>,
@@ -91,10 +97,9 @@ pub enum Error {
     },
 
     #[snafu(display(
-        "Failed to cast vector of type '{:?}' to type '{:?}', source: {}",
+        "Failed to cast vector of type '{:?}' to type '{:?}'",
         from_type,
         to_type,
-        source
     ))]
     CastVector {
         from_type: ConcreteDataType,

src/common/runtime/Cargo.toml

@@ -7,6 +7,7 @@ license.workspace = true
 [dependencies]
 async-trait.workspace = true
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-telemetry = { workspace = true }
 metrics.workspace = true
 once_cell.workspace = true

src/common/runtime/src/error.rs

@@ -15,31 +15,31 @@
 use std::any::Any;
 
 use common_error::ext::ErrorExt;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tokio::task::JoinError;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub(crate)))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to build runtime, source: {}", source))]
+    #[snafu(display("Failed to build runtime"))]
     BuildRuntime {
-        source: std::io::Error,
+        #[snafu(source)]
+        error: std::io::Error,
         location: Location,
     },
 
     #[snafu(display("Repeated task {} is already started", name))]
     IllegalState { name: String, location: Location },
 
-    #[snafu(display(
-        "Failed to wait for repeated task {} to stop, source: {}",
-        name,
-        source
-    ))]
+    #[snafu(display("Failed to wait for repeated task {} to stop", name))]
     WaitGcTaskStop {
         name: String,
-        source: JoinError,
+        #[snafu(source)]
+        error: JoinError,
         location: Location,
     },
 }

src/common/substrait/Cargo.toml

@@ -11,6 +11,7 @@ bytes = "1.1"
 catalog = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-telemetry = { workspace = true }
 datafusion-common.workspace = true
 datafusion-expr.workspace = true

src/common/substrait/src/error.rs

@@ -16,13 +16,15 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use prost::{DecodeError, EncodeError};
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unsupported physical plan: {}", name))]
     UnsupportedPlan { name: String, location: Location },
@@ -39,15 +41,17 @@ pub enum Error {
     #[snafu(display("Unsupported substrait type: {}", ty))]
     UnsupportedSubstraitType { ty: String, location: Location },
 
-    #[snafu(display("Failed to decode substrait relation, source: {}", source))]
+    #[snafu(display("Failed to decode substrait relation"))]
     DecodeRel {
-        source: DecodeError,
+        #[snafu(source)]
+        error: DecodeError,
         location: Location,
     },
 
-    #[snafu(display("Failed to encode substrait relation, source: {}", source))]
+    #[snafu(display("Failed to encode substrait relation"))]
     EncodeRel {
-        source: EncodeError,
+        #[snafu(source)]
+        error: EncodeError,
         location: Location,
     },
 
@@ -67,13 +71,14 @@ pub enum Error {
     #[snafu(display("Invalid parameters: {}", reason))]
     InvalidParameters { reason: String, location: Location },
 
-    #[snafu(display("Internal error from DataFusion: {}", source))]
+    #[snafu(display("Internal error from DataFusion"))]
     DFInternal {
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
-    #[snafu(display("Internal error: {}", source))]
+    #[snafu(display("Internal error"))]
     Internal {
         location: Location,
         source: BoxedError,
@@ -95,28 +100,30 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to convert DataFusion schema, source: {}", source))]
+    #[snafu(display("Failed to convert DataFusion schema"))]
     ConvertDfSchema {
         location: Location,
         source: datatypes::error::Error,
     },
 
-    #[snafu(display("Unable to resolve table: {table_name}, error: {source}"))]
+    #[snafu(display("Unable to resolve table: {table_name}, error: "))]
     ResolveTable {
         table_name: String,
         location: Location,
         source: catalog::error::Error,
     },
 
-    #[snafu(display("Failed to encode DataFusion plan, source: {}", source))]
+    #[snafu(display("Failed to encode DataFusion plan"))]
     EncodeDfPlan {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 
-    #[snafu(display("Failed to decode DataFusion plan, source: {}", source))]
+    #[snafu(display("Failed to decode DataFusion plan"))]
     DecodeDfPlan {
-        source: datafusion::error::DataFusionError,
+        #[snafu(source)]
+        error: datafusion::error::DataFusionError,
         location: Location,
     },
 }

src/common/telemetry/src/macros.rs

@@ -38,94 +38,40 @@ macro_rules! error {
 
     // error!(e; target: "my_target", "a {} event", "log")
     ($e:expr; target: $target:expr, $($arg:tt)+) => ({
-        use $crate::common_error::ext::ErrorExt;
-        use std::error::Error;
-        match ($e.source(), $e.location_opt()) {
-            (Some(source), Some(location)) => {
-                $crate::log!(
-                    target: $target,
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (Some(source), None) => {
-                $crate::log!(
-                    target: $target,
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    $($arg)+
-                )
-            },
-            (None, Some(location)) => {
-                $crate::log!(
-                    target: $target,
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (None, None) => {
-                $crate::log!(
-                    target: $target,
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    $($arg)+
-                )
-            }
-        }
+        $crate::log!(
+            target: $target,
+            $crate::logging::Level::ERROR,
+            err = ?$e,
+            $($arg)+
+        )
+    });
+
+    // error!(%e; target: "my_target", "a {} event", "log")
+    (%$e:expr; target: $target:expr, $($arg:tt)+) => ({
+        $crate::log!(
+            target: $target,
+            $crate::logging::Level::ERROR,
+            err = %$e,
+            $($arg)+
+        )
     });
 
     // error!(e; "a {} event", "log")
     ($e:expr; $($arg:tt)+) => ({
-        use std::error::Error;
-        use $crate::common_error::ext::ErrorExt;
-        match ($e.source(), $e.location_opt()) {
-            (Some(source), Some(location)) => {
-                $crate::log!(
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (Some(source), None) => {
-                $crate::log!(
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    $($arg)+
-                )
-            },
-            (None, Some(location)) => {
-                $crate::log!(
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (None, None) => {
-                $crate::log!(
-                    $crate::logging::Level::ERROR,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    $($arg)+
-                )
-            }
-        }
+        $crate::log!(
+            $crate::logging::Level::ERROR,
+            err = ?$e,
+            $($arg)+
+        )
+    });
+
+    // error!(%e; "a {} event", "log")
+    (%$e:expr; $($arg:tt)+) => ({
+        $crate::log!(
+            $crate::logging::Level::ERROR,
+            err = %$e,
+            $($arg)+
+        )
     });
 
     // error!("a {} event", "log")
@@ -144,46 +90,20 @@ macro_rules! warn {
 
     // warn!(e; "a {} event", "log")
     ($e:expr; $($arg:tt)+) => ({
-        use std::error::Error;
-        use $crate::common_error::ext::ErrorExt;
-        match ($e.source(), $e.location_opt()) {
-            (Some(source), Some(location)) => {
-                $crate::log!(
-                    $crate::logging::Level::WARN,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (Some(source), None) => {
-                $crate::log!(
-                    $crate::logging::Level::WARN,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.source = source,
-                    $($arg)+
-                )
-            },
-            (None, Some(location)) => {
-                $crate::log!(
-                    $crate::logging::Level::WARN,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    err.location = %location,
-                    $($arg)+
-                )
-            },
-            (None, None) => {
-                $crate::log!(
-                    $crate::logging::Level::WARN,
-                    err.msg = %$e,
-                    err.code = %$e.status_code(),
-                    $($arg)+
-                )
-            }
-        }
+        $crate::log!(
+            $crate::logging::Level::WARN,
+            err = ?$e,
+            $($arg)+
+        )
+    });
+
+    // warn!(%e; "a {} event", "log")
+    (%$e:expr; $($arg:tt)+) => ({
+        $crate::log!(
+            $crate::logging::Level::WARN,
+            err = %$e,
+            $($arg)+
+        )
     });
 
     // warn!("a {} event", "log")
@@ -305,8 +225,10 @@ mod tests {
         error!(target: "my_target", "hello {}", "world");
         // Supports both owned and reference type.
         error!(err; target: "my_target", "hello {}", "world");
+        error!(%err; target: "my_target", "hello {}", "world");
         error!(err_ref; target: "my_target", "hello {}", "world");
         error!(err_ref2; "hello {}", "world");
+        error!(%err_ref2; "hello {}", "world");
         error!("hello {}", "world");
 
         let root_err = MockError::with_source(err);

src/common/time/Cargo.toml

@@ -9,6 +9,7 @@ arrow.workspace = true
 chrono-tz = "0.8"
 chrono.workspace = true
 common-error = { workspace = true }
+common-macro = { workspace = true }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }

src/common/time/src/datetime.rs

@@ -20,11 +20,12 @@ use serde::{Deserialize, Serialize};
 
 use crate::error::{Error, InvalidDateStrSnafu, Result};
 use crate::util::{format_utc_datetime, local_datetime_to_utc};
+use crate::Date;
 
 const DATETIME_FORMAT: &str = "%F %T";
 const DATETIME_FORMAT_WITH_TZ: &str = "%F %T%z";
 
-/// [DateTime] represents the **seconds elapsed since "1970-01-01 00:00:00 UTC" (UNIX Epoch)**.
+/// [DateTime] represents the **milliseconds elapsed since "1970-01-01 00:00:00 UTC" (UNIX Epoch)**.
 #[derive(
     Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Serialize, Deserialize,
 )]
@@ -32,7 +33,7 @@ pub struct DateTime(i64);
 
 impl Display for DateTime {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        if let Some(abs_time) = NaiveDateTime::from_timestamp_opt(self.0, 0) {
+        if let Some(abs_time) = NaiveDateTime::from_timestamp_millis(self.0) {
             write!(
                 f,
                 "{}",
@@ -52,7 +53,7 @@ impl From<DateTime> for serde_json::Value {
 
 impl From<NaiveDateTime> for DateTime {
     fn from(value: NaiveDateTime) -> Self {
-        DateTime::from(value.timestamp())
+        DateTime::from(value.timestamp_millis())
     }
 }
 
@@ -61,20 +62,20 @@ impl FromStr for DateTime {
 
     fn from_str(s: &str) -> Result<Self> {
         let s = s.trim();
-        let timestamp = if let Ok(d) = NaiveDateTime::parse_from_str(s, DATETIME_FORMAT) {
+        let timestamp_millis = if let Ok(d) = NaiveDateTime::parse_from_str(s, DATETIME_FORMAT) {
             match local_datetime_to_utc(&d) {
                 LocalResult::None => {
                     return InvalidDateStrSnafu { raw: s }.fail();
                 }
-                LocalResult::Single(d) | LocalResult::Ambiguous(d, _) => d.timestamp(),
+                LocalResult::Single(d) | LocalResult::Ambiguous(d, _) => d.timestamp_millis(),
             }
         } else if let Ok(v) = chrono::DateTime::parse_from_str(s, DATETIME_FORMAT_WITH_TZ) {
-            v.timestamp()
+            v.timestamp_millis()
         } else {
             return InvalidDateStrSnafu { raw: s }.fail();
         };
 
-        Ok(Self(timestamp))
+        Ok(Self(timestamp_millis))
     }
 }
 
@@ -84,6 +85,13 @@ impl From<i64> for DateTime {
     }
 }
 
+impl From<Date> for DateTime {
+    fn from(value: Date) -> Self {
+        // It's safe, i32 * 86400000 won't be overflow
+        Self(value.to_secs() * 1000)
+    }
+}
+
 impl DateTime {
     pub fn new(val: i64) -> Self {
         Self(val)
@@ -94,7 +102,11 @@ impl DateTime {
     }
 
     pub fn to_chrono_datetime(&self) -> Option<NaiveDateTime> {
-        NaiveDateTime::from_timestamp_opt(self.0, 0)
+        NaiveDateTime::from_timestamp_millis(self.0)
+    }
+
+    pub fn to_date(&self) -> Option<Date> {
+        self.to_chrono_datetime().map(|d| Date::from(d.date()))
     }
 }
 
@@ -106,8 +118,8 @@ mod tests {
     pub fn test_new_date_time() {
         std::env::set_var("TZ", "Asia/Shanghai");
         assert_eq!("1970-01-01 08:00:00+0800", DateTime::new(0).to_string());
-        assert_eq!("1970-01-01 08:00:01+0800", DateTime::new(1).to_string());
-        assert_eq!("1970-01-01 07:59:59+0800", DateTime::new(-1).to_string());
+        assert_eq!("1970-01-01 08:00:01+0800", DateTime::new(1000).to_string());
+        assert_eq!("1970-01-01 07:59:59+0800", DateTime::new(-1000).to_string());
     }
 
     #[test]
@@ -130,7 +142,7 @@ mod tests {
     fn test_parse_local_date_time() {
         std::env::set_var("TZ", "Asia/Shanghai");
         assert_eq!(
-            -28800,
+            -28800000,
             DateTime::from_str("1970-01-01 00:00:00").unwrap().val()
         );
         assert_eq!(0, DateTime::from_str("1970-01-01 08:00:00").unwrap().val());
@@ -141,6 +153,24 @@ mod tests {
         let ts = DateTime::from_str("1970-01-01 08:00:00+0000")
             .unwrap()
             .val();
-        assert_eq!(28800, ts);
+        assert_eq!(28800000, ts);
+    }
+
+    #[test]
+    fn test_from_max_date() {
+        let date = Date::new(i32::MAX);
+        let datetime = DateTime::from(date);
+        assert_eq!(datetime.val(), 185542587100800000);
+    }
+
+    #[test]
+    fn test_conversion_between_datetime_and_chrono_datetime() {
+        let cases = [1, 10, 100, 1000, 100000];
+        for case in cases {
+            let dt = DateTime::new(case);
+            let ndt = dt.to_chrono_datetime().unwrap();
+            let dt2 = DateTime::from(ndt);
+            assert_eq!(dt, dt2);
+        }
     }
 }

src/common/time/src/error.rs

@@ -18,13 +18,19 @@ use std::num::{ParseIntError, TryFromIntError};
 use chrono::ParseError;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to parse string to date, raw: {}, source: {}", raw, source))]
-    ParseDateStr { raw: String, source: ParseError },
+    #[snafu(display("Failed to parse string to date, raw: {}", raw))]
+    ParseDateStr {
+        raw: String,
+        #[snafu(source)]
+        error: ParseError,
+    },
 
     #[snafu(display("Invalid date string, raw: {}", raw))]
     InvalidDateStr { raw: String, location: Location },
@@ -35,9 +41,10 @@ pub enum Error {
     #[snafu(display("Failed to parse a string into Interval, raw string: {}", raw))]
     ParseInterval { raw: String, location: Location },
 
-    #[snafu(display("Current timestamp overflow, source: {}", source))]
+    #[snafu(display("Current timestamp overflow"))]
     TimestampOverflow {
-        source: TryFromIntError,
+        #[snafu(source)]
+        error: TryFromIntError,
         location: Location,
     },
 
@@ -51,10 +58,11 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Invalid offset string {raw}: {source}"))]
+    #[snafu(display("Invalid offset string {raw}: "))]
     ParseOffsetStr {
         raw: String,
-        source: ParseIntError,
+        #[snafu(source)]
+        error: ParseIntError,
         location: Location,
     },
 

src/datanode/Cargo.toml

@@ -23,6 +23,7 @@ common-function = { workspace = true }
 common-greptimedb-telemetry = { workspace = true }
 common-grpc = { workspace = true }
 common-grpc-expr = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-procedure = { workspace = true }
 common-query = { workspace = true }

src/datanode/src/config.rs

@@ -18,6 +18,9 @@ use std::time::Duration;
 
 use common_base::readable_size::ReadableSize;
 use common_config::WalConfig;
+use common_grpc::channel_manager::{
+    DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
+};
 pub use common_procedure::options::ProcedureConfig;
 use common_telemetry::logging::LoggingOptions;
 use file_engine::config::EngineConfig as FileEngineConfig;
@@ -83,7 +86,7 @@ impl Default for StorageConfig {
     }
 }
 
-#[derive(Debug, Clone, Serialize, Default, Deserialize)]
+#[derive(Debug, Clone, Serialize, Default, Deserialize, Eq, PartialEq)]
 #[serde(default)]
 pub struct FileConfig {}
 
@@ -324,6 +327,10 @@ pub struct DatanodeOptions {
     pub rpc_addr: String,
     pub rpc_hostname: Option<String>,
     pub rpc_runtime_size: usize,
+    // Max gRPC receiving(decoding) message size
+    pub rpc_max_recv_message_size: usize,
+    // Max gRPC sending(encoding) message size
+    pub rpc_max_send_message_size: usize,
     pub heartbeat: HeartbeatOptions,
     pub http: HttpOptions,
     pub meta_client: Option<MetaClientOptions>,
@@ -344,6 +351,8 @@ impl Default for DatanodeOptions {
             rpc_addr: "127.0.0.1:3001".to_string(),
             rpc_hostname: None,
             rpc_runtime_size: 8,
+            rpc_max_recv_message_size: DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE,
+            rpc_max_send_message_size: DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
             http: HttpOptions::default(),
             meta_client: None,
             wal: WalConfig::default(),
@@ -369,7 +378,7 @@ impl DatanodeOptions {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub enum RegionEngineConfig {
     #[serde(rename = "mito")]
     Mito(MitoConfig),

src/datanode/src/datanode.rs

@@ -338,6 +338,7 @@ impl DatanodeBuilder {
             // query engine in datanode only executes plan with resolved table source.
             MemoryCatalogManager::with_default_setup(),
             None,
+            None,
             false,
             plugins,
         );

src/datanode/src/error.rs

@@ -16,6 +16,7 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use common_procedure::ProcedureId;
 use serde_json::error::Error as JsonError;
 use servers::define_into_tonic_status;
@@ -24,46 +25,47 @@ use store_api::storage::RegionId;
 use table::error::Error as TableError;
 
 /// Business error of datanode.
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to handle heartbeat response, source: {}", source))]
+    #[snafu(display("Failed to handle heartbeat response"))]
     HandleHeartbeatResponse {
         location: Location,
         source: common_meta::error::Error,
     },
 
-    #[snafu(display("Failed to get info from meta server, source: {}", source))]
+    #[snafu(display("Failed to get info from meta server"))]
     GetMetadata {
         location: Location,
         source: common_meta::error::Error,
     },
 
-    #[snafu(display("Failed to execute sql, source: {}", source))]
+    #[snafu(display("Failed to execute sql"))]
     ExecuteSql {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to plan statement, source: {}", source))]
+    #[snafu(display("Failed to plan statement"))]
     PlanStatement {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to execute statement, source: {}", source))]
+    #[snafu(display("Failed to execute statement"))]
     ExecuteStatement {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to execute logical plan, source: {}", source))]
+    #[snafu(display("Failed to execute logical plan"))]
     ExecuteLogicalPlan {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to decode logical plan, source: {}", source))]
+    #[snafu(display("Failed to decode logical plan"))]
     DecodeLogicalPlan {
         location: Location,
         source: substrait::error::Error,
@@ -78,32 +80,28 @@ pub enum Error {
     #[snafu(display("Schema not found: {}", name))]
     SchemaNotFound { name: String, location: Location },
 
-    #[snafu(display("Failed to create table: {}, source: {}", table_name, source))]
+    #[snafu(display("Failed to create table: {}", table_name))]
     CreateTable {
         table_name: String,
         location: Location,
         source: TableError,
     },
 
-    #[snafu(display("Failed to drop table {}, source: {}", table_name, source))]
+    #[snafu(display("Failed to drop table {}", table_name))]
     DropTable {
         table_name: String,
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
+    #[snafu(display("Table engine not found: {}", engine_name))]
     TableEngineNotFound {
         engine_name: String,
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display(
-        "Table engine procedure not found: {}, source: {}",
-        engine_name,
-        source
-    ))]
+    #[snafu(display("Table engine procedure not found: {}", engine_name))]
     EngineProcedureNotFound {
         engine_name: String,
         location: Location,
@@ -132,74 +130,80 @@ pub enum Error {
     ))]
     ColumnValuesNumberMismatch { columns: usize, values: usize },
 
-    #[snafu(display("Missing insert body, source: {source}"))]
+    #[snafu(display("Missing insert body"))]
     MissingInsertBody {
         source: sql::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to insert value to table: {}, source: {}", table_name, source))]
+    #[snafu(display("Failed to insert value to table: {}", table_name))]
     Insert {
         table_name: String,
         location: Location,
         source: TableError,
     },
 
-    #[snafu(display(
-        "Failed to delete value from table: {}, source: {}",
-        table_name,
-        source
-    ))]
+    #[snafu(display("Failed to delete value from table: {}", table_name))]
     Delete {
         table_name: String,
         location: Location,
         source: TableError,
     },
 
-    #[snafu(display("Failed to flush table: {}, source: {}", table_name, source))]
+    #[snafu(display("Failed to flush table: {}", table_name))]
     FlushTable {
         table_name: String,
         location: Location,
         source: TableError,
     },
 
-    #[snafu(display("Failed to start server, source: {}", source))]
+    #[snafu(display("Failed to start server"))]
     StartServer {
         location: Location,
         source: servers::error::Error,
     },
 
-    #[snafu(display("Failed to wait for GRPC serving, source: {}", source))]
+    #[snafu(display("Failed to wait for GRPC serving"))]
     WaitForGrpcServing {
         source: servers::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to parse address {}, source: {}", addr, source))]
+    #[snafu(display("Failed to parse address {}", addr))]
     ParseAddr {
         addr: String,
-        source: std::net::AddrParseError,
+        #[snafu(source)]
+        error: std::net::AddrParseError,
     },
 
-    #[snafu(display("Failed to create directory {}, source: {}", dir, source))]
-    CreateDir { dir: String, source: std::io::Error },
+    #[snafu(display("Failed to create directory {}", dir))]
+    CreateDir {
+        dir: String,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
 
-    #[snafu(display("Failed to remove directory {}, source: {}", dir, source))]
-    RemoveDir { dir: String, source: std::io::Error },
+    #[snafu(display("Failed to remove directory {}", dir))]
+    RemoveDir {
+        dir: String,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
 
-    #[snafu(display("Failed to open log store, source: {}", source))]
+    #[snafu(display("Failed to open log store"))]
     OpenLogStore {
         location: Location,
         source: Box<log_store::error::Error>,
     },
 
-    #[snafu(display("Failed to init backend, source: {}", source))]
+    #[snafu(display("Failed to init backend"))]
     InitBackend {
-        source: object_store::Error,
+        #[snafu(source)]
+        error: object_store::Error,
         location: Location,
     },
 
-    #[snafu(display("Runtime resource error, source: {}", source))]
+    #[snafu(display("Runtime resource error"))]
     RuntimeResource {
         location: Location,
         source: common_runtime::error::Error,
@@ -208,7 +212,7 @@ pub enum Error {
     #[snafu(display("Expect KvBackend but not found"))]
     MissingKvBackend { location: Location },
 
-    #[snafu(display("Expect MetaClient but not found, location: {}", location))]
+    #[snafu(display("Expect MetaClient but not found"))]
     MissingMetaClient { location: Location },
 
     #[snafu(display("Invalid SQL, error: {}", msg))]
@@ -232,7 +236,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to register a new schema, source: {}", source))]
+    #[snafu(display("Failed to register a new schema"))]
     RegisterSchema {
         location: Location,
         source: catalog::error::Error,
@@ -241,37 +245,37 @@ pub enum Error {
     #[snafu(display("Schema {} already exists", name))]
     SchemaExists { name: String, location: Location },
 
-    #[snafu(display("Failed to convert delete expr to request: {}", source))]
+    #[snafu(display("Failed to convert delete expr to request"))]
     DeleteExprToRequest {
         location: Location,
         source: common_grpc_expr::error::Error,
     },
 
-    #[snafu(display("Failed to parse SQL, source: {}", source))]
+    #[snafu(display("Failed to parse SQL"))]
     ParseSql {
         location: Location,
         source: sql::error::Error,
     },
 
-    #[snafu(display("Failed to prepare immutable table: {}", source))]
+    #[snafu(display("Failed to prepare immutable table"))]
     PrepareImmutableTable {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to access catalog, source: {}", source))]
+    #[snafu(display("Failed to access catalog"))]
     Catalog {
         location: Location,
         source: catalog::error::Error,
     },
 
-    #[snafu(display("Failed to initialize meta client, source: {}", source))]
+    #[snafu(display("Failed to initialize meta client"))]
     MetaClientInit {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Failed to insert data, source: {}", source))]
+    #[snafu(display("Failed to insert data"))]
     InsertData {
         location: Location,
         source: common_grpc_expr::error::Error,
@@ -282,16 +286,16 @@ pub enum Error {
     ))]
     TableIdProviderNotFound { location: Location },
 
-    #[snafu(display("Failed to bump table id, source: {}", source))]
+    #[snafu(display("Failed to bump table id"))]
     BumpTableId {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Missing node id in Datanode config, location: {}", location))]
+    #[snafu(display("Missing node id in Datanode config"))]
     MissingNodeId { location: Location },
 
-    #[snafu(display("Missing node id option in distributed mode, location: {}", location))]
+    #[snafu(display("Missing node id option in distributed mode"))]
     MissingMetasrvOpts { location: Location },
 
     #[snafu(display("Missing required field: {}", name))]
@@ -300,11 +304,7 @@ pub enum Error {
     #[snafu(display("Cannot find requested database: {}-{}", catalog, schema))]
     DatabaseNotFound { catalog: String, schema: String },
 
-    #[snafu(display(
-        "Failed to build default value, column: {}, source: {}",
-        column,
-        source
-    ))]
+    #[snafu(display("Failed to build default value, column: {}", column))]
     ColumnDefaultValue {
         column: String,
         location: Location,
@@ -317,42 +317,43 @@ pub enum Error {
     ))]
     ColumnNoneDefaultValue { column: String, location: Location },
 
-    #[snafu(display("Unrecognized table option: {}", source))]
+    #[snafu(display("Unrecognized table option"))]
     UnrecognizedTableOption {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to submit procedure {}, source: {}", procedure_id, source))]
+    #[snafu(display("Failed to submit procedure {}", procedure_id))]
     SubmitProcedure {
         procedure_id: ProcedureId,
         location: Location,
         source: common_procedure::error::Error,
     },
 
-    #[snafu(display("Failed to wait procedure {} done, source: {}", procedure_id, source))]
+    #[snafu(display("Failed to wait procedure {} done", procedure_id))]
     WaitProcedure {
         procedure_id: ProcedureId,
         location: Location,
         source: common_procedure::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown server, source: {}", source))]
+    #[snafu(display("Failed to shutdown server"))]
     ShutdownServer {
         location: Location,
         source: servers::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown instance, source: {}", source))]
+    #[snafu(display("Failed to shutdown instance"))]
     ShutdownInstance {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to encode object into json, source: {}", source))]
+    #[snafu(display("Failed to encode object into json"))]
     EncodeJson {
         location: Location,
-        source: JsonError,
+        #[snafu(source)]
+        error: JsonError,
     },
 
     #[snafu(display("Payload not exist"))]
@@ -361,13 +362,14 @@ pub enum Error {
     #[snafu(display("Missing WAL dir config"))]
     MissingWalDirConfig { location: Location },
 
-    #[snafu(display("Failed to join task, source: {}", source))]
+    #[snafu(display("Failed to join task"))]
     JoinTask {
-        source: common_runtime::JoinError,
+        #[snafu(source)]
+        error: common_runtime::JoinError,
         location: Location,
     },
 
-    #[snafu(display("Column datatype error, source: {}", source))]
+    #[snafu(display("Column datatype error"))]
     ColumnDataType {
         location: Location,
         source: api::error::Error,
@@ -379,46 +381,35 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display(
-        "Failed to handle request for region {}, source: {}, location: {}",
-        region_id,
-        source,
-        location
-    ))]
+    #[snafu(display("Failed to handle request for region {}", region_id))]
     HandleRegionRequest {
         region_id: RegionId,
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("RegionId {} not found, location: {}", region_id, location))]
+    #[snafu(display("RegionId {} not found", region_id))]
     RegionNotFound {
         region_id: RegionId,
         location: Location,
     },
 
-    #[snafu(display("Region engine {} is not registered, location: {}", name, location))]
+    #[snafu(display("Region engine {} is not registered", name))]
     RegionEngineNotFound { name: String, location: Location },
 
-    #[snafu(display("Unsupported gRPC request, kind: {}, location: {}", kind, location))]
+    #[snafu(display("Unsupported gRPC request, kind: {}", kind))]
     UnsupportedGrpcRequest { kind: String, location: Location },
 
-    #[snafu(display(
-        "Unsupported output type, expected: {}, location: {}",
-        expected,
-        location
-    ))]
+    #[snafu(display("Unsupported output type, expected: {}", expected))]
     UnsupportedOutput {
         expected: String,
         location: Location,
     },
 
     #[snafu(display(
-        "Failed to get metadata from engine {} for region_id {}, location: {}, source: {}",
+        "Failed to get metadata from engine {} for region_id {}",
         engine,
         region_id,
-        location,
-        source
     ))]
     GetRegionMetadata {
         engine: String,
@@ -427,22 +418,13 @@ pub enum Error {
         source: BoxedError,
     },
 
-    #[snafu(display(
-        "Failed to build region requests, location:{}, source: {}",
-        location,
-        source
-    ))]
+    #[snafu(display("Failed to build region requests"))]
     BuildRegionRequests {
         location: Location,
         source: store_api::metadata::MetadataError,
     },
 
-    #[snafu(display(
-        "Failed to stop region engine {}, location:{}, source: {}",
-        name,
-        location,
-        source
-    ))]
+    #[snafu(display("Failed to stop region engine {}", name))]
     StopRegionEngine {
         name: String,
         location: Location,

src/datanode/src/heartbeat.rs

@@ -18,6 +18,7 @@ use std::time::Duration;
 
 use api::v1::meta::{HeartbeatRequest, Peer, RegionStat, Role};
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
+use common_meta::distributed_time_constants::META_KEEP_ALIVE_INTERVAL_SECS;
 use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
 use common_meta::heartbeat::handler::{
     HandlerGroupExecutor, HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutorRef,
@@ -97,6 +98,7 @@ impl HeartbeatTask {
         handler_executor: HeartbeatResponseHandlerExecutorRef,
         mailbox: MailboxRef,
         mut notify: Option<Arc<Notify>>,
+        quit_signal: Arc<Notify>,
     ) -> Result<HeartbeatSender> {
         let client_id = meta_client.id();
 
@@ -123,7 +125,8 @@ impl HeartbeatTask {
                     info!("Heartbeat task shutdown");
                 }
             }
-            info!("Heartbeat handling loop exit.")
+            quit_signal.notify_one();
+            info!("Heartbeat handling loop exit.");
         });
         Ok(tx)
     }
@@ -167,12 +170,15 @@ impl HeartbeatTask {
         let (outgoing_tx, mut outgoing_rx) = mpsc::channel(16);
         let mailbox = Arc::new(HeartbeatMailbox::new(outgoing_tx));
 
+        let quit_signal = Arc::new(tokio::sync::Notify::new());
+
         let mut tx = Self::create_streams(
             &meta_client,
             running.clone(),
             handler_executor.clone(),
             mailbox.clone(),
             notify,
+            quit_signal.clone(),
         )
         .await?;
 
@@ -187,7 +193,6 @@ impl HeartbeatTask {
         common_runtime::spawn_bg(async move {
             let sleep = tokio::time::sleep(Duration::from_millis(0));
             tokio::pin!(sleep);
-
             loop {
                 if !running.load(Ordering::Relaxed) {
                     info!("shutdown heartbeat task");
@@ -228,6 +233,11 @@ impl HeartbeatTask {
                         sleep.as_mut().reset(now + Duration::from_millis(interval));
                         Some(req)
                     }
+                    // If the heartbeat stream is broken, send a dummy heartbeat request to re-create the heartbeat stream.
+                    _ = quit_signal.notified() => {
+                        let req = HeartbeatRequest::default();
+                        Some(req)
+                    }
                 };
                 if let Some(req) = req {
                     debug!("Sending heartbeat request: {:?}", req);
@@ -239,14 +249,24 @@ impl HeartbeatTask {
                             handler_executor.clone(),
                             mailbox.clone(),
                             None,
+                            quit_signal.clone(),
                         )
                         .await
                         {
                             Ok(new_tx) => {
                                 info!("Reconnected to metasrv");
                                 tx = new_tx;
+                                // Triggers to send heartbeat immediately.
+                                sleep.as_mut().reset(Instant::now());
                             }
                             Err(e) => {
+                                // Before the META_LEASE_SECS expires,
+                                // any retries are meaningless, it always reads the old meta leader address.
+                                // Triggers to retry after META_KEEP_ALIVE_INTERVAL_SECS.
+                                sleep.as_mut().reset(
+                                    Instant::now()
+                                        + Duration::from_secs(META_KEEP_ALIVE_INTERVAL_SECS),
+                                );
                                 error!(e;"Failed to reconnect to metasrv!");
                             }
                         }
@@ -315,13 +335,19 @@ pub async fn new_metasrv_client(
         .timeout(Duration::from_millis(meta_config.timeout_millis))
         .connect_timeout(Duration::from_millis(meta_config.connect_timeout_millis))
         .tcp_nodelay(meta_config.tcp_nodelay);
-    let channel_manager = ChannelManager::with_config(config);
+    let channel_manager = ChannelManager::with_config(config.clone());
+    let heartbeat_channel_manager = ChannelManager::with_config(
+        config
+            .timeout(Duration::from_millis(meta_config.heartbeat_timeout_millis))
+            .connect_timeout(Duration::from_millis(meta_config.heartbeat_timeout_millis)),
+    );
 
     let mut meta_client = MetaClientBuilder::new(cluster_id, member_id, Role::Datanode)
         .enable_heartbeat()
         .enable_router()
         .enable_store()
         .channel_manager(channel_manager)
+        .heartbeat_channel_manager(heartbeat_channel_manager)
         .build();
     meta_client
         .start(&meta_config.metasrv_addrs)

src/datanode/src/lib.rs

@@ -15,8 +15,6 @@
 #![feature(assert_matches)]
 #![feature(trait_upcasting)]
 
-use query::query_engine::SqlStatementExecutor;
-
 pub mod alive_keeper;
 pub mod config;
 pub mod datanode;
@@ -31,17 +29,3 @@ mod store;
 #[cfg(test)]
 #[allow(dead_code)]
 mod tests;
-
-// TODO(ruihang): remove this
-pub struct Instance;
-
-#[async_trait::async_trait]
-impl SqlStatementExecutor for Instance {
-    async fn execute_sql(
-        &self,
-        _stmt: sql::statements::statement::Statement,
-        _query_ctx: session::context::QueryContextRef,
-    ) -> query::error::Result<common_query::Output> {
-        unreachable!()
-    }
-}

src/datanode/src/region_server.rs

@@ -36,7 +36,7 @@ use datafusion::datasource::TableProvider;
 use datafusion::error::Result as DfResult;
 use datafusion::execution::context::SessionState;
 use datafusion_common::DataFusionError;
-use datafusion_expr::{Expr as DfExpr, TableType};
+use datafusion_expr::{Expr as DfExpr, TableProviderFilterPushDown, TableType};
 use datatypes::arrow::datatypes::SchemaRef;
 use futures_util::future::try_join_all;
 use prost::Message;
@@ -498,4 +498,11 @@ impl TableProvider for DummyTableProvider {
             StreamScanAdapter::new(stream),
         ))))
     }
+
+    fn supports_filters_pushdown(
+        &self,
+        filters: &[&DfExpr],
+    ) -> DfResult<Vec<TableProviderFilterPushDown>> {
+        Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()])
+    }
 }

src/datanode/src/server.rs

@@ -16,7 +16,7 @@ use std::net::SocketAddr;
 use std::sync::Arc;
 
 use futures::future;
-use servers::grpc::GrpcServer;
+use servers::grpc::{GrpcServer, GrpcServerConfig};
 use servers::http::{HttpServer, HttpServerBuilder};
 use servers::metrics_handler::MetricsHandler;
 use servers::server::Server;
@@ -39,9 +39,14 @@ impl Services {
         let flight_handler = Some(Arc::new(region_server.clone()) as _);
         let region_server_handler = Some(Arc::new(region_server.clone()) as _);
         let runtime = region_server.runtime();
+        let grpc_config = GrpcServerConfig {
+            max_recv_message_size: opts.rpc_max_recv_message_size,
+            max_send_message_size: opts.rpc_max_send_message_size,
+        };
 
         Ok(Self {
             grpc_server: GrpcServer::new(
+                Some(grpc_config),
                 None,
                 None,
                 flight_handler,

src/datanode/src/tests.rs

@@ -23,7 +23,7 @@ use common_meta::heartbeat::handler::{
     HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutor,
 };
 use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MessageMeta};
-use common_meta::instruction::{Instruction, InstructionReply, OpenRegion, RegionIdent};
+use common_meta::instruction::{Instruction, OpenRegion, RegionIdent};
 use common_query::prelude::ScalarUdf;
 use common_query::Output;
 use common_runtime::Runtime;
@@ -34,29 +34,9 @@ use query::query_engine::DescribeResult;
 use query::QueryEngine;
 use session::context::QueryContextRef;
 use table::TableRef;
-use tokio::sync::mpsc::{self, Receiver};
 
 use crate::event_listener::NoopRegionServerEventListener;
 use crate::region_server::RegionServer;
-use crate::Instance;
-
-struct HandlerTestGuard {
-    instance: Instance,
-    mailbox: Arc<HeartbeatMailbox>,
-    rx: Receiver<(MessageMeta, InstructionReply)>,
-}
-
-async fn prepare_handler_test(_name: &str) -> HandlerTestGuard {
-    let instance = Instance;
-    let (tx, rx) = mpsc::channel(8);
-    let mailbox = Arc::new(HeartbeatMailbox::new(tx));
-
-    HandlerTestGuard {
-        instance,
-        mailbox,
-        rx,
-    }
-}
 
 pub fn test_message_meta(id: u64, subject: &str, to: &str, from: &str) -> MessageMeta {
     MessageMeta {

src/datatypes/Cargo.toml

@@ -14,6 +14,7 @@ arrow-schema.workspace = true
 arrow.workspace = true
 common-base = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-telemetry = { workspace = true }
 common-time = { workspace = true }
 datafusion-common.workspace = true

src/datatypes/src/data_type.rs

@@ -179,6 +179,10 @@ impl ConcreteDataType {
         )
     }
 
+    pub fn is_timestamp(&self) -> bool {
+        matches!(self, ConcreteDataType::Timestamp(_))
+    }
+
     pub fn numerics() -> Vec<ConcreteDataType> {
         vec![
             ConcreteDataType::int8_datatype(),
@@ -217,9 +221,6 @@ impl ConcreteDataType {
     /// Try to cast data type as a [`TimestampType`].
     pub fn as_timestamp(&self) -> Option<TimestampType> {
         match self {
-            ConcreteDataType::Int64(_) => {
-                Some(TimestampType::Millisecond(TimestampMillisecondType))
-            }
             ConcreteDataType::Timestamp(t) => Some(*t),
             _ => None,
         }
@@ -473,10 +474,6 @@ pub trait DataType: std::fmt::Debug + Send + Sync {
     /// Creates a mutable vector with given `capacity` of this type.
     fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector>;
 
-    /// Returns true if the data type is compatible with timestamp type so we can
-    /// use it as a timestamp.
-    fn is_timestamp_compatible(&self) -> bool;
-
     /// Casts the value to specific DataType.
     /// Return None if cast failed.
     fn try_cast(&self, from: Value) -> Option<Value>;
@@ -596,41 +593,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_is_timestamp_compatible() {
-        assert!(ConcreteDataType::timestamp_datatype(TimeUnit::Second).is_timestamp_compatible());
-        assert!(
-            ConcreteDataType::timestamp_datatype(TimeUnit::Millisecond).is_timestamp_compatible()
-        );
-        assert!(
-            ConcreteDataType::timestamp_datatype(TimeUnit::Microsecond).is_timestamp_compatible()
-        );
-        assert!(
-            ConcreteDataType::timestamp_datatype(TimeUnit::Nanosecond).is_timestamp_compatible()
-        );
-        assert!(ConcreteDataType::timestamp_second_datatype().is_timestamp_compatible());
-        assert!(ConcreteDataType::timestamp_millisecond_datatype().is_timestamp_compatible());
-        assert!(ConcreteDataType::timestamp_microsecond_datatype().is_timestamp_compatible());
-        assert!(ConcreteDataType::timestamp_nanosecond_datatype().is_timestamp_compatible());
-        assert!(ConcreteDataType::int64_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::null_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::binary_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::boolean_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::date_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::datetime_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::string_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::int32_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::uint64_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::time_second_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::time_millisecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::time_microsecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::time_nanosecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::duration_second_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::duration_millisecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::duration_microsecond_datatype().is_timestamp_compatible());
-        assert!(!ConcreteDataType::duration_nanosecond_datatype().is_timestamp_compatible());
-    }
-
     #[test]
     fn test_is_null() {
         assert!(ConcreteDataType::null_datatype().is_null());

src/datatypes/src/error.rs

@@ -16,20 +16,24 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to serialize data, source: {}", source))]
+    #[snafu(display("Failed to serialize data"))]
     Serialize {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
     },
 
-    #[snafu(display("Failed to deserialize data, source: {}, json: {}", source, json))]
+    #[snafu(display("Failed to deserialize data, json: {}", json))]
     Deserialize {
-        source: serde_json::Error,
+        #[snafu(source)]
+        error: serde_json::Error,
         location: Location,
         json: String,
     },
@@ -60,14 +64,11 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display(
-        "Failed to parse version in schema meta, value: {}, source: {}",
-        value,
-        source
-    ))]
+    #[snafu(display("Failed to parse version in schema meta, value: {}", value))]
     ParseSchemaVersion {
         value: String,
-        source: std::num::ParseIntError,
+        #[snafu(source)]
+        error: std::num::ParseIntError,
         location: Location,
     },
 
@@ -79,19 +80,22 @@ pub enum Error {
 
     #[snafu(display("Failed to cast arrow time i32 type into i64"))]
     CastTimeType {
-        source: std::num::TryFromIntError,
+        #[snafu(source)]
+        error: std::num::TryFromIntError,
         location: Location,
     },
 
-    #[snafu(display("Arrow failed to compute, source: {}", source))]
+    #[snafu(display("Arrow failed to compute"))]
     ArrowCompute {
-        source: arrow::error::ArrowError,
+        #[snafu(source)]
+        error: arrow::error::ArrowError,
         location: Location,
     },
 
-    #[snafu(display("Failed to project arrow schema, source: {}", source))]
+    #[snafu(display("Failed to project arrow schema"))]
     ProjectArrowSchema {
-        source: arrow::error::ArrowError,
+        #[snafu(source)]
+        error: arrow::error::ArrowError,
         location: Location,
     },