ci: tolerate error while building arm64 releases (#1143 )

* ci: allow failure while building arm64 docker * ci: Remove continue-on-error on docker step
ci: Disable arm64 release temporarily (#1141 )
2025-12-27 08:29:59 +00:00 · 2023-03-08 21:11:40 +08:00 · 2023-03-08 19:13:00 +08:00 · 2023-03-08 18:12:06 +08:00 · 2023-03-08 08:10:12 +00:00 · 2023-03-08 07:29:34 +00:00
504 changed files with 25779 additions and 7723 deletions
--- a/.github/pr-title-breaking-change-label-config.json
+++ b/.github/pr-title-breaking-change-label-config.json
@@ -0,0 +1,13 @@
+{
+    "LABEL": {
+        "name": "breaking change",
+        "color": "D93F0B"
+    },
+    "CHECKS": {
+        "regexp": "^(?:(?!!:).)*$",
+        "ignoreLabels": [
+            "ignore-title"
+        ],
+        "alwaysPassCI": true
+    }
+}
--- a/.github/pr-title-checker-config.json
+++ b/.github/pr-title-checker-config.json
@@ -1,10 +1,12 @@
 {
-  "LABEL": {
-    "name": "Invalid PR Title",
-    "color": "B60205"
-  },
-  "CHECKS": {
-    "regexp": "^(feat|fix|test|refactor|chore|style|docs|perf|build|ci|revert)(\\(.*\\))?:.*",
-    "ignoreLabels" : ["ignore-title"]
-  }
+    "LABEL": {
+        "name": "Invalid PR Title",
+        "color": "B60205"
+    },
+    "CHECKS": {
+        "regexp": "^(feat|fix|test|refactor|chore|style|docs|perf|build|ci|revert)(\\(.*\\))?\\!?:.*",
+        "ignoreLabels": [
+            "ignore-title"
+        ]
+    }
 }
--- a/.github/workflows/apidoc.yml
+++ b/.github/workflows/apidoc.yml
@@ -0,0 +1,42 @@
+on:
+  push:
+    branches:
+      - develop
+    paths-ignore:
+      - 'docs/**'
+      - 'config/**'
+      - '**.md'
+      - '.dockerignore'
+      - 'docker/**'
+      - '.gitignore'
+
+name: Build API docs
+
+env:
+  RUST_TOOLCHAIN: nightly-2023-02-26
+
+jobs:
+  apidoc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: arduino/setup-protoc@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ env.RUST_TOOLCHAIN }}
+    - run: cargo doc --workspace --no-deps --document-private-items
+    - run: |
+        cat <<EOF > target/doc/index.html
+        <!DOCTYPE html>
+        <html>
+        <head>
+        <meta http-equiv="refresh" content="0; url='greptime/'" />
+        </head>
+        <body></body></html>
+        EOF
+    - name: Publish dist directory
+      uses: JamesIves/github-pages-deploy-action@v4
+      with:
+        folder: target/doc
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -24,15 +24,15 @@ on:
 name: CI

 env:
-  RUST_TOOLCHAIN: nightly-2022-12-20
+  RUST_TOOLCHAIN: nightly-2023-02-26

 jobs:
  typos:
    name: Spell Check with Typos
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
-      - uses: crate-ci/typos@v1.0.4
+      - uses: actions/checkout@v3
+      - uses: crate-ci/typos@v1.13.10

  check:
    name: Check
@@ -116,6 +116,7 @@ jobs:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest-8-cores
    timeout-minutes: 60
+    needs: [clippy]
    steps:
      - uses: actions/checkout@v3
      - uses: arduino/setup-protoc@v1
@@ -131,7 +132,7 @@ jobs:
          ETCD_VER=v3.5.7
          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-          mkdir -p /tmp/etcd-download 
+          mkdir -p /tmp/etcd-download
          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz

@@ -188,6 +189,7 @@ jobs:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest-8-cores
    timeout-minutes: 60
+    needs: [clippy]
    steps:
      - uses: actions/checkout@v3
      - uses: arduino/setup-protoc@v1
@@ -205,10 +207,16 @@ jobs:
        uses: Swatinem/rust-cache@v2
      - name: Install latest nextest release
        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov
      - name: Collect coverage data
-        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info
+        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend
        env:
          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
          RUST_BACKTRACE: 1
@@ -223,5 +231,5 @@ jobs:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./lcov.info
          flags: rust
-          fail_ci_if_error: true
+          fail_ci_if_error: false
          verbose: true
--- a/.github/workflows/doc-issue.yml
+++ b/.github/workflows/doc-issue.yml
@@ -1,4 +1,4 @@
-name: Create Issue in docs repo on doc related changes
+name: Create Issue in downstream repos

 on:
  issues:
@@ -23,3 +23,17 @@ jobs:
          body: |
            A document change request is generated from
            ${{ github.event.issue.html_url || github.event.pull_request.html_url }}
+  cloud_issue:
+    if: github.event.label.name == 'cloud followup required'
+    runs-on: ubuntu-latest
+    steps:
+      - name: create an issue in cloud repo
+        uses: dacbd/create-issue-action@main
+        with:
+          owner: GreptimeTeam
+          repo: greptimedb-cloud
+          token: ${{ secrets.DOCS_REPO_TOKEN }}
+          title: Followup changes in ${{ github.event.issue.title || github.event.pull_request.title }}
+          body: |
+            A followup request is generated from
+            ${{ github.event.issue.html_url || github.event.pull_request.html_url }}
--- a/.github/workflows/license.yaml
+++ b/.github/workflows/license.yaml
@@ -13,4 +13,4 @@ jobs:
    steps:
    - uses: actions/checkout@v2
    - name: Check License Header
-      uses: apache/skywalking-eyes/header@main
+      uses: apache/skywalking-eyes/header@df70871af1a8109c9a5b1dc824faaf65246c5236
--- a/.github/workflows/pr-title-checker.yml
+++ b/.github/workflows/pr-title-checker.yml
@@ -18,3 +18,12 @@ jobs:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          pass_on_octokit_error: false
          configuration_path: ".github/pr-title-checker-config.json"
+  breaking:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: thehanimo/pr-title-checker@v1.3.4
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          pass_on_octokit_error: false
+          configuration_path: ".github/pr-title-breaking-change-label-config.json"
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -10,7 +10,7 @@ on:
 name: Release

 env:
-  RUST_TOOLCHAIN: nightly-2022-12-20
+  RUST_TOOLCHAIN: nightly-2023-02-26

  # FIXME(zyy17): Would be better to use `gh release list -L 1 | cut -f 3` to get the latest release version tag, but for a long time, we will stay at 'v0.1.0-alpha-*'.
  SCHEDULED_BUILD_VERSION_PREFIX: v0.1.0-alpha
@@ -30,16 +30,21 @@ jobs:
          - arch: x86_64-unknown-linux-gnu
            os: ubuntu-2004-16-cores
            file: greptime-linux-amd64
+            continue-on-error: false
          - arch: aarch64-unknown-linux-gnu
            os: ubuntu-2004-16-cores
            file: greptime-linux-arm64
+            continue-on-error: true
          - arch: aarch64-apple-darwin
            os: macos-latest
            file: greptime-darwin-arm64
+            continue-on-error: true
          - arch: x86_64-apple-darwin
            os: macos-latest
            file: greptime-darwin-amd64
+            continue-on-error: true
    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.continue-on-error }}
    if: github.repository == 'GreptimeTeam/greptimedb'
    steps:
      - name: Checkout sources
@@ -76,10 +81,10 @@ jobs:
          ETCD_VER=v3.5.7
          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-          mkdir -p /tmp/etcd-download 
-          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1 
+          mkdir -p /tmp/etcd-download
+          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-          
+
          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
          nohup etcd >/tmp/etcd.log 2>&1 &

@@ -181,30 +186,6 @@ jobs:
      - name: Checkout sources
        uses: actions/checkout@v3

-      - name: Download amd64 binary
-        uses: actions/download-artifact@v3
-        with:
-          name: greptime-linux-amd64
-          path: amd64
-
-      - name: Unzip the amd64 artifacts
-        run: |
-          cd amd64
-          tar xvf greptime-linux-amd64.tgz
-          rm greptime-linux-amd64.tgz
-
-      - name: Download arm64 binary
-        uses: actions/download-artifact@v3
-        with:
-          name: greptime-linux-arm64
-          path: arm64
-
-      - name: Unzip the arm64 artifacts
-        run: |
-          cd arm64
-          tar xvf greptime-linux-arm64.tgz
-          rm greptime-linux-arm64.tgz
-
      - name: Login to UCloud Container Registry
        uses: docker/login-action@v2
        with:
@@ -239,8 +220,37 @@ jobs:
      - name: Set up buildx
        uses: docker/setup-buildx-action@v2

-      - name: Build and push
+      - name: Download amd64 binary
+        uses: actions/download-artifact@v3
+        with:
+          name: greptime-linux-amd64
+          path: amd64
+
+      - name: Unzip the amd64 artifacts
+        id: unzip-amd64
+        run: |
+          cd amd64
+          tar xvf greptime-linux-amd64.tgz
+          rm greptime-linux-amd64.tgz
+
+      - name: Download arm64 binary
+        id: download-arm64
+        uses: actions/download-artifact@v3
+        with:
+          name: greptime-linux-arm64
+          path: arm64
+
+      - name: Unzip the arm64 artifacts
+        id: unzip-arm64
+        if: success() || steps.download-arm64.conclusion == 'success'
+        run: |
+          cd arm64
+          tar xvf greptime-linux-arm64.tgz
+          rm greptime-linux-arm64.tgz
+
+      - name: Build and push all
        uses: docker/build-push-action@v3
+        if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds
        with:
          context: .
          file: ./docker/ci/Dockerfile
@@ -251,3 +261,17 @@ jobs:
            greptime/greptimedb:${{ env.IMAGE_TAG }}
            uhub.service.ucloud.cn/greptime/greptimedb:latest
            uhub.service.ucloud.cn/greptime/greptimedb:${{ env.IMAGE_TAG }}
+
+      - name: Build and push amd64 only
+        uses: docker/build-push-action@v3
+        if: success() || steps.unzip-arm64.conclusion == 'failure' # Only build and push amd64 platform if unzip-arm64 fails
+        with:
+          context: .
+          file: ./docker/ci/Dockerfile
+          push: true
+          platforms: linux/amd64
+          tags: |
+            greptime/greptimedb:latest
+            greptime/greptimedb:${{ env.IMAGE_TAG }}
+            uhub.service.ucloud.cn/greptime/greptimedb:latest
+            uhub.service.ucloud.cn/greptime/greptimedb:${{ env.IMAGE_TAG }}
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,4 +1,4 @@
-# Welcome!
+# Welcome 👋

 Thanks a lot for considering contributing to GreptimeDB. We believe people like you would make GreptimeDB a great product. We intend to build a community where individuals can have open talks, show respect for one another, and speak with true ❤️. Meanwhile, we are to keep transparency and make your effort count here.

@@ -50,34 +50,33 @@ GreptimeDB uses the [Apache 2.0 license](https://github.com/GreptimeTeam/greptim

 - To ensure that community is free and confident in its ability to use your contributions, please sign the Contributor License Agreement (CLA) which will be incorporated in the pull request process.
 - Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/).
- Make sure all unit tests are passed.
+- Make sure all unit tests are passed (using `cargo test --workspace` or [nextest](https://nexte.st/index.html) `cargo nextest run`).
 - Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings -D clippy::print_stdout -D clippy::print_stderr`).

 #### `pre-commit` Hooks
+
 You could setup the [`pre-commit`](https://pre-commit.com/#plugins) hooks to run these checks on every commit automatically.

 1. Install `pre-commit`
-```
-$ pip install pre-commit
-```
-or
-```
-$ brew install pre-commit
-```
+
+        pip install pre-commit
+
+    or
+
+        brew install pre-commit

 2. Install the `pre-commit` hooks
-```
-$ pre-commit install
-pre-commit installed at .git/hooks/pre-commit

-$ pre-commit install --hook-type commit-msg
-pre-commit installed at .git/hooks/commit-msg
+        $ pre-commit install
+        pre-commit installed at .git/hooks/pre-commit

-$ pre-commit install --hook-type pre-push
-pre-commit installed at .git/hooks/pre-pus
-```
+        $ pre-commit install --hook-type commit-msg
+        pre-commit installed at .git/hooks/commit-msg

-now `pre-commit` will run automatically on `git commit`.
+        $ pre-commit install --hook-type pre-push
+        pre-commit installed at .git/hooks/pre-push
+
+Now, `pre-commit` will run automatically on `git commit`.

 ### Title

@@ -102,10 +101,12 @@ of what you were trying to do and what went wrong. You can also reach for help i
 ## Community

 The core team will be thrilled if you participate in any way you like. When you are stuck, try ask for help by filing an issue, with a detailed description of what you were trying to do and what went wrong. If you have any questions or if you would like to get involved in our community, please check out:
+
 - [GreptimeDB Community Slack](https://greptime.com/slack)
 - [GreptimeDB Github Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)

 Also, see some extra GreptimeDB content:
+
 - [GreptimeDB Docs](https://greptime.com/docs)
 - [Learn GreptimeDB](https://greptime.com/products/db)
 - [Greptime Inc. Website](https://greptime.com)
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,12 +12,14 @@ members = [
    "src/common/function-macro",
    "src/common/grpc",
    "src/common/grpc-expr",
+    "src/common/mem-prof",
    "src/common/procedure",
    "src/common/query",
    "src/common/recordbatch",
    "src/common/runtime",
    "src/common/substrait",
    "src/common/telemetry",
+    "src/common/test-util",
    "src/common/time",
    "src/datanode",
    "src/datatypes",
@@ -37,6 +39,7 @@ members = [
    "src/storage",
    "src/store-api",
    "src/table",
+    "src/table-procedure",
    "tests-integration",
    "tests/runner",
 ]
@@ -47,31 +50,32 @@ edition = "2021"
 license = "Apache-2.0"

 [workspace.dependencies]
-arrow = "29.0"
-arrow-flight = "29.0"
-arrow-schema = { version = "29.0", features = ["serde"] }
+arrow = { version = "33.0", features = ["pyarrow"] }
+arrow-array = "33.0"
+arrow-flight = "33.0"
+arrow-schema = { version = "33.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-# TODO(LFC): Use released Datafusion when it officially dpendent on Arrow 29.0
-datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
-datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
-datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
-datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
-datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
-datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "fad360df0132a2fcb264a7c07b2b02f0b1dfc644" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "fad360df0132a2fcb264a7c07b2b02f0b1dfc644" }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "fad360df0132a2fcb264a7c07b2b02f0b1dfc644" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "fad360df0132a2fcb264a7c07b2b02f0b1dfc644" }
+datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "fad360df0132a2fcb264a7c07b2b02f0b1dfc644" }
+datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "fad360df0132a2fcb264a7c07b2b02f0b1dfc644" }
 futures = "0.3"
 futures-util = "0.3"
-parquet = "29.0"
+parquet = "33.0"
 paste = "1.0"
 prost = "0.11"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }
-sqlparser = "0.28"
+sqlparser = "0.30"
+tempfile = "3"
 tokio = { version = "1.24.2", features = ["full"] }
 tokio-util = "0.7"
-tonic = "0.8"
+tonic = { version = "0.8", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }

 [profile.release]
--- a/4
+++ b/4
@@ -19,6 +19,10 @@ clean: ## Clean the project.
 fmt: ## Format all the Rust code.
 	cargo fmt --all

+.PHONY: fmt-toml
+fmt-toml: ## Format all TOML files.
+	taplo format --check --option "indent_string=    "
+
 .PHONY: docker-image
 docker-image: ## Build docker image.
 	docker build --network host -f docker/Dockerfile -t ${IMAGE_REGISTRY}:${IMAGE_TAG} .
--- a/README.md
+++ b/README.md
@@ -61,6 +61,12 @@ To compile GreptimeDB from source, you'll need:
  find an installation instructions [here](https://grpc.io/docs/protoc-installation/).
  **Note that `protoc` version needs to be >= 3.15** because we have used the `optional`
  keyword. You can check it with `protoc --version`.
+- python3-dev or python3-devel(Optional, only needed if you want to run scripts
+  in cpython): this install a Python shared library required for running python
+  scripting engine(In CPython Mode). This is available as `python3-dev` on
+  ubuntu, you can install it with `sudo apt install python3-dev`, or
+  `python3-devel` on RPM based distributions (e.g. Fedora, Red Hat, SuSE). Mac's
+  `Python3` package should have this shared library by default.

 #### Build with Docker

--- a/benchmarks/src/bin/nyc-taxi.rs
+++ b/benchmarks/src/bin/nyc-taxi.rs
@@ -27,7 +27,7 @@ use arrow::record_batch::RecordBatch;
 use clap::Parser;
 use client::api::v1::column::Values;
 use client::api::v1::{Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, TableId};
-use client::{Client, Database};
+use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
 use tokio::task::JoinSet;
@@ -208,6 +208,7 @@ fn build_values(column: &ArrayRef) -> Values {
        | DataType::Dictionary(_, _)
        | DataType::Decimal128(_, _)
        | DataType::Decimal256(_, _)
+        | DataType::RunEndEncoded(_, _)
        | DataType::Map(_, _) => todo!(),
    }
 }
@@ -422,7 +423,7 @@ fn main() {
        .unwrap()
        .block_on(async {
            let client = Client::with_urls(vec![&args.endpoint]);
-            let db = Database::with_client(client);
+            let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);

            if !args.skip_write {
                do_write(&args, &db).await;
--- a/codecov.yml
+++ b/codecov.yml
@@ -8,3 +8,5 @@ coverage:
 ignore:
  - "**/error*.rs" # ignore all error.rs files
  - "tests/runner/*.rs" # ignore integration test runner
+comment:                  # this is a top-level key
+  layout: "diff"
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -1,26 +1,52 @@
-node_id = 42
-mode = 'distributed'
-rpc_addr = '127.0.0.1:3001'
-rpc_hostname = '127.0.0.1'
-rpc_runtime_size = 8
-mysql_addr = '127.0.0.1:4406'
-mysql_runtime_size = 4
+# Node running mode, see `standalone.example.toml`.
+mode = "distributed"
+# Whether to use in-memory catalog, see `standalone.example.toml`.
 enable_memory_catalog = false
+# The datanode identifier, should be unique.
+node_id = 42
+# gRPC server address, "127.0.0.1:3001" by default.
+rpc_addr = "127.0.0.1:3001"
+# Hostname of this node.
+rpc_hostname = "127.0.0.1"
+# The number of gRPC server worker threads, 8 by default.
+rpc_runtime_size = 8
+# MySQL server address, "127.0.0.1:4406" by default.
+mysql_addr = "127.0.0.1:4406"
+# The number of MySQL server worker threads, 2 by default.
+mysql_runtime_size = 2

+# Metasrv client options.
+[meta_client_options]
+# Metasrv address list.
+metasrv_addrs = ["127.0.0.1:3002"]
+# Operation timeout in milliseconds, 3000 by default.
+timeout_millis = 3000
+# Connect server timeout in milliseconds, 5000 by default.
+connect_timeout_millis = 5000
+# `TCP_NODELAY` option for accepted connections, true by default.
+tcp_nodelay = true
+
+# WAL options, see `standalone.example.toml`.
 [wal]
 dir = "/tmp/greptimedb/wal"
-file_size = '1GB'
-purge_interval = '10m'
-purge_threshold = '50GB'
+file_size = "1GB"
+purge_threshold = "50GB"
+purge_interval = "10m"
 read_batch_size = 128
 sync_write = false

+# Storage options, see `standalone.example.toml`.
 [storage]
-type = 'File'
-data_dir = '/tmp/greptimedb/data/'
+type = "File"
+data_dir = "/tmp/greptimedb/data/"

-[meta_client_opts]
-metasrv_addrs = ['127.0.0.1:3002']
-timeout_millis = 3000
-connect_timeout_millis = 5000
-tcp_nodelay = false
+# Compaction options, see `standalone.example.toml`.
+[compaction]
+max_inflight_tasks = 4
+max_files_in_level0 = 8
+max_purge_tasks = 32
+
+# Procedure storage options, see `standalone.example.toml`.
+# [procedure.store]
+# type = 'File'
+# data_dir = '/tmp/greptimedb/procedure/'
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -1,12 +1,58 @@
-mode = 'distributed'
-datanode_rpc_addr = '127.0.0.1:3001'
+# Node running mode, see `standalone.example.toml`.
+mode = "distributed"

+# HTTP server options, see `standalone.example.toml`.
 [http_options]
-addr = '127.0.0.1:4000'
+addr = "127.0.0.1:4000"
 timeout = "30s"

-[meta_client_opts]
-metasrv_addrs = ['127.0.0.1:3002']
+# gRPC server options, see `standalone.example.toml`.
+[grpc_options]
+addr = "127.0.0.1:4001"
+runtime_size = 8
+
+# MySQL server options, see `standalone.example.toml`.
+[mysql_options]
+addr = "127.0.0.1:4002"
+runtime_size = 2
+
+# MySQL server TLS options, see `standalone.example.toml`.
+[mysql_options.tls]
+mode = "disable"
+cert_path = ""
+key_path = ""
+
+# PostgresSQL server options, see `standalone.example.toml`.
+[postgres_options]
+addr = "127.0.0.1:4003"
+runtime_size = 2
+
+# PostgresSQL server TLS options, see `standalone.example.toml`.
+[postgres_options.tls]
+mode = "disable"
+cert_path = ""
+key_path = ""
+
+# OpenTSDB protocol options, see `standalone.example.toml`.
+[opentsdb_options]
+addr = "127.0.0.1:4242"
+runtime_size = 2
+
+# InfluxDB protocol options, see `standalone.example.toml`.
+[influxdb_options]
+enable = true
+
+# Prometheus protocol options, see `standalone.example.toml`.
+[prometheus_options]
+enable = true
+
+# Prometheus protocol options, see `standalone.example.toml`.
+[prom_options]
+addr = "127.0.0.1:4004"
+
+# Metasrv client options, see `datanode.example.toml`.
+[meta_client_options]
+metasrv_addrs = ["127.0.0.1:3002"]
 timeout_millis = 3000
 connect_timeout_millis = 5000
-tcp_nodelay = false
+tcp_nodelay = true
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -1,6 +1,15 @@
-bind_addr = '127.0.0.1:3002'
-server_addr = '127.0.0.1:3002'
-store_addr = '127.0.0.1:2379'
+# The bind address of metasrv, "127.0.0.1:3002" by default.
+bind_addr = "127.0.0.1:3002"
+# The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost.
+server_addr = "127.0.0.1:3002"
+# Etcd server address, "127.0.0.1:2379" by default.
+store_addr = "127.0.0.1:2379"
+# Datanode lease in seconds, 15 seconds by default.
 datanode_lease_secs = 15
-# selector: 'LeaseBased', 'LoadBased'
-selector = 'LeaseBased'
+# Datanode selector type.
+# - "LeaseBased" (default value).
+# - "LoadBased"
+# For details, please see "https://docs.greptime.com/developer-guide/meta/selector".
+selector = "LeaseBased"
+# Store data in memory, false by default.
+use_memory_store = false
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -1,44 +1,116 @@
-node_id = 0
-mode = 'standalone'
+# Node running mode, "standalone" or "distributed".
+mode = "standalone"
+# Whether to use in-memory catalog, `false` by default.
 enable_memory_catalog = false

+# HTTP server options.
 [http_options]
-addr = '127.0.0.1:4000'
+# Server address, "127.0.0.1:4000" by default.
+addr = "127.0.0.1:4000"
+# HTTP request timeout, 30s by default.
 timeout = "30s"

-[wal]
-dir = "/tmp/greptimedb/wal"
-file_size = '1GB'
-purge_interval = '10m'
-purge_threshold = '50GB'
-read_batch_size = 128
-sync_write = false
-
-
-[storage]
-type = 'File'
-data_dir = '/tmp/greptimedb/data/'
-
+# gRPC server options.
 [grpc_options]
-addr = '127.0.0.1:4001'
+# Server address, "127.0.0.1:4001" by default.
+addr = "127.0.0.1:4001"
+# The number of server worker threads, 8 by default.
 runtime_size = 8

+# MySQL server options.
 [mysql_options]
-addr = '127.0.0.1:4002'
+# Server address, "127.0.0.1:4002" by default.
+addr = "127.0.0.1:4002"
+# The number of server worker threads, 2 by default.
 runtime_size = 2

-[influxdb_options]
-enable = true
-
-[opentsdb_options]
-addr = '127.0.0.1:4242'
-enable = true
-runtime_size = 2
-
-[prometheus_options]
-enable = true
+# MySQL server TLS options.
+[mysql_options.tls]
+# TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
+# - "disable" (default value)
+# - "prefer"
+# - "require"
+# - "verify-ca"
+# - "verify-full"
+mode = "disable"
+# Certificate file path.
+cert_path = ""
+# Private key file path.
+key_path = ""

+# PostgresSQL server options.
 [postgres_options]
-addr = '127.0.0.1:4003'
+# Server address, "127.0.0.1:4003" by default.
+addr = "127.0.0.1:4003"
+# The number of server worker threads, 2 by default.
 runtime_size = 2
-check_pwd = false
+
+# PostgresSQL server TLS options, see `[mysql_options.tls]` section.
+[postgres_options.tls]
+# TLS mode.
+mode = "disable"
+# certificate file path.
+cert_path = ""
+# private key file path.
+key_path = ""
+
+# OpenTSDB protocol options.
+[opentsdb_options]
+# OpenTSDB telnet API server address, "127.0.0.1:4242" by default.
+addr = "127.0.0.1:4242"
+# The number of server worker threads, 2 by default.
+runtime_size = 2
+
+# InfluxDB protocol options.
+[influxdb_options]
+# Whether to enable InfluxDB protocol in HTTP API, true by default.
+enable = true
+
+# Prometheus protocol options.
+[prometheus_options]
+# Whether to enable Prometheus remote write and read in HTTP API, true by default.
+enable = true
+
+# Prom protocol options.
+[prom_options]
+# Prometheus API server address, "127.0.0.1:4004" by default.
+addr = "127.0.0.1:4004"
+
+# WAL options.
+[wal]
+# WAL data directory.
+dir = "/tmp/greptimedb/wal"
+# WAL file size in bytes.
+file_size = "1GB"
+# WAL purge threshold in bytes.
+purge_threshold = "50GB"
+# WAL purge interval in seconds.
+purge_interval = "10m"
+# WAL read batch size.
+read_batch_size = 128
+# Whether to sync log file after every write.
+sync_write = false
+
+# Storage options.
+[storage]
+# Storage type.
+type = "File"
+# Data directory, "/tmp/greptimedb/data" by default.
+data_dir = "/tmp/greptimedb/data/"
+
+# Compaction options.
+[compaction]
+# Max task number that can concurrently run.
+max_inflight_tasks = 4
+# Max files in level 0 to trigger compaction.
+max_files_in_level0 = 8
+# Max task number for SST purge task after compaction.
+max_purge_tasks = 32
+
+# Procedure storage options.
+# Uncomment to enable.
+# [procedure.store]
+# # Storage type.
+# type = "File"
+# # Procedure data path.
+# data_dir = "/tmp/greptimedb/procedure/"
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -9,7 +9,10 @@ RUN apt-get update && apt-get install -y \
    protobuf-compiler \
    curl \
    build-essential \
-    pkg-config
+    pkg-config \
+    python3 \
+    python3-dev \
+    && pip install pyarrow

 # Install Rust.
 SHELL ["/bin/bash", "-c"]
--- a/docs/rfcs/2022-12-20-promql-in-rust/rfc.md
+++ b/docs/rfcs/2022-12-20-promql-in-rust/rfc.md
@@ -149,10 +149,10 @@ inputs:
  - title: 'Series Normalize: \noffset = 0'
    operator: prom
    inputs:
-    - title: 'Filter: \ntimetamp > 2022-12-20T10:00:00 && timestamp < 2022-12-21T10:00:00'
+    - title: 'Filter: \ntimestamp > 2022-12-20T10:00:00 && timestamp < 2022-12-21T10:00:00'
      operator: filter
      inputs:
-      - title: 'Table Scan: \ntable = request_duration, timetamp > 2022-12-20T10:00:00 && timestamp < 2022-12-21T10:00:00'
+      - title: 'Table Scan: \ntable = request_duration, timestamp > 2022-12-20T10:00:00 && timestamp < 2022-12-21T10:00:00'
        operator: scan -->

 ![example](example.png)
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2022-12-20"
+channel = "nightly-2023-02-26"
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@@ -10,7 +10,7 @@ common-base = { path = "../common/base" }
 common-error = { path = "../common/error" }
 common-time = { path = "../common/time" }
 datatypes = { path = "../datatypes" }
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "3e6349be127b65a8b42a38cda9d527ec423ca77d" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "ad0187295035e83f76272da553453e649b7570de" }
 prost.workspace = true
 snafu = { version = "0.7", features = ["backtraces"] }
 tonic.workspace = true
--- a/src/api/src/helper.rs
+++ b/src/api/src/helper.rs
@@ -97,7 +97,9 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
                TimestampType::Microsecond(_) => ColumnDataType::TimestampMicrosecond,
                TimestampType::Nanosecond(_) => ColumnDataType::TimestampNanosecond,
            },
-            ConcreteDataType::Null(_) | ConcreteDataType::List(_) => {
+            ConcreteDataType::Null(_)
+            | ConcreteDataType::List(_)
+            | ConcreteDataType::Dictionary(_) => {
                return error::IntoColumnDataTypeSnafu { from: datatype }.fail()
            }
        });
@@ -194,7 +196,7 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
    null_mask.reserve_exact(origin_count + len);
    null_mask.extend(BitVec::repeat(false, len));

-    (0..len).into_iter().for_each(|idx| match vector.get(idx) {
+    (0..len).for_each(|idx| match vector.get(idx) {
        Value::Null => null_mask.set(idx + origin_count, true),
        Value::Boolean(val) => values.bool_values.push(val),
        Value::UInt8(val) => values.u8_values.push(val.into()),
--- a/src/catalog/Cargo.toml
+++ b/src/catalog/Cargo.toml
@@ -18,25 +18,28 @@ common-recordbatch = { path = "../common/recordbatch" }
 common-runtime = { path = "../common/runtime" }
 common-telemetry = { path = "../common/telemetry" }
 common-time = { path = "../common/time" }
+dashmap = "5.4"
 datafusion.workspace = true
 datatypes = { path = "../datatypes" }
 futures = "0.3"
 futures-util.workspace = true
 lazy_static = "1.4"
 meta-client = { path = "../meta-client" }
+parking_lot = "0.12"
 regex = "1.6"
 serde = "1.0"
 serde_json = "1.0"
+session = { path = "../session" }
 snafu = { version = "0.7", features = ["backtraces"] }
 storage = { path = "../storage" }
 table = { path = "../table" }
 tokio.workspace = true

 [dev-dependencies]
+common-test-util = { path = "../common/test-util" }
 chrono.workspace = true
 log-store = { path = "../log-store" }
 mito = { path = "../mito", features = ["test"] }
 object-store = { path = "../object-store" }
 storage = { path = "../storage" }
-tempdir = "0.3"
 tokio.workspace = true
--- a/src/catalog/src/error.rs
+++ b/src/catalog/src/error.rs
@@ -19,7 +19,6 @@ use common_error::ext::{BoxedError, ErrorExt};
 use common_error::prelude::{Snafu, StatusCode};
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
-use datatypes::schema::RawSchema;
 use snafu::{Backtrace, ErrorCompat};

 use crate::DeregisterTableRequest;
@@ -162,19 +161,6 @@ pub enum Error {
        source: table::error::Error,
    },

-    #[snafu(display(
-        "Invalid table schema in catalog entry, table:{}, schema: {:?}, source: {}",
-        table_info,
-        schema,
-        source
-    ))]
-    InvalidTableSchema {
-        table_info: String,
-        schema: RawSchema,
-        #[snafu(backtrace)]
-        source: datatypes::error::Error,
-    },
-
    #[snafu(display("Failure during SchemaProvider operation, source: {}", source))]
    SchemaProviderOperation {
        #[snafu(backtrace)]
@@ -215,6 +201,9 @@ pub enum Error {
        #[snafu(backtrace)]
        source: common_catalog::error::Error,
    },
+
+    #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
+    QueryAccessDenied { catalog: String, schema: String },
 }

 pub type Result<T> = std::result::Result<T, Error>;
@@ -254,13 +243,13 @@ impl ErrorExt for Error {
            Error::MetaSrv { source, .. } => source.status_code(),
            Error::SystemCatalogTableScan { source } => source.status_code(),
            Error::SystemCatalogTableScanExec { source } => source.status_code(),
-            Error::InvalidTableSchema { source, .. }
-            | Error::InvalidTableInfoInCatalog { source } => source.status_code(),
+            Error::InvalidTableInfoInCatalog { source } => source.status_code(),
            Error::SchemaProviderOperation { source } | Error::Internal { source } => {
                source.status_code()
            }

            Error::Unimplemented { .. } => StatusCode::Unsupported,
+            Error::QueryAccessDenied { .. } => StatusCode::AccessDenied,
        }
    }

--- a/src/catalog/src/helper.rs
+++ b/src/catalog/src/helper.rs
@@ -24,10 +24,10 @@ use serde::{Deserialize, Serialize, Serializer};
 use snafu::{ensure, OptionExt, ResultExt};
 use table::metadata::{RawTableInfo, TableId, TableVersion};

-const CATALOG_KEY_PREFIX: &str = "__c";
-const SCHEMA_KEY_PREFIX: &str = "__s";
-const TABLE_GLOBAL_KEY_PREFIX: &str = "__tg";
-const TABLE_REGIONAL_KEY_PREFIX: &str = "__tr";
+pub const CATALOG_KEY_PREFIX: &str = "__c";
+pub const SCHEMA_KEY_PREFIX: &str = "__s";
+pub const TABLE_GLOBAL_KEY_PREFIX: &str = "__tg";
+pub const TABLE_REGIONAL_KEY_PREFIX: &str = "__tr";

 const ALPHANUMERICS_NAME_PATTERN: &str = "[a-zA-Z_][a-zA-Z0-9_]*";

@@ -370,4 +370,10 @@ mod tests {
        let deserialized = TableGlobalValue::parse(serialized).unwrap();
        assert_eq!(value, deserialized);
    }
+
+    #[test]
+    fn test_table_global_value_compatibility() {
+        let s = r#"{"node_id":1,"regions_id_map":{"1":[0]},"table_info":{"ident":{"table_id":1098,"version":1},"name":"container_cpu_limit","desc":"Created on insertion","catalog_name":"greptime","schema_name":"dd","meta":{"schema":{"column_schemas":[{"name":"container_id","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"container_name","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"docker_image","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"host","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"image_name","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"image_tag","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"interval","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"runtime","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"short_image","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"type","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"dd_value","data_type":{"Float64":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":true,"default_constraint":null,"metadata":{"greptime:time_index":"true"}},{"name":"git.repository_url","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}}],"timestamp_index":11,"version":1},"primary_key_indices":[0,1,2,3,4,5,6,7,8,9,12],"value_indices":[10,11],"engine":"mito","next_column_id":12,"region_numbers":[],"engine_options":{},"options":{},"created_on":"1970-01-01T00:00:00Z"},"table_type":"Base"}}"#;
+        TableGlobalValue::parse(s).unwrap();
+    }
 }
--- a/src/catalog/src/lib.rs
+++ b/src/catalog/src/lib.rs
@@ -34,6 +34,7 @@ pub mod local;
 pub mod remote;
 pub mod schema;
 pub mod system;
+pub mod table_source;
 pub mod tables;

 /// Represent a list of named catalogs
@@ -107,7 +108,12 @@ pub trait CatalogManager: CatalogList {
    fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>>;

    /// Returns the table by catalog, schema and table name.
-    fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Result<Option<TableRef>>;
+    async fn table(
+        &self,
+        catalog: &str,
+        schema: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>>;
 }

 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
@@ -186,7 +192,8 @@ pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
        let table_name = &req.create_table_request.table_name;
        let table_id = req.create_table_request.id;

-        let table = if let Some(table) = manager.table(catalog_name, schema_name, table_name)? {
+        let table = manager.table(catalog_name, schema_name, table_name).await?;
+        let table = if let Some(table) = table {
            table
        } else {
            let table = engine
@@ -219,7 +226,7 @@ pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
 }

 /// The number of regions in the datanode node.
-pub fn region_number(catalog_manager: &CatalogManagerRef) -> Result<u64> {
+pub async fn region_number(catalog_manager: &CatalogManagerRef) -> Result<u64> {
    let mut region_number: u64 = 0;

    for catalog_name in catalog_manager.catalog_names()? {
@@ -239,11 +246,13 @@ pub fn region_number(catalog_manager: &CatalogManagerRef) -> Result<u64> {
                })?;

            for table_name in schema.table_names()? {
-                let table = schema
-                    .table(&table_name)?
-                    .context(error::TableNotFoundSnafu {
-                        table_info: &table_name,
-                    })?;
+                let table =
+                    schema
+                        .table(&table_name)
+                        .await?
+                        .context(error::TableNotFoundSnafu {
+                            table_info: &table_name,
+                        })?;

                let region_numbers = &table.table_info().meta.region_numbers;
                region_number += region_numbers.len() as u64;
--- a/src/catalog/src/local/manager.rs
+++ b/src/catalog/src/local/manager.rs
@@ -345,7 +345,7 @@ impl CatalogManager for LocalCatalogManager {

        {
            let _lock = self.register_lock.lock().await;
-            if let Some(existing) = schema.table(&request.table_name)? {
+            if let Some(existing) = schema.table(&request.table_name).await? {
                if existing.table_info().ident.table_id != request.table_id {
                    error!(
                        "Unexpected table register request: {:?}, existing: {:?}",
@@ -434,9 +434,10 @@ impl CatalogManager for LocalCatalogManager {
            } = &request;
            let table_id = self
                .catalogs
-                .table(catalog, schema, table_name)?
+                .table(catalog, schema, table_name)
+                .await?
                .with_context(|| error::TableNotExistSnafu {
-                    table: format!("{catalog}.{schema}.{table_name}"),
+                    table: format_full_table_name(catalog, schema, table_name),
                })?
                .table_info()
                .ident
@@ -505,7 +506,7 @@ impl CatalogManager for LocalCatalogManager {
            .schema(schema)
    }

-    fn table(
+    async fn table(
        &self,
        catalog_name: &str,
        schema_name: &str,
@@ -521,7 +522,7 @@ impl CatalogManager for LocalCatalogManager {
                catalog: catalog_name,
                schema: schema_name,
            })?;
-        schema.table(table_name)
+        schema.table(table_name).await
    }
 }

--- a/src/catalog/src/local/memory.rs
+++ b/src/catalog/src/local/memory.rs
@@ -18,6 +18,7 @@ use std::collections::HashMap;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::{Arc, RwLock};

+use async_trait::async_trait;
 use common_catalog::consts::MIN_USER_TABLE_ID;
 use common_telemetry::error;
 use snafu::{ensure, OptionExt};
@@ -155,16 +156,20 @@ impl CatalogManager for MemoryCatalogManager {
        }
    }

-    fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Result<Option<TableRef>> {
-        let c = self.catalogs.read().unwrap();
-        let catalog = if let Some(c) = c.get(catalog) {
+    async fn table(
+        &self,
+        catalog: &str,
+        schema: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        let catalog = {
+            let c = self.catalogs.read().unwrap();
+            let Some(c) = c.get(catalog) else { return Ok(None) };
            c.clone()
-        } else {
-            return Ok(None);
        };
        match catalog.schema(schema)? {
            None => Ok(None),
-            Some(s) => s.table(table_name),
+            Some(s) => s.table(table_name).await,
        }
    }
 }
@@ -283,6 +288,7 @@ impl Default for MemorySchemaProvider {
    }
 }

+#[async_trait]
 impl SchemaProvider for MemorySchemaProvider {
    fn as_any(&self) -> &dyn Any {
        self
@@ -293,7 +299,7 @@ impl SchemaProvider for MemorySchemaProvider {
        Ok(tables.keys().cloned().collect())
    }

-    fn table(&self, name: &str) -> Result<Option<TableRef>> {
+    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
        let tables = self.tables.read().unwrap();
        Ok(tables.get(name).cloned())
    }
@@ -355,8 +361,8 @@ mod tests {

    use super::*;

-    #[test]
-    fn test_new_memory_catalog_list() {
+    #[tokio::test]
+    async fn test_new_memory_catalog_list() {
        let catalog_list = new_memory_catalog_list().unwrap();
        let default_catalog = catalog_list.catalog(DEFAULT_CATALOG_NAME).unwrap().unwrap();

@@ -369,9 +375,9 @@ mod tests {
            .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
            .unwrap();

-        let table = default_schema.table("numbers").unwrap();
+        let table = default_schema.table("numbers").await.unwrap();
        assert!(table.is_some());
-        assert!(default_schema.table("not_exists").unwrap().is_none());
+        assert!(default_schema.table("not_exists").await.unwrap().is_none());
    }

    #[tokio::test]
@@ -419,7 +425,7 @@ mod tests {

        // test new table name exists
        assert!(provider.table_exist(new_table_name).unwrap());
-        let registered_table = provider.table(new_table_name).unwrap().unwrap();
+        let registered_table = provider.table(new_table_name).await.unwrap().unwrap();
        assert_eq!(
            registered_table.table_info().ident.table_id,
            test_table.table_info().ident.table_id
@@ -468,6 +474,7 @@ mod tests {

        let registered_table = catalog
            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
            .unwrap()
            .unwrap();
        assert_eq!(registered_table.table_info().ident.table_id, table_id);
--- a/src/catalog/src/remote/manager.rs
+++ b/src/catalog/src/remote/manager.rs
@@ -13,16 +13,19 @@
 // limitations under the License.

 use std::any::Any;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::pin::Pin;
 use std::sync::Arc;

 use arc_swap::ArcSwap;
 use async_stream::stream;
+use async_trait::async_trait;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID};
-use common_telemetry::{debug, info};
+use common_telemetry::{debug, error, info};
+use dashmap::DashMap;
 use futures::Stream;
 use futures_util::StreamExt;
+use parking_lot::RwLock;
 use snafu::{OptionExt, ResultExt};
 use table::engine::{EngineContext, TableEngineRef};
 use table::metadata::TableId;
@@ -32,12 +35,13 @@ use table::TableRef;
 use tokio::sync::Mutex;

 use crate::error::{
-    CatalogNotFoundSnafu, CreateTableSnafu, InvalidCatalogValueSnafu, InvalidTableSchemaSnafu,
-    OpenTableSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, UnimplementedSnafu,
+    CatalogNotFoundSnafu, CreateTableSnafu, InvalidCatalogValueSnafu, OpenTableSnafu, Result,
+    SchemaNotFoundSnafu, TableExistsSnafu, UnimplementedSnafu,
 };
 use crate::helper::{
    build_catalog_prefix, build_schema_prefix, build_table_global_prefix, CatalogKey, CatalogValue,
    SchemaKey, SchemaValue, TableGlobalKey, TableGlobalValue, TableRegionalKey, TableRegionalValue,
+    CATALOG_KEY_PREFIX,
 };
 use crate::remote::{Kv, KvBackendRef};
 use crate::{
@@ -50,10 +54,9 @@ use crate::{
 pub struct RemoteCatalogManager {
    node_id: u64,
    backend: KvBackendRef,
-    catalogs: Arc<ArcSwap<HashMap<String, CatalogProviderRef>>>,
+    catalogs: Arc<RwLock<DashMap<String, CatalogProviderRef>>>,
    engine: TableEngineRef,
    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
-    mutex: Arc<Mutex<()>>,
 }

 impl RemoteCatalogManager {
@@ -64,7 +67,6 @@ impl RemoteCatalogManager {
            backend,
            catalogs: Default::default(),
            system_table_requests: Default::default(),
-            mutex: Default::default(),
        }
    }

@@ -108,9 +110,13 @@ impl RemoteCatalogManager {
                    debug!("Ignoring non-catalog key: {}", String::from_utf8_lossy(&k));
                    continue;
                }
-                let key = CatalogKey::parse(&String::from_utf8_lossy(&k))
-                    .context(InvalidCatalogValueSnafu)?;
-                yield Ok(key)
+
+                let catalog_key = String::from_utf8_lossy(&k);
+                if let Ok(key) = CatalogKey::parse(&catalog_key) {
+                    yield Ok(key)
+                } else {
+                    error!("Invalid catalog key: {:?}", catalog_key);
+                }
            }
        }))
    }
@@ -346,21 +352,13 @@ impl RemoteCatalogManager {
                );

                let meta = &table_info.meta;
-                let schema = meta
-                    .schema
-                    .clone()
-                    .try_into()
-                    .context(InvalidTableSchemaSnafu {
-                        table_info: format!("{catalog_name}.{schema_name}.{table_name}"),
-                        schema: meta.schema.clone(),
-                    })?;
                let req = CreateTableRequest {
                    id: table_id,
                    catalog_name: catalog_name.clone(),
                    schema_name: schema_name.clone(),
                    table_name: table_name.clone(),
                    desc: None,
-                    schema: Arc::new(schema),
+                    schema: meta.schema.clone(),
                    region_numbers: region_numbers.clone(),
                    primary_key_indices: meta.primary_key_indices.clone(),
                    create_if_not_exists: true,
@@ -389,7 +387,14 @@ impl CatalogManager for RemoteCatalogManager {
            "Initialized catalogs: {:?}",
            catalogs.keys().cloned().collect::<Vec<_>>()
        );
-        self.catalogs.store(Arc::new(catalogs));
+
+        {
+            let self_catalogs = self.catalogs.read();
+            catalogs.into_iter().for_each(|(k, v)| {
+                self_catalogs.insert(k, v);
+            });
+        }
+
        info!("Max table id allocated: {}", max_table_id);

        let mut system_table_requests = self.system_table_requests.lock().await;
@@ -476,7 +481,7 @@ impl CatalogManager for RemoteCatalogManager {
            .schema(schema)
    }

-    fn table(
+    async fn table(
        &self,
        catalog_name: &str,
        schema_name: &str,
@@ -491,7 +496,7 @@ impl CatalogManager for RemoteCatalogManager {
                catalog: catalog_name,
                schema: schema_name,
            })?;
-        schema.table(table_name)
+        schema.table(table_name).await
    }
 }

@@ -507,12 +512,10 @@ impl CatalogList for RemoteCatalogManager {
    ) -> Result<Option<CatalogProviderRef>> {
        let key = self.build_catalog_key(&name).to_string();
        let backend = self.backend.clone();
-        let mutex = self.mutex.clone();
        let catalogs = self.catalogs.clone();

        std::thread::spawn(|| {
            common_runtime::block_on_write(async move {
-                let _guard = mutex.lock().await;
                backend
                    .set(
                        key.as_bytes(),
@@ -521,11 +524,10 @@ impl CatalogList for RemoteCatalogManager {
                            .context(InvalidCatalogValueSnafu)?,
                    )
                    .await?;
-                let prev_catalogs = catalogs.load();
-                let mut new_catalogs = HashMap::with_capacity(prev_catalogs.len() + 1);
-                new_catalogs.clone_from(&prev_catalogs);
-                let prev = new_catalogs.insert(name, catalog);
-                catalogs.store(Arc::new(new_catalogs));
+
+                let catalogs = catalogs.read();
+                let prev = catalogs.insert(name, catalog.clone());
+
                Ok(prev)
            })
        })
@@ -535,12 +537,65 @@ impl CatalogList for RemoteCatalogManager {

    /// List all catalogs from metasrv
    fn catalog_names(&self) -> Result<Vec<String>> {
-        Ok(self.catalogs.load().keys().cloned().collect::<Vec<_>>())
+        let catalogs = self.catalogs.read();
+        Ok(catalogs.iter().map(|k| k.key().to_string()).collect())
    }

    /// Read catalog info of given name from metasrv.
    fn catalog(&self, name: &str) -> Result<Option<CatalogProviderRef>> {
-        Ok(self.catalogs.load().get(name).cloned())
+        {
+            let catalogs = self.catalogs.read();
+            let catalog = catalogs.get(name);
+
+            if let Some(catalog) = catalog {
+                return Ok(Some(catalog.clone()));
+            }
+        }
+
+        let catalogs = self.catalogs.write();
+
+        let catalog = catalogs.get(name);
+        if let Some(catalog) = catalog {
+            return Ok(Some(catalog.clone()));
+        }
+
+        // It's for lack of incremental catalog syncing between datanode and meta. Here we fetch catalog
+        // from meta on demand. This can be removed when incremental catalog syncing is done in datanode.
+
+        let backend = self.backend.clone();
+
+        let catalogs_from_meta: HashSet<String> = std::thread::spawn(|| {
+            common_runtime::block_on_read(async move {
+                let mut stream = backend.range(CATALOG_KEY_PREFIX.as_bytes());
+                let mut catalogs = HashSet::new();
+
+                while let Some(catalog) = stream.next().await {
+                    if let Ok(catalog) = catalog {
+                        let catalog_key = String::from_utf8_lossy(&catalog.0);
+
+                        if let Ok(key) = CatalogKey::parse(&catalog_key) {
+                            catalogs.insert(key.catalog_name);
+                        }
+                    }
+                }
+
+                catalogs
+            })
+        })
+        .join()
+        .unwrap();
+
+        catalogs.retain(|catalog_name, _| catalogs_from_meta.get(catalog_name).is_some());
+
+        for catalog in catalogs_from_meta {
+            catalogs
+                .entry(catalog.clone())
+                .or_insert(self.new_catalog_provider(&catalog));
+        }
+
+        let catalog = catalogs.get(name);
+
+        Ok(catalog.as_deref().cloned())
    }
 }

@@ -700,6 +755,7 @@ impl RemoteSchemaProvider {
    }
 }

+#[async_trait]
 impl SchemaProvider for RemoteSchemaProvider {
    fn as_any(&self) -> &dyn Any {
        self
@@ -709,7 +765,7 @@ impl SchemaProvider for RemoteSchemaProvider {
        Ok(self.tables.load().keys().cloned().collect::<Vec<_>>())
    }

-    fn table(&self, name: &str) -> Result<Option<TableRef>> {
+    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
        Ok(self.tables.load().get(name).cloned())
    }

--- a/src/catalog/src/schema.rs
+++ b/src/catalog/src/schema.rs
@@ -15,11 +15,13 @@
 use std::any::Any;
 use std::sync::Arc;

+use async_trait::async_trait;
 use table::TableRef;

 use crate::error::Result;

 /// Represents a schema, comprising a number of named tables.
+#[async_trait]
 pub trait SchemaProvider: Sync + Send {
    /// Returns the schema provider as [`Any`](std::any::Any)
    /// so that it can be downcast to a specific implementation.
@@ -29,7 +31,7 @@ pub trait SchemaProvider: Sync + Send {
    fn table_names(&self) -> Result<Vec<String>>;

    /// Retrieves a specific table from the schema by name, provided it exists.
-    fn table(&self, name: &str) -> Result<Option<TableRef>>;
+    async fn table(&self, name: &str) -> Result<Option<TableRef>>;

    /// If supported by the implementation, adds a new table to this schema.
    /// If a table of the same name existed before, it returns "Table already exists" error.
--- a/src/catalog/src/system.rs
+++ b/src/catalog/src/system.rs
@@ -26,13 +26,15 @@ use common_recordbatch::SendableRecordBatchStream;
 use common_telemetry::debug;
 use common_time::util;
 use datatypes::prelude::{ConcreteDataType, ScalarVector, VectorRef};
-use datatypes::schema::{ColumnSchema, Schema, SchemaBuilder, SchemaRef};
+use datatypes::schema::{ColumnSchema, RawSchema, SchemaRef};
 use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
 use table::engine::{EngineContext, TableEngineRef};
 use table::metadata::{TableId, TableInfoRef};
-use table::requests::{CreateTableRequest, DeleteRequest, InsertRequest, OpenTableRequest};
+use table::requests::{
+    CreateTableRequest, DeleteRequest, InsertRequest, OpenTableRequest, TableOptions,
+};
 use table::{Table, TableRef};

 use crate::error::{
@@ -88,7 +90,7 @@ impl SystemCatalogTable {
            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
            table_id: SYSTEM_CATALOG_TABLE_ID,
        };
-        let schema = Arc::new(build_system_catalog_schema());
+        let schema = build_system_catalog_schema();
        let ctx = EngineContext::default();

        if let Some(table) = engine
@@ -105,11 +107,11 @@ impl SystemCatalogTable {
                schema_name: INFORMATION_SCHEMA_NAME.to_string(),
                table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
                desc: Some("System catalog table".to_string()),
-                schema: schema.clone(),
+                schema,
                region_numbers: vec![0],
                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX],
                create_if_not_exists: true,
-                table_options: HashMap::new(),
+                table_options: TableOptions::default(),
            };

            let table = engine
@@ -143,7 +145,7 @@ impl SystemCatalogTable {
 /// - value: JSON-encoded value of entry's metadata.
 /// - gmt_created: create time of this metadata.
 /// - gmt_modified: last updated time of this metadata.
-fn build_system_catalog_schema() -> Schema {
+fn build_system_catalog_schema() -> RawSchema {
    let cols = vec![
        ColumnSchema::new(
            "entry_type".to_string(),
@@ -178,8 +180,7 @@ fn build_system_catalog_schema() -> Schema {
        ),
    ];

-    // The schema of this table must be valid.
-    SchemaBuilder::try_from(cols).unwrap().build().unwrap()
+    RawSchema::new(cols)
 }

 /// Formats key string for table entry in system catalog
@@ -218,7 +219,7 @@ fn build_primary_key_columns(entry_type: EntryType, key: &[u8]) -> HashMap<Strin
    let mut m = HashMap::with_capacity(3);
    m.insert(
        "entry_type".to_string(),
-        Arc::new(UInt8Vector::from_slice(&[entry_type as u8])) as _,
+        Arc::new(UInt8Vector::from_slice([entry_type as u8])) as _,
    );
    m.insert(
        "key".to_string(),
@@ -227,7 +228,7 @@ fn build_primary_key_columns(entry_type: EntryType, key: &[u8]) -> HashMap<Strin
    // Timestamp in key part is intentionally left to 0
    m.insert(
        "timestamp".to_string(),
-        Arc::new(TimestampMillisecondVector::from_slice(&[0])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice([0])) as _,
    );
    m
 }
@@ -257,12 +258,12 @@ pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) ->
    let now = util::current_time_millis();
    columns_values.insert(
        "gmt_created".to_string(),
-        Arc::new(TimestampMillisecondVector::from_slice(&[now])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
    );

    columns_values.insert(
        "gmt_modified".to_string(),
-        Arc::new(TimestampMillisecondVector::from_slice(&[now])) as _,
+        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
    );

    InsertRequest {
@@ -394,16 +395,17 @@ pub struct TableEntryValue {
 #[cfg(test)]
 mod tests {
    use common_recordbatch::RecordBatches;
+    use common_test_util::temp_dir::{create_temp_dir, TempDir};
    use datatypes::value::Value;
    use log_store::NoopLogStore;
    use mito::config::EngineConfig;
    use mito::engine::MitoEngine;
-    use object_store::ObjectStore;
+    use object_store::{ObjectStore, ObjectStoreBuilder};
+    use storage::compaction::noop::NoopCompactionScheduler;
    use storage::config::EngineConfig as StorageEngineConfig;
    use storage::EngineImpl;
    use table::metadata::TableType;
    use table::metadata::TableType::Base;
-    use tempdir::TempDir;

    use super::*;

@@ -478,19 +480,21 @@ mod tests {
    }

    pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
-        let dir = TempDir::new("system-table-test").unwrap();
+        let dir = create_temp_dir("system-table-test");
        let store_dir = dir.path().to_string_lossy();
-        let accessor = object_store::backend::fs::Builder::default()
+        let accessor = object_store::services::Fs::default()
            .root(&store_dir)
            .build()
            .unwrap();
-        let object_store = ObjectStore::new(accessor);
+        let object_store = ObjectStore::new(accessor).finish();
+        let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
        let table_engine = Arc::new(MitoEngine::new(
            EngineConfig::default(),
            EngineImpl::new(
                StorageEngineConfig::default(),
                Arc::new(NoopLogStore::default()),
                object_store.clone(),
+                noop_compaction_scheduler,
            ),
            object_store,
        ));
--- a/src/catalog/src/table_source.rs
+++ b/src/catalog/src/table_source.rs
@@ -0,0 +1,178 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use common_catalog::format_full_table_name;
+use datafusion::common::{OwnedTableReference, ResolvedTableReference, TableReference};
+use datafusion::datasource::provider_as_source;
+use datafusion::logical_expr::TableSource;
+use session::context::QueryContext;
+use snafu::{ensure, OptionExt};
+use table::table::adapter::DfTableProviderAdapter;
+
+use crate::error::{
+    CatalogNotFoundSnafu, QueryAccessDeniedSnafu, Result, SchemaNotFoundSnafu, TableNotExistSnafu,
+};
+use crate::CatalogListRef;
+
+pub struct DfTableSourceProvider {
+    catalog_list: CatalogListRef,
+    resolved_tables: HashMap<String, Arc<dyn TableSource>>,
+    disallow_cross_schema_query: bool,
+    default_catalog: String,
+    default_schema: String,
+}
+
+impl DfTableSourceProvider {
+    pub fn new(
+        catalog_list: CatalogListRef,
+        disallow_cross_schema_query: bool,
+        query_ctx: &QueryContext,
+    ) -> Self {
+        Self {
+            catalog_list,
+            disallow_cross_schema_query,
+            resolved_tables: HashMap::new(),
+            default_catalog: query_ctx.current_catalog(),
+            default_schema: query_ctx.current_schema(),
+        }
+    }
+
+    pub fn resolve_table_ref<'a>(
+        &'a self,
+        table_ref: TableReference<'a>,
+    ) -> Result<ResolvedTableReference<'a>> {
+        if self.disallow_cross_schema_query {
+            match &table_ref {
+                TableReference::Bare { .. } => (),
+                TableReference::Partial { schema, .. } => {
+                    ensure!(
+                        schema.as_ref() == self.default_schema,
+                        QueryAccessDeniedSnafu {
+                            catalog: &self.default_catalog,
+                            schema: schema.as_ref(),
+                        }
+                    );
+                }
+                TableReference::Full {
+                    catalog, schema, ..
+                } => {
+                    ensure!(
+                        catalog.as_ref() == self.default_catalog
+                            && schema.as_ref() == self.default_schema,
+                        QueryAccessDeniedSnafu {
+                            catalog: catalog.as_ref(),
+                            schema: schema.as_ref()
+                        }
+                    );
+                }
+            };
+        }
+
+        Ok(table_ref.resolve(&self.default_catalog, &self.default_schema))
+    }
+
+    pub async fn resolve_table(
+        &mut self,
+        table_ref: OwnedTableReference,
+    ) -> Result<Arc<dyn TableSource>> {
+        let table_ref = table_ref.as_table_reference();
+        let table_ref = self.resolve_table_ref(table_ref)?;
+
+        let resolved_name = table_ref.to_string();
+        if let Some(table) = self.resolved_tables.get(&resolved_name) {
+            return Ok(table.clone());
+        }
+
+        let catalog_name = table_ref.catalog.as_ref();
+        let schema_name = table_ref.schema.as_ref();
+        let table_name = table_ref.table.as_ref();
+
+        let catalog = self
+            .catalog_list
+            .catalog(catalog_name)?
+            .context(CatalogNotFoundSnafu { catalog_name })?;
+        let schema = catalog.schema(schema_name)?.context(SchemaNotFoundSnafu {
+            catalog: catalog_name,
+            schema: schema_name,
+        })?;
+        let table = schema
+            .table(table_name)
+            .await?
+            .with_context(|| TableNotExistSnafu {
+                table: format_full_table_name(catalog_name, schema_name, table_name),
+            })?;
+
+        let table = DfTableProviderAdapter::new(table);
+        let table = provider_as_source(Arc::new(table));
+        self.resolved_tables.insert(resolved_name, table.clone());
+        Ok(table)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+
+    use session::context::QueryContext;
+
+    use super::*;
+    use crate::local::MemoryCatalogManager;
+
+    #[test]
+    fn test_validate_table_ref() {
+        let query_ctx = &QueryContext::with("greptime", "public");
+
+        let table_provider =
+            DfTableSourceProvider::new(Arc::new(MemoryCatalogManager::default()), true, query_ctx);
+
+        let table_ref = TableReference::Bare {
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_ok());
+
+        let table_ref = TableReference::Partial {
+            schema: Cow::Borrowed("public"),
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_ok());
+
+        let table_ref = TableReference::Partial {
+            schema: Cow::Borrowed("wrong_schema"),
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_err());
+
+        let table_ref = TableReference::Full {
+            catalog: Cow::Borrowed("greptime"),
+            schema: Cow::Borrowed("public"),
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_ok());
+
+        let table_ref = TableReference::Full {
+            catalog: Cow::Borrowed("wrong_catalog"),
+            schema: Cow::Borrowed("public"),
+            table: Cow::Borrowed("table_name"),
+        };
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_err());
+    }
+}
--- a/src/catalog/src/tables.rs
+++ b/src/catalog/src/tables.rs
@@ -20,6 +20,7 @@ use std::sync::Arc;
 use std::task::{Context, Poll};

 use async_stream::stream;
+use async_trait::async_trait;
 use common_catalog::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_TABLE_NAME};
 use common_error::ext::BoxedError;
 use common_query::logical_plan::Expr;
@@ -162,16 +163,10 @@ fn tables_to_record_batch(

    for table_name in table_names {
        // Safety: All these vectors are string type.
-        catalog_vec
-            .push_value_ref(ValueRef::String(catalog_name))
-            .unwrap();
-        schema_vec
-            .push_value_ref(ValueRef::String(schema_name))
-            .unwrap();
-        table_name_vec
-            .push_value_ref(ValueRef::String(&table_name))
-            .unwrap();
-        engine_vec.push_value_ref(ValueRef::String(engine)).unwrap();
+        catalog_vec.push_value_ref(ValueRef::String(catalog_name));
+        schema_vec.push_value_ref(ValueRef::String(schema_name));
+        table_name_vec.push_value_ref(ValueRef::String(&table_name));
+        engine_vec.push_value_ref(ValueRef::String(engine));
    }

    vec![
@@ -206,6 +201,7 @@ pub struct InformationSchema {
    pub system: Arc<SystemCatalogTable>,
 }

+#[async_trait]
 impl SchemaProvider for InformationSchema {
    fn as_any(&self) -> &dyn Any {
        self
@@ -218,7 +214,7 @@ impl SchemaProvider for InformationSchema {
        ])
    }

-    fn table(&self, name: &str) -> Result<Option<TableRef>, Error> {
+    async fn table(&self, name: &str) -> Result<Option<TableRef>, Error> {
        if name.eq_ignore_ascii_case("tables") {
            Ok(Some(self.tables.clone()))
        } else if name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME) {
--- a/src/catalog/tests/local_catalog_tests.rs
+++ b/src/catalog/tests/local_catalog_tests.rs
@@ -71,6 +71,7 @@ mod tests {

        let registered_table = catalog_manager
            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
            .unwrap()
            .unwrap();
        assert_eq!(registered_table.table_info().ident.table_id, table_id);
@@ -158,6 +159,7 @@ mod tests {
            let table = guard.as_ref().unwrap();
            let table_registered = catalog_manager
                .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "test_table")
+                .await
                .unwrap()
                .unwrap();
            assert_eq!(
--- a/src/catalog/tests/mock.rs
+++ b/src/catalog/tests/mock.rs
@@ -147,6 +147,7 @@ impl TableEngine for MockTableEngine {
        let table_id = TableId::from_str(
            request
                .table_options
+                .extra_options
                .get("table_id")
                .unwrap_or(&default_table_id),
        )
--- a/src/catalog/tests/remote_catalog_tests.rs
+++ b/src/catalog/tests/remote_catalog_tests.rs
@@ -28,7 +28,7 @@ mod tests {
    };
    use catalog::{CatalogList, CatalogManager, RegisterTableRequest};
    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-    use datatypes::schema::Schema;
+    use datatypes::schema::RawSchema;
    use futures_util::StreamExt;
    use table::engine::{EngineContext, TableEngineRef};
    use table::requests::CreateTableRequest;
@@ -116,7 +116,7 @@ mod tests {
        let schema_name = "nonexistent_schema".to_string();
        let table_name = "fail_table".to_string();
        // this schema has no effect
-        let table_schema = Arc::new(Schema::new(vec![]));
+        let table_schema = RawSchema::new(vec![]);
        let table = table_engine
            .create_table(
                &EngineContext {},
@@ -126,7 +126,7 @@ mod tests {
                    schema_name: schema_name.clone(),
                    table_name: table_name.clone(),
                    desc: None,
-                    schema: table_schema.clone(),
+                    schema: table_schema,
                    region_numbers: vec![0],
                    primary_key_indices: vec![],
                    create_if_not_exists: false,
@@ -176,7 +176,7 @@ mod tests {
        let table_name = "test_table".to_string();
        let table_id = 1;
        // this schema has no effect
-        let table_schema = Arc::new(Schema::new(vec![]));
+        let table_schema = RawSchema::new(vec![]);
        let table = table_engine
            .create_table(
                &EngineContext {},
@@ -186,7 +186,7 @@ mod tests {
                    schema_name: schema_name.clone(),
                    table_name: table_name.clone(),
                    desc: None,
-                    schema: table_schema.clone(),
+                    schema: table_schema,
                    region_numbers: vec![0],
                    primary_key_indices: vec![],
                    create_if_not_exists: false,
@@ -246,7 +246,7 @@ mod tests {
                    schema_name: schema_name.clone(),
                    table_name: "".to_string(),
                    desc: None,
-                    schema: Arc::new(Schema::new(vec![])),
+                    schema: RawSchema::new(vec![]),
                    region_numbers: vec![0],
                    primary_key_indices: vec![],
                    create_if_not_exists: false,
--- a/src/client/Cargo.toml
+++ b/src/client/Cargo.toml
@@ -32,12 +32,8 @@ substrait = { path = "../common/substrait" }
 tokio.workspace = true
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-
-# TODO(ruihang): upgrade to 0.11 once substrait-rs supports it.
-[dev-dependencies.prost_09]
-package = "prost"
-version = "0.9"
+prost.workspace = true

 [dev-dependencies.substrait_proto]
 package = "substrait"
-version = "0.2"
+version = "0.4"
--- a/src/client/examples/logical.rs
+++ b/src/client/examples/logical.rs
@@ -14,11 +14,12 @@

 use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, TableId};
 use client::{Client, Database};
-use prost_09::Message;
-use substrait_proto::protobuf::plan_rel::RelType as PlanRelType;
-use substrait_proto::protobuf::read_rel::{NamedTable, ReadType};
-use substrait_proto::protobuf::rel::RelType;
-use substrait_proto::protobuf::{PlanRel, ReadRel, Rel};
+use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use prost::Message;
+use substrait_proto::proto::plan_rel::RelType as PlanRelType;
+use substrait_proto::proto::read_rel::{NamedTable, ReadType};
+use substrait_proto::proto::rel::RelType;
+use substrait_proto::proto::{PlanRel, ReadRel, Rel};
 use tracing::{event, Level};

 fn main() {
@@ -65,7 +66,7 @@ async fn run() {
        region_ids: vec![0],
    };

-    let db = Database::with_client(client);
+    let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
    let result = db.create(create_table_expr).await.unwrap();
    event!(Level::INFO, "create table result: {:#?}", result);

@@ -88,12 +89,8 @@ fn mock_logical_plan() -> Vec<u8> {
    let read_type = ReadType::NamedTable(named_table);

    let read_rel = ReadRel {
-        common: None,
-        base_schema: None,
-        filter: None,
-        projection: None,
-        advanced_extension: None,
        read_type: Some(read_type),
+        ..Default::default()
    };

    let mut buf = vec![];
--- a/src/client/src/database.rs
+++ b/src/client/src/database.rs
@@ -14,15 +14,15 @@

 use std::str::FromStr;

+use api::v1::auth_header::AuthScheme;
 use api::v1::ddl_request::Expr as DdlExpr;
 use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
-    AlterExpr, CreateTableExpr, DdlRequest, DropTableExpr, GreptimeRequest, InsertRequest,
-    QueryRequest, RequestHeader,
+    AlterExpr, AuthHeader, CreateTableExpr, DdlRequest, DropTableExpr, GreptimeRequest,
+    InsertRequest, QueryRequest, RequestHeader,
 };
 use arrow_flight::{FlightData, Ticket};
-use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_error::prelude::*;
 use common_grpc::flight::{flight_messages_to_recordbatches, FlightDecoder, FlightMessage};
 use common_query::Output;
@@ -42,6 +42,7 @@ pub struct Database {
    schema: String,

    client: Client,
+    ctx: FlightContext,
 }

 impl Database {
@@ -50,17 +51,32 @@ impl Database {
            catalog: catalog.into(),
            schema: schema.into(),
            client,
+            ctx: FlightContext::default(),
        }
    }

-    pub fn with_client(client: Client) -> Self {
-        Self::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client)
+    pub fn catalog(&self) -> &String {
+        &self.catalog
+    }
+
+    pub fn set_catalog(&mut self, catalog: impl Into<String>) {
+        self.catalog = catalog.into();
+    }
+
+    pub fn schema(&self) -> &String {
+        &self.schema
    }

    pub fn set_schema(&mut self, schema: impl Into<String>) {
        self.schema = schema.into();
    }

+    pub fn set_auth(&mut self, auth: AuthScheme) {
+        self.ctx.auth_header = Some(AuthHeader {
+            auth_scheme: Some(auth),
+        });
+    }
+
    pub async fn insert(&self, request: InsertRequest) -> Result<Output> {
        self.do_get(Request::Insert(request)).await
    }
@@ -105,11 +121,12 @@ impl Database {
            header: Some(RequestHeader {
                catalog: self.catalog.clone(),
                schema: self.schema.clone(),
+                authorization: self.ctx.auth_header.clone(),
            }),
            request: Some(request),
        };
        let request = Ticket {
-            ticket: request.encode_to_vec(),
+            ticket: request.encode_to_vec().into(),
        };

        let mut client = self.client.make_client()?;
@@ -164,12 +181,18 @@ fn get_metadata_value(e: &tonic::Status, key: &str) -> Option<String> {
        .and_then(|v| String::from_utf8(v.as_bytes().to_vec()).ok())
 }

+#[derive(Default, Debug, Clone)]
+pub struct FlightContext {
+    auth_header: Option<AuthHeader>,
+}
+
 #[cfg(test)]
 mod tests {
    use std::sync::Arc;

    use api::helper::ColumnDataTypeWrapper;
-    use api::v1::Column;
+    use api::v1::auth_header::AuthScheme;
+    use api::v1::{AuthHeader, Basic, Column};
    use common_grpc::select::{null_mask, values};
    use common_grpc_expr::column_to_vector;
    use datatypes::prelude::{Vector, VectorRef};
@@ -179,6 +202,8 @@ mod tests {
        UInt32Vector, UInt64Vector, UInt8Vector,
    };

+    use crate::database::FlightContext;
+
    #[test]
    fn test_column_to_vector() {
        let mut column = create_test_column(Arc::new(BooleanVector::from(vec![true])));
@@ -262,4 +287,26 @@ mod tests {
            datatype: wrapper.datatype() as i32,
        }
    }
+
+    #[test]
+    fn test_flight_ctx() {
+        let mut ctx = FlightContext::default();
+        assert!(ctx.auth_header.is_none());
+
+        let basic = AuthScheme::Basic(Basic {
+            username: "u".to_string(),
+            password: "p".to_string(),
+        });
+
+        ctx.auth_header = Some(AuthHeader {
+            auth_scheme: Some(basic),
+        });
+
+        assert!(matches!(
+            ctx.auth_header,
+            Some(AuthHeader {
+                auth_scheme: Some(AuthScheme::Basic(_)),
+            })
+        ))
+    }
 }
--- a/src/client/src/lib.rs
+++ b/src/client/src/lib.rs
@@ -18,6 +18,7 @@ mod error;
 pub mod load_balance;

 pub use api;
+pub use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};

 pub use self::client::Client;
 pub use self::database::Database;
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -9,28 +9,46 @@ default-run = "greptime"
 name = "greptime"
 path = "src/bin/greptime.rs"

+[features]
+mem-prof = ["tikv-jemallocator", "tikv-jemalloc-ctl"]
+
 [dependencies]
 anymap = "1.0.0-beta.2"
+catalog = { path = "../catalog" }
 clap = { version = "3.1", features = ["derive"] }
+client = { path = "../client" }
 common-base = { path = "../common/base" }
 common-error = { path = "../common/error" }
+common-query = { path = "../common/query" }
+common-recordbatch = { path = "../common/recordbatch" }
 common-telemetry = { path = "../common/telemetry", features = [
    "deadlock_detection",
 ] }
 datanode = { path = "../datanode" }
+either = "1.8"
 frontend = { path = "../frontend" }
 futures.workspace = true
 meta-client = { path = "../meta-client" }
 meta-srv = { path = "../meta-srv" }
+nu-ansi-term = "0.46"
+partition = { path = "../partition" }
+query = { path = "../query" }
+rustyline = "10.1"
 serde.workspace = true
 servers = { path = "../servers" }
+session = { path = "../session" }
 snafu.workspace = true
+substrait = { path = "../common/substrait" }
+tikv-jemalloc-ctl = { version = "0.5", optional = true }
+tikv-jemallocator = { version = "0.5", optional = true }
 tokio.workspace = true
 toml = "0.5"

+
 [dev-dependencies]
+common-test-util = { path = "../common/test-util" }
+rexpect = "0.5"
 serde.workspace = true
-tempdir = "0.3"

 [build-dependencies]
 build-data = "0.1.3"
--- a/src/cmd/src/bin/greptime.rs
+++ b/src/cmd/src/bin/greptime.rs
@@ -16,7 +16,7 @@ use std::fmt;

 use clap::Parser;
 use cmd::error::Result;
-use cmd::{datanode, frontend, metasrv, standalone};
+use cmd::{cli, datanode, frontend, metasrv, standalone};
 use common_telemetry::logging::{error, info};

 #[derive(Parser)]
@@ -46,6 +46,8 @@ enum SubCommand {
    Metasrv(metasrv::Command),
    #[clap(name = "standalone")]
    Standalone(standalone::Command),
+    #[clap(name = "cli")]
+    Cli(cli::Command),
 }

 impl SubCommand {
@@ -55,6 +57,7 @@ impl SubCommand {
            SubCommand::Frontend(cmd) => cmd.run().await,
            SubCommand::Metasrv(cmd) => cmd.run().await,
            SubCommand::Standalone(cmd) => cmd.run().await,
+            SubCommand::Cli(cmd) => cmd.run().await,
        }
    }
 }
@@ -66,6 +69,7 @@ impl fmt::Display for SubCommand {
            SubCommand::Frontend(..) => write!(f, "greptime-frontend"),
            SubCommand::Metasrv(..) => write!(f, "greptime-metasrv"),
            SubCommand::Standalone(..) => write!(f, "greptime-standalone"),
+            SubCommand::Cli(_) => write!(f, "greptime-cli"),
        }
    }
 }
@@ -83,6 +87,10 @@ fn print_version() -> &'static str {
    )
 }

+#[cfg(feature = "mem-prof")]
+#[global_allocator]
+static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
 #[tokio::main]
 async fn main() -> Result<()> {
    let cmd = Command::parse();
--- a/src/cmd/src/cli.rs
+++ b/src/cmd/src/cli.rs
@@ -0,0 +1,64 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod cmd;
+mod helper;
+mod repl;
+
+use clap::Parser;
+use repl::Repl;
+
+use crate::error::Result;
+
+#[derive(Parser)]
+pub struct Command {
+    #[clap(subcommand)]
+    cmd: SubCommand,
+}
+
+impl Command {
+    pub async fn run(self) -> Result<()> {
+        self.cmd.run().await
+    }
+}
+
+#[derive(Parser)]
+enum SubCommand {
+    Attach(AttachCommand),
+}
+
+impl SubCommand {
+    async fn run(self) -> Result<()> {
+        match self {
+            SubCommand::Attach(cmd) => cmd.run().await,
+        }
+    }
+}
+
+#[derive(Debug, Parser)]
+pub(crate) struct AttachCommand {
+    #[clap(long)]
+    pub(crate) grpc_addr: String,
+    #[clap(long)]
+    pub(crate) meta_addr: Option<String>,
+    #[clap(long, action)]
+    pub(crate) disable_helper: bool,
+}
+
+impl AttachCommand {
+    async fn run(self) -> Result<()> {
+        let mut repl = Repl::try_new(&self).await?;
+        repl.run().await
+    }
+}
--- a/src/cmd/src/cli/cmd.rs
+++ b/src/cmd/src/cli/cmd.rs
@@ -0,0 +1,154 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, InvalidReplCommandSnafu, Result};
+
+/// Represents the parsed command from the user (which may be over many lines)
+#[derive(Debug, PartialEq)]
+pub(crate) enum ReplCommand {
+    Help,
+    UseDatabase { db_name: String },
+    Sql { sql: String },
+    Exit,
+}
+
+impl TryFrom<&str> for ReplCommand {
+    type Error = Error;
+
+    fn try_from(input: &str) -> Result<Self> {
+        let input = input.trim();
+        if input.is_empty() {
+            return InvalidReplCommandSnafu {
+                reason: "No command specified".to_string(),
+            }
+            .fail();
+        }
+
+        // If line ends with ';', it must be treated as a complete input.
+        // However, the opposite is not true.
+        let input_is_completed = input.ends_with(';');
+
+        let input = input.strip_suffix(';').map(|x| x.trim()).unwrap_or(input);
+        let lowercase = input.to_lowercase();
+        match lowercase.as_str() {
+            "help" => Ok(Self::Help),
+            "exit" | "quit" => Ok(Self::Exit),
+            _ => match input.split_once(' ') {
+                Some((maybe_use, database)) if maybe_use.to_lowercase() == "use" => {
+                    Ok(Self::UseDatabase {
+                        db_name: database.trim().to_string(),
+                    })
+                }
+                // Any valid SQL must contains at least one whitespace.
+                Some(_) if input_is_completed => Ok(Self::Sql {
+                    sql: input.to_string(),
+                }),
+                _ => InvalidReplCommandSnafu {
+                    reason: format!("unknown command '{input}', maybe input is not completed"),
+                }
+                .fail(),
+            },
+        }
+    }
+}
+
+impl ReplCommand {
+    pub fn help() -> &'static str {
+        r#"
+Available commands (case insensitive):
+- 'help': print this help
+- 'exit' or 'quit': exit the REPL
+- 'use <your database name>': switch to another database/schema context
+- Other typed in text will be treated as SQL.
+  You can enter new line while typing, just remember to end it with ';'.
+"#
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::error::Error::InvalidReplCommand;
+
+    #[test]
+    fn test_from_str() {
+        fn test_ok(s: &str, expected: ReplCommand) {
+            let actual: ReplCommand = s.try_into().unwrap();
+            assert_eq!(expected, actual, "'{}'", s);
+        }
+
+        fn test_err(s: &str) {
+            let result: Result<ReplCommand> = s.try_into();
+            assert!(matches!(result, Err(InvalidReplCommand { .. })))
+        }
+
+        test_err("");
+        test_err("  ");
+        test_err("\t");
+
+        test_ok("help", ReplCommand::Help);
+        test_ok("help", ReplCommand::Help);
+        test_ok("  help", ReplCommand::Help);
+        test_ok("  help  ", ReplCommand::Help);
+        test_ok("  HELP  ", ReplCommand::Help);
+        test_ok("  Help;  ", ReplCommand::Help);
+        test_ok("  help  ; ", ReplCommand::Help);
+
+        test_ok("exit", ReplCommand::Exit);
+        test_ok("exit;", ReplCommand::Exit);
+        test_ok("exit ;", ReplCommand::Exit);
+        test_ok("EXIT", ReplCommand::Exit);
+
+        test_ok("quit", ReplCommand::Exit);
+        test_ok("quit;", ReplCommand::Exit);
+        test_ok("quit ;", ReplCommand::Exit);
+        test_ok("QUIT", ReplCommand::Exit);
+
+        test_ok(
+            "use Foo",
+            ReplCommand::UseDatabase {
+                db_name: "Foo".to_string(),
+            },
+        );
+        test_ok(
+            "  use Foo ;  ",
+            ReplCommand::UseDatabase {
+                db_name: "Foo".to_string(),
+            },
+        );
+        // ensure that database name is case sensitive
+        test_ok(
+            "  use FOO ;  ",
+            ReplCommand::UseDatabase {
+                db_name: "FOO".to_string(),
+            },
+        );
+
+        // ensure that we aren't messing with capitalization
+        test_ok(
+            "SELECT * from foo;",
+            ReplCommand::Sql {
+                sql: "SELECT * from foo".to_string(),
+            },
+        );
+        // Input line (that don't belong to any other cases above) must ends with ';' to make it a valid SQL.
+        test_err("insert blah");
+        test_ok(
+            "insert blah;",
+            ReplCommand::Sql {
+                sql: "insert blah".to_string(),
+            },
+        );
+    }
+}
--- a/src/cmd/src/cli/helper.rs
+++ b/src/cmd/src/cli/helper.rs
@@ -0,0 +1,112 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+
+use rustyline::completion::Completer;
+use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
+use rustyline::hint::{Hinter, HistoryHinter};
+use rustyline::validate::{ValidationContext, ValidationResult, Validator};
+
+use crate::cli::cmd::ReplCommand;
+
+pub(crate) struct RustylineHelper {
+    hinter: HistoryHinter,
+    highlighter: MatchingBracketHighlighter,
+}
+
+impl Default for RustylineHelper {
+    fn default() -> Self {
+        Self {
+            hinter: HistoryHinter {},
+            highlighter: MatchingBracketHighlighter::default(),
+        }
+    }
+}
+
+impl rustyline::Helper for RustylineHelper {}
+
+impl Validator for RustylineHelper {
+    fn validate(&self, ctx: &mut ValidationContext<'_>) -> rustyline::Result<ValidationResult> {
+        let input = ctx.input();
+        match ReplCommand::try_from(input) {
+            Ok(_) => Ok(ValidationResult::Valid(None)),
+            Err(e) => {
+                if input.trim_end().ends_with(';') {
+                    // If line ends with ';', it HAS to be a valid command.
+                    Ok(ValidationResult::Invalid(Some(e.to_string())))
+                } else {
+                    Ok(ValidationResult::Incomplete)
+                }
+            }
+        }
+    }
+}
+
+impl Hinter for RustylineHelper {
+    type Hint = String;
+
+    fn hint(&self, line: &str, pos: usize, ctx: &rustyline::Context<'_>) -> Option<Self::Hint> {
+        self.hinter.hint(line, pos, ctx)
+    }
+}
+
+impl Highlighter for RustylineHelper {
+    fn highlight<'l>(&self, line: &'l str, pos: usize) -> Cow<'l, str> {
+        self.highlighter.highlight(line, pos)
+    }
+
+    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(
+        &'s self,
+        prompt: &'p str,
+        default: bool,
+    ) -> Cow<'b, str> {
+        self.highlighter.highlight_prompt(prompt, default)
+    }
+
+    fn highlight_hint<'h>(&self, hint: &'h str) -> Cow<'h, str> {
+        use nu_ansi_term::Style;
+        Cow::Owned(Style::new().dimmed().paint(hint).to_string())
+    }
+
+    fn highlight_candidate<'c>(
+        &self,
+        candidate: &'c str,
+        completion: rustyline::CompletionType,
+    ) -> Cow<'c, str> {
+        self.highlighter.highlight_candidate(candidate, completion)
+    }
+
+    fn highlight_char(&self, line: &str, pos: usize) -> bool {
+        self.highlighter.highlight_char(line, pos)
+    }
+}
+
+impl Completer for RustylineHelper {
+    type Candidate = String;
+
+    fn complete(
+        &self,
+        line: &str,
+        pos: usize,
+        ctx: &rustyline::Context<'_>,
+    ) -> rustyline::Result<(usize, Vec<Self::Candidate>)> {
+        // If there is a hint, use that as the auto-complete when user hits `tab`
+        if let Some(hint) = self.hinter.hint(line, pos, ctx) {
+            Ok((pos, vec![hint]))
+        } else {
+            Ok((0, vec![]))
+        }
+    }
+}
--- a/src/cmd/src/cli/repl.rs
+++ b/src/cmd/src/cli/repl.rs
@@ -0,0 +1,267 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Instant;
+
+use catalog::remote::MetaKvBackend;
+use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_error::prelude::ErrorExt;
+use common_query::Output;
+use common_recordbatch::RecordBatches;
+use common_telemetry::logging;
+use either::Either;
+use frontend::catalog::FrontendCatalogManager;
+use frontend::datanode::DatanodeClients;
+use meta_client::client::MetaClientBuilder;
+use partition::manager::PartitionRuleManager;
+use partition::route::TableRoutes;
+use query::datafusion::DatafusionQueryEngine;
+use query::logical_optimizer::LogicalOptimizer;
+use query::parser::QueryLanguageParser;
+use query::plan::LogicalPlan;
+use query::QueryEngine;
+use rustyline::error::ReadlineError;
+use rustyline::Editor;
+use session::context::QueryContext;
+use snafu::{ErrorCompat, ResultExt};
+use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
+
+use crate::cli::cmd::ReplCommand;
+use crate::cli::helper::RustylineHelper;
+use crate::cli::AttachCommand;
+use crate::error::{
+    CollectRecordBatchesSnafu, ParseSqlSnafu, PlanStatementSnafu, PrettyPrintRecordBatchesSnafu,
+    ReadlineSnafu, ReplCreationSnafu, RequestDatabaseSnafu, Result, StartMetaClientSnafu,
+    SubstraitEncodeLogicalPlanSnafu,
+};
+
+/// Captures the state of the repl, gathers commands and executes them one by one
+pub(crate) struct Repl {
+    /// Rustyline editor for interacting with user on command line
+    rl: Editor<RustylineHelper>,
+
+    /// Current prompt
+    prompt: String,
+
+    /// Client for interacting with GreptimeDB
+    database: Database,
+
+    query_engine: Option<DatafusionQueryEngine>,
+}
+
+#[allow(clippy::print_stdout)]
+impl Repl {
+    fn print_help(&self) {
+        println!("{}", ReplCommand::help())
+    }
+
+    pub(crate) async fn try_new(cmd: &AttachCommand) -> Result<Self> {
+        let mut rl = Editor::new().context(ReplCreationSnafu)?;
+
+        if !cmd.disable_helper {
+            rl.set_helper(Some(RustylineHelper::default()));
+
+            let history_file = history_file();
+            if let Err(e) = rl.load_history(&history_file) {
+                logging::debug!(
+                    "failed to load history file on {}, error: {e}",
+                    history_file.display()
+                );
+            }
+        }
+
+        let client = Client::with_urls([&cmd.grpc_addr]);
+        let database = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+
+        let query_engine = if let Some(meta_addr) = &cmd.meta_addr {
+            create_query_engine(meta_addr).await.map(Some)?
+        } else {
+            None
+        };
+
+        Ok(Self {
+            rl,
+            prompt: "> ".to_string(),
+            database,
+            query_engine,
+        })
+    }
+
+    /// Parse the next command
+    fn next_command(&mut self) -> Result<ReplCommand> {
+        match self.rl.readline(&self.prompt) {
+            Ok(ref line) => {
+                let request = line.trim();
+
+                self.rl.add_history_entry(request.to_string());
+
+                request.try_into()
+            }
+            Err(ReadlineError::Eof) | Err(ReadlineError::Interrupted) => Ok(ReplCommand::Exit),
+            // Some sort of real underlying error
+            Err(e) => Err(e).context(ReadlineSnafu),
+        }
+    }
+
+    /// Read Evaluate Print Loop (interactive command line) for GreptimeDB
+    ///
+    /// Inspired / based on repl.rs from InfluxDB IOX
+    pub(crate) async fn run(&mut self) -> Result<()> {
+        println!("Ready for commands. (Hint: try 'help')");
+
+        loop {
+            match self.next_command()? {
+                ReplCommand::Help => {
+                    self.print_help();
+                }
+                ReplCommand::UseDatabase { db_name } => {
+                    if self.execute_sql(format!("USE {db_name}")).await {
+                        println!("Using {db_name}");
+                        self.database.set_schema(&db_name);
+                        self.prompt = format!("[{db_name}] > ");
+                    }
+                }
+                ReplCommand::Sql { sql } => {
+                    self.execute_sql(sql).await;
+                }
+                ReplCommand::Exit => {
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    async fn execute_sql(&self, sql: String) -> bool {
+        self.do_execute_sql(sql)
+            .await
+            .map_err(|e| {
+                let status_code = e.status_code();
+                let root_cause = e.iter_chain().last().unwrap();
+                println!("Error: {}({status_code}), {root_cause}", status_code as u32)
+            })
+            .is_ok()
+    }
+
+    async fn do_execute_sql(&self, sql: String) -> Result<()> {
+        let start = Instant::now();
+
+        let output = if let Some(query_engine) = &self.query_engine {
+            let stmt = QueryLanguageParser::parse_sql(&sql)
+                .with_context(|_| ParseSqlSnafu { sql: sql.clone() })?;
+
+            let query_ctx = Arc::new(QueryContext::with(
+                self.database.catalog(),
+                self.database.schema(),
+            ));
+            let LogicalPlan::DfPlan(plan) = query_engine
+                .statement_to_plan(stmt, query_ctx)
+                .await
+                .and_then(|x| query_engine.optimize(&x))
+                .context(PlanStatementSnafu)?;
+
+            let plan = DFLogicalSubstraitConvertor {}
+                .encode(plan)
+                .context(SubstraitEncodeLogicalPlanSnafu)?;
+
+            self.database.logical_plan(plan.to_vec()).await
+        } else {
+            self.database.sql(&sql).await
+        }
+        .context(RequestDatabaseSnafu { sql: &sql })?;
+
+        let either = match output {
+            Output::Stream(s) => {
+                let x = RecordBatches::try_collect(s)
+                    .await
+                    .context(CollectRecordBatchesSnafu)?;
+                Either::Left(x)
+            }
+            Output::RecordBatches(x) => Either::Left(x),
+            Output::AffectedRows(rows) => Either::Right(rows),
+        };
+
+        let end = Instant::now();
+
+        match either {
+            Either::Left(recordbatches) => {
+                let total_rows: usize = recordbatches.iter().map(|x| x.num_rows()).sum();
+                if total_rows > 0 {
+                    println!(
+                        "{}",
+                        recordbatches
+                            .pretty_print()
+                            .context(PrettyPrintRecordBatchesSnafu)?
+                    );
+                }
+                println!("Total Rows: {total_rows}")
+            }
+            Either::Right(rows) => println!("Affected Rows: {rows}"),
+        };
+
+        println!("Cost {} ms", (end - start).as_millis());
+        Ok(())
+    }
+}
+
+impl Drop for Repl {
+    fn drop(&mut self) {
+        if self.rl.helper().is_some() {
+            let history_file = history_file();
+            if let Err(e) = self.rl.save_history(&history_file) {
+                logging::debug!(
+                    "failed to save history file on {}, error: {e}",
+                    history_file.display()
+                );
+            }
+        }
+    }
+}
+
+/// Return the location of the history file (defaults to $HOME/".greptimedb_cli_history")
+fn history_file() -> PathBuf {
+    let mut buf = match std::env::var("HOME") {
+        Ok(home) => PathBuf::from(home),
+        Err(_) => PathBuf::new(),
+    };
+    buf.push(".greptimedb_cli_history");
+    buf
+}
+
+async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
+    let mut meta_client = MetaClientBuilder::default().enable_store().build();
+    meta_client
+        .start([meta_addr])
+        .await
+        .context(StartMetaClientSnafu)?;
+    let meta_client = Arc::new(meta_client);
+
+    let backend = Arc::new(MetaKvBackend {
+        client: meta_client.clone(),
+    });
+
+    let table_routes = Arc::new(TableRoutes::new(meta_client));
+    let partition_manager = Arc::new(PartitionRuleManager::new(table_routes));
+
+    let datanode_clients = Arc::new(DatanodeClients::default());
+
+    let catalog_list = Arc::new(FrontendCatalogManager::new(
+        backend,
+        partition_manager,
+        datanode_clients,
+    ));
+
+    Ok(DatafusionQueryEngine::new(catalog_list, Default::default()))
+}
--- a/src/cmd/src/datanode.rs
+++ b/src/cmd/src/datanode.rs
@@ -14,8 +14,10 @@

 use clap::Parser;
 use common_telemetry::logging;
-use datanode::datanode::{Datanode, DatanodeOptions, FileConfig, ObjectStoreConfig};
-use meta_client::MetaClientOpts;
+use datanode::datanode::{
+    Datanode, DatanodeOptions, FileConfig, ObjectStoreConfig, ProcedureConfig,
+};
+use meta_client::MetaClientOptions;
 use servers::Mode;
 use snafu::ResultExt;

@@ -65,6 +67,8 @@ struct StartCommand {
    data_dir: Option<String>,
    #[clap(long)]
    wal_dir: Option<String>,
+    #[clap(long)]
+    procedure_dir: Option<String>,
 }

 impl StartCommand {
@@ -110,8 +114,8 @@ impl TryFrom<StartCommand> for DatanodeOptions {
        }

        if let Some(meta_addr) = cmd.metasrv_addr {
-            opts.meta_client_opts
-                .get_or_insert_with(MetaClientOpts::default)
+            opts.meta_client_options
+                .get_or_insert_with(MetaClientOptions::default)
                .metasrv_addrs = meta_addr
                .split(',')
                .map(&str::trim)
@@ -134,6 +138,11 @@ impl TryFrom<StartCommand> for DatanodeOptions {
        if let Some(wal_dir) = cmd.wal_dir {
            opts.wal.dir = wal_dir;
        }
+
+        if let Some(procedure_dir) = cmd.procedure_dir {
+            opts.procedure = Some(ProcedureConfig::from_file_path(procedure_dir));
+        }
+
        Ok(opts)
    }
 }
@@ -141,38 +150,79 @@ impl TryFrom<StartCommand> for DatanodeOptions {
 #[cfg(test)]
 mod tests {
    use std::assert_matches::assert_matches;
+    use std::io::Write;
    use std::time::Duration;

-    use datanode::datanode::ObjectStoreConfig;
+    use common_test_util::temp_dir::create_named_temp_file;
+    use datanode::datanode::{CompactionConfig, ObjectStoreConfig};
    use servers::Mode;

    use super::*;

    #[test]
    fn test_read_from_config_file() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            mode = "distributed"
+            enable_memory_catalog = false
+            node_id = 42
+            rpc_addr = "127.0.0.1:3001"
+            rpc_hostname = "127.0.0.1"
+            rpc_runtime_size = 8
+            mysql_addr = "127.0.0.1:4406"
+            mysql_runtime_size = 2
+
+            [meta_client_options]
+            metasrv_addrs = ["127.0.0.1:3002"]
+            timeout_millis = 3000
+            connect_timeout_millis = 5000
+            tcp_nodelay = true
+
+            [wal]
+            dir = "/tmp/greptimedb/wal"
+            file_size = "1GB"
+            purge_threshold = "50GB"
+            purge_interval = "10m"
+            read_batch_size = 128
+            sync_write = false
+
+            [storage]
+            type = "File"
+            data_dir = "/tmp/greptimedb/data/"
+
+            [compaction]
+            max_inflight_tasks = 4
+            max_files_in_level0 = 8
+            max_purge_tasks = 32
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
        let cmd = StartCommand {
-            config_file: Some(format!(
-                "{}/../../config/datanode.example.toml",
-                std::env::current_dir().unwrap().as_path().to_str().unwrap()
-            )),
+            config_file: Some(file.path().to_str().unwrap().to_string()),
            ..Default::default()
        };
        let options: DatanodeOptions = cmd.try_into().unwrap();
        assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
-        assert_eq!("/tmp/greptimedb/wal".to_string(), options.wal.dir);
        assert_eq!("127.0.0.1:4406".to_string(), options.mysql_addr);
-        assert_eq!(4, options.mysql_runtime_size);
-        let MetaClientOpts {
+        assert_eq!(2, options.mysql_runtime_size);
+        assert_eq!(Some(42), options.node_id);
+
+        assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
+        assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
+        assert_eq!(1024 * 1024 * 1024 * 50, options.wal.purge_threshold.0);
+        assert!(!options.wal.sync_write);
+
+        let MetaClientOptions {
            metasrv_addrs: metasrv_addr,
            timeout_millis,
            connect_timeout_millis,
            tcp_nodelay,
-        } = options.meta_client_opts.unwrap();
+        } = options.meta_client_options.unwrap();

        assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
        assert_eq!(5000, connect_timeout_millis);
        assert_eq!(3000, timeout_millis);
-        assert!(!tcp_nodelay);
+        assert!(tcp_nodelay);

        match options.storage {
            ObjectStoreConfig::File(FileConfig { data_dir }) => {
@@ -181,6 +231,15 @@ mod tests {
            ObjectStoreConfig::S3 { .. } => unreachable!(),
            ObjectStoreConfig::Oss { .. } => unreachable!(),
        };
+
+        assert_eq!(
+            CompactionConfig {
+                max_inflight_tasks: 4,
+                max_files_in_level0: 8,
+                max_purge_tasks: 32,
+            },
+            options.compaction
+        );
    }

    #[test]
@@ -214,32 +273,4 @@ mod tests {
        })
        .unwrap();
    }
-
-    #[test]
-    fn test_merge_config() {
-        let dn_opts = DatanodeOptions::try_from(StartCommand {
-            config_file: Some(format!(
-                "{}/../../config/datanode.example.toml",
-                std::env::current_dir().unwrap().as_path().to_str().unwrap()
-            )),
-            ..Default::default()
-        })
-        .unwrap();
-        assert_eq!("/tmp/greptimedb/wal", dn_opts.wal.dir);
-        assert_eq!(Duration::from_secs(600), dn_opts.wal.purge_interval);
-        assert_eq!(1024 * 1024 * 1024, dn_opts.wal.file_size.0);
-        assert_eq!(1024 * 1024 * 1024 * 50, dn_opts.wal.purge_threshold.0);
-        assert!(!dn_opts.wal.sync_write);
-        assert_eq!(Some(42), dn_opts.node_id);
-        let MetaClientOpts {
-            metasrv_addrs: metasrv_addr,
-            timeout_millis,
-            connect_timeout_millis,
-            tcp_nodelay,
-        } = dn_opts.meta_client_opts.unwrap();
-        assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
-        assert_eq!(3000, timeout_millis);
-        assert_eq!(5000, connect_timeout_millis);
-        assert!(!tcp_nodelay);
-    }
 }
--- a/src/cmd/src/error.rs
+++ b/src/cmd/src/error.rs
@@ -15,6 +15,7 @@
 use std::any::Any;

 use common_error::prelude::*;
+use rustyline::error::ReadlineError;

 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
@@ -68,6 +69,65 @@ pub enum Error {
        #[snafu(backtrace)]
        source: meta_srv::error::Error,
    },
+
+    #[snafu(display("Invalid REPL command: {reason}"))]
+    InvalidReplCommand { reason: String },
+
+    #[snafu(display("Cannot create REPL: {}", source))]
+    ReplCreation {
+        source: ReadlineError,
+        backtrace: Backtrace,
+    },
+
+    #[snafu(display("Error reading command: {}", source))]
+    Readline {
+        source: ReadlineError,
+        backtrace: Backtrace,
+    },
+
+    #[snafu(display("Failed to request database, sql: {sql}, source: {source}"))]
+    RequestDatabase {
+        sql: String,
+        #[snafu(backtrace)]
+        source: client::Error,
+    },
+
+    #[snafu(display("Failed to collect RecordBatches, source: {source}"))]
+    CollectRecordBatches {
+        #[snafu(backtrace)]
+        source: common_recordbatch::error::Error,
+    },
+
+    #[snafu(display("Failed to pretty print Recordbatches, source: {source}"))]
+    PrettyPrintRecordBatches {
+        #[snafu(backtrace)]
+        source: common_recordbatch::error::Error,
+    },
+
+    #[snafu(display("Failed to start Meta client, source: {}", source))]
+    StartMetaClient {
+        #[snafu(backtrace)]
+        source: meta_client::error::Error,
+    },
+
+    #[snafu(display("Failed to parse SQL: {}, source: {}", sql, source))]
+    ParseSql {
+        sql: String,
+        #[snafu(backtrace)]
+        source: query::error::Error,
+    },
+
+    #[snafu(display("Failed to plan statement, source: {}", source))]
+    PlanStatement {
+        #[snafu(backtrace)]
+        source: query::error::Error,
+    },
+
+    #[snafu(display("Failed to encode logical plan in substrait, source: {}", source))]
+    SubstraitEncodeLogicalPlan {
+        #[snafu(backtrace)]
+        source: substrait::error::Error,
+    },
 }

 pub type Result<T> = std::result::Result<T, Error>;
@@ -82,8 +142,20 @@ impl ErrorExt for Error {
            Error::ReadConfig { .. } | Error::ParseConfig { .. } | Error::MissingConfig { .. } => {
                StatusCode::InvalidArguments
            }
-            Error::IllegalConfig { .. } => StatusCode::InvalidArguments,
+            Error::IllegalConfig { .. } | Error::InvalidReplCommand { .. } => {
+                StatusCode::InvalidArguments
+            }
            Error::IllegalAuthConfig { .. } => StatusCode::InvalidArguments,
+            Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
+            Error::RequestDatabase { source, .. } => source.status_code(),
+            Error::CollectRecordBatches { source } | Error::PrettyPrintRecordBatches { source } => {
+                source.status_code()
+            }
+            Error::StartMetaClient { source } => source.status_code(),
+            Error::ParseSql { source, .. } | Error::PlanStatement { source } => {
+                source.status_code()
+            }
+            Error::SubstraitEncodeLogicalPlan { source } => source.status_code(),
        }
    }

--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -23,7 +23,8 @@ use frontend::instance::Instance;
 use frontend::mysql::MysqlOptions;
 use frontend::opentsdb::OpentsdbOptions;
 use frontend::postgres::PostgresOptions;
-use meta_client::MetaClientOpts;
+use frontend::prom::PromOptions;
+use meta_client::MetaClientOptions;
 use servers::auth::UserProviderRef;
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -67,6 +68,8 @@ pub struct StartCommand {
    #[clap(long)]
    mysql_addr: Option<String>,
    #[clap(long)]
+    prom_addr: Option<String>,
+    #[clap(long)]
    postgres_addr: Option<String>,
    #[clap(long)]
    opentsdb_addr: Option<String>,
@@ -141,6 +144,9 @@ impl TryFrom<StartCommand> for FrontendOptions {
                ..Default::default()
            });
        }
+        if let Some(addr) = cmd.prom_addr {
+            opts.prom_options = Some(PromOptions { addr });
+        }
        if let Some(addr) = cmd.postgres_addr {
            opts.postgres_options = Some(PostgresOptions {
                addr,
@@ -158,8 +164,8 @@ impl TryFrom<StartCommand> for FrontendOptions {
            opts.influxdb_options = Some(InfluxdbOptions { enable });
        }
        if let Some(metasrv_addr) = cmd.metasrv_addr {
-            opts.meta_client_opts
-                .get_or_insert_with(MetaClientOpts::default)
+            opts.meta_client_options
+                .get_or_insert_with(MetaClientOptions::default)
                .metasrv_addrs = metasrv_addr
                .split(',')
                .map(&str::trim)
@@ -173,8 +179,10 @@ impl TryFrom<StartCommand> for FrontendOptions {

 #[cfg(test)]
 mod tests {
+    use std::io::Write;
    use std::time::Duration;

+    use common_test_util::temp_dir::create_named_temp_file;
    use servers::auth::{Identity, Password, UserProviderRef};

    use super::*;
@@ -184,6 +192,7 @@ mod tests {
        let command = StartCommand {
            http_addr: Some("127.0.0.1:1234".to_string()),
            grpc_addr: None,
+            prom_addr: Some("127.0.0.1:4444".to_string()),
            mysql_addr: Some("127.0.0.1:5678".to_string()),
            postgres_addr: Some("127.0.0.1:5432".to_string()),
            opentsdb_addr: Some("127.0.0.1:4321".to_string()),
@@ -207,6 +216,7 @@ mod tests {
            opts.opentsdb_options.as_ref().unwrap().addr,
            "127.0.0.1:4321"
        );
+        assert_eq!(opts.prom_options.as_ref().unwrap().addr, "127.0.0.1:4444");

        let default_opts = FrontendOptions::default();
        assert_eq!(
@@ -231,17 +241,25 @@ mod tests {

    #[test]
    fn test_read_from_config_file() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            mode = "distributed"
+
+            [http_options]
+            addr = "127.0.0.1:4000"
+            timeout = "30s"
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
        let command = StartCommand {
            http_addr: None,
            grpc_addr: None,
            mysql_addr: None,
+            prom_addr: None,
            postgres_addr: None,
            opentsdb_addr: None,
            influxdb_enable: None,
-            config_file: Some(format!(
-                "{}/../../config/frontend.example.toml",
-                std::env::current_dir().unwrap().as_path().to_str().unwrap()
-            )),
+            config_file: Some(file.path().to_str().unwrap().to_string()),
            metasrv_addr: None,
            tls_mode: None,
            tls_cert_path: None,
@@ -267,6 +285,7 @@ mod tests {
            http_addr: None,
            grpc_addr: None,
            mysql_addr: None,
+            prom_addr: None,
            postgres_addr: None,
            opentsdb_addr: None,
            influxdb_enable: None,
--- a/src/cmd/src/lib.rs
+++ b/src/cmd/src/lib.rs
@@ -14,6 +14,7 @@

 #![feature(assert_matches)]

+pub mod cli;
 pub mod datanode;
 pub mod error;
 pub mod frontend;
--- a/src/cmd/src/metasrv.rs
+++ b/src/cmd/src/metasrv.rs
@@ -113,6 +113,9 @@ impl TryFrom<StartCommand> for MetaSrvOptions {

 #[cfg(test)]
 mod tests {
+    use std::io::Write;
+
+    use common_test_util::temp_dir::create_named_temp_file;
    use meta_srv::selector::SelectorType;

    use super::*;
@@ -136,15 +139,23 @@ mod tests {

    #[test]
    fn test_read_from_config_file() {
+        let mut file = create_named_temp_file();
+        let toml_str = r#"
+            bind_addr = "127.0.0.1:3002"
+            server_addr = "127.0.0.1:3002"
+            store_addr = "127.0.0.1:2379"
+            datanode_lease_secs = 15
+            selector = "LeaseBased"
+            use_memory_store = false
+        "#;
+        write!(file, "{}", toml_str).unwrap();
+
        let cmd = StartCommand {
            bind_addr: None,
            server_addr: None,
            store_addr: None,
            selector: None,
-            config_file: Some(format!(
-                "{}/../../config/metasrv.example.toml",
-                std::env::current_dir().unwrap().as_path().to_str().unwrap()
-            )),
+            config_file: Some(file.path().to_str().unwrap().to_string()),
            use_memory_store: false,
        };
        let options: MetaSrvOptions = cmd.try_into().unwrap();
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -17,7 +17,9 @@ use std::sync::Arc;
 use clap::Parser;
 use common_base::Plugins;
 use common_telemetry::info;
-use datanode::datanode::{Datanode, DatanodeOptions, ObjectStoreConfig, WalConfig};
+use datanode::datanode::{
+    CompactionConfig, Datanode, DatanodeOptions, ObjectStoreConfig, ProcedureConfig, WalConfig,
+};
 use datanode::instance::InstanceRef;
 use frontend::frontend::{Frontend, FrontendOptions};
 use frontend::grpc::GrpcOptions;
@@ -26,8 +28,8 @@ use frontend::instance::Instance as FeInstance;
 use frontend::mysql::MysqlOptions;
 use frontend::opentsdb::OpentsdbOptions;
 use frontend::postgres::PostgresOptions;
+use frontend::prom::PromOptions;
 use frontend::prometheus::PrometheusOptions;
-use frontend::promql::PromqlOptions;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -66,6 +68,8 @@ impl SubCommand {
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
 pub struct StandaloneOptions {
+    pub mode: Mode,
+    pub enable_memory_catalog: bool,
    pub http_options: Option<HttpOptions>,
    pub grpc_options: Option<GrpcOptions>,
    pub mysql_options: Option<MysqlOptions>,
@@ -73,16 +77,18 @@ pub struct StandaloneOptions {
    pub opentsdb_options: Option<OpentsdbOptions>,
    pub influxdb_options: Option<InfluxdbOptions>,
    pub prometheus_options: Option<PrometheusOptions>,
-    pub promql_options: Option<PromqlOptions>,
-    pub mode: Mode,
+    pub prom_options: Option<PromOptions>,
    pub wal: WalConfig,
    pub storage: ObjectStoreConfig,
-    pub enable_memory_catalog: bool,
+    pub compaction: CompactionConfig,
+    pub procedure: Option<ProcedureConfig>,
 }

 impl Default for StandaloneOptions {
    fn default() -> Self {
        Self {
+            mode: Mode::Standalone,
+            enable_memory_catalog: false,
            http_options: Some(HttpOptions::default()),
            grpc_options: Some(GrpcOptions::default()),
            mysql_options: Some(MysqlOptions::default()),
@@ -90,11 +96,11 @@ impl Default for StandaloneOptions {
            opentsdb_options: Some(OpentsdbOptions::default()),
            influxdb_options: Some(InfluxdbOptions::default()),
            prometheus_options: Some(PrometheusOptions::default()),
-            promql_options: Some(PromqlOptions::default()),
-            mode: Mode::Standalone,
+            prom_options: Some(PromOptions::default()),
            wal: WalConfig::default(),
            storage: ObjectStoreConfig::default(),
-            enable_memory_catalog: false,
+            compaction: CompactionConfig::default(),
+            procedure: None,
        }
    }
 }
@@ -102,6 +108,7 @@ impl Default for StandaloneOptions {
 impl StandaloneOptions {
    fn frontend_options(self) -> FrontendOptions {
        FrontendOptions {
+            mode: self.mode,
            http_options: self.http_options,
            grpc_options: self.grpc_options,
            mysql_options: self.mysql_options,
@@ -109,17 +116,18 @@ impl StandaloneOptions {
            opentsdb_options: self.opentsdb_options,
            influxdb_options: self.influxdb_options,
            prometheus_options: self.prometheus_options,
-            promql_options: self.promql_options,
-            mode: self.mode,
-            meta_client_opts: None,
+            prom_options: self.prom_options,
+            meta_client_options: None,
        }
    }

    fn datanode_options(self) -> DatanodeOptions {
        DatanodeOptions {
+            enable_memory_catalog: self.enable_memory_catalog,
            wal: self.wal,
            storage: self.storage,
-            enable_memory_catalog: self.enable_memory_catalog,
+            compaction: self.compaction,
+            procedure: self.procedure,
            ..Default::default()
        }
    }
@@ -134,6 +142,8 @@ struct StartCommand {
    #[clap(long)]
    mysql_addr: Option<String>,
    #[clap(long)]
+    prom_addr: Option<String>,
+    #[clap(long)]
    postgres_addr: Option<String>,
    #[clap(long)]
    opentsdb_addr: Option<String>,
@@ -246,6 +256,11 @@ impl TryFrom<StartCommand> for FrontendOptions {
                ..Default::default()
            })
        }
+
+        if let Some(addr) = cmd.prom_addr {
+            opts.prom_options = Some(PromOptions { addr })
+        }
+
        if let Some(addr) = cmd.postgres_addr {
            opts.postgres_options = Some(PostgresOptions {
                addr,
@@ -294,6 +309,7 @@ mod tests {
            http_addr: None,
            rpc_addr: None,
            mysql_addr: None,
+            prom_addr: None,
            postgres_addr: None,
            opentsdb_addr: None,
            config_file: Some(format!(
@@ -339,6 +355,7 @@ mod tests {
        let command = StartCommand {
            http_addr: None,
            rpc_addr: None,
+            prom_addr: None,
            mysql_addr: None,
            postgres_addr: None,
            opentsdb_addr: None,
@@ -362,4 +379,11 @@ mod tests {
            .await;
        assert!(result.is_ok());
    }
+
+    #[test]
+    fn test_toml() {
+        let opts = StandaloneOptions::default();
+        let toml_string = toml::to_string(&opts).unwrap();
+        let _parsed: StandaloneOptions = toml::from_str(&toml_string).unwrap();
+    }
 }
--- a/src/cmd/src/toml_loader.rs
+++ b/src/cmd/src/toml_loader.rs
@@ -29,9 +29,9 @@ mod tests {
    use std::fs::File;
    use std::io::Write;

+    use common_test_util::temp_dir::create_temp_dir;
    use serde::{Deserialize, Serialize};
    use snafu::ResultExt;
-    use tempdir::TempDir;

    use super::*;
    use crate::error::Result;
@@ -62,7 +62,7 @@ mod tests {
            host: "greptime.test".to_string(),
        };

-        let dir = TempDir::new("test_from_file").unwrap();
+        let dir = create_temp_dir("test_from_file");
        let test_file = format!("{}/test.toml", dir.path().to_str().unwrap());

        let s = toml::to_string(&config).unwrap();
--- a/src/cmd/tests/cli.rs
+++ b/src/cmd/tests/cli.rs
@@ -0,0 +1,145 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(target_os = "macos")]
+mod tests {
+    use std::path::PathBuf;
+    use std::process::{Command, Stdio};
+    use std::time::Duration;
+
+    use common_test_util::temp_dir::create_temp_dir;
+    use rexpect::session::PtyReplSession;
+
+    struct Repl {
+        repl: PtyReplSession,
+    }
+
+    impl Repl {
+        fn send_line(&mut self, line: &str) {
+            self.repl.send_line(line).unwrap();
+
+            // read a line to consume the prompt
+            self.read_line();
+        }
+
+        fn read_line(&mut self) -> String {
+            self.repl.read_line().unwrap()
+        }
+
+        fn read_expect(&mut self, expect: &str) {
+            assert_eq!(self.read_line(), expect);
+        }
+
+        fn read_contains(&mut self, pat: &str) {
+            assert!(self.read_line().contains(pat));
+        }
+    }
+
+    #[test]
+    fn test_repl() {
+        let data_dir = create_temp_dir("data");
+        let wal_dir = create_temp_dir("wal");
+
+        let mut bin_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        bin_path.push("../../target/debug");
+        let bin_path = bin_path.to_str().unwrap();
+
+        let mut datanode = Command::new("./greptime")
+            .current_dir(bin_path)
+            .args([
+                "datanode",
+                "start",
+                "--rpc-addr=0.0.0.0:4321",
+                "--node-id=1",
+                &format!("--data-dir={}", data_dir.path().display()),
+                &format!("--wal-dir={}", wal_dir.path().display()),
+            ])
+            .stdout(Stdio::null())
+            .spawn()
+            .unwrap();
+
+        // wait for Datanode actually started
+        std::thread::sleep(Duration::from_secs(3));
+
+        let mut repl_cmd = Command::new("./greptime");
+        repl_cmd.current_dir(bin_path).args([
+            "--log-level=off",
+            "cli",
+            "attach",
+            "--grpc-addr=0.0.0.0:4321",
+            // history commands can sneaky into stdout and mess up our tests, so disable it
+            "--disable-helper",
+        ]);
+        let pty_session = rexpect::session::spawn_command(repl_cmd, Some(5_000)).unwrap();
+        let repl = PtyReplSession {
+            prompt: "> ".to_string(),
+            pty_session,
+            quit_command: None,
+            echo_on: false,
+        };
+        let repl = &mut Repl { repl };
+        repl.read_expect("Ready for commands. (Hint: try 'help')");
+
+        test_create_database(repl);
+
+        test_use_database(repl);
+
+        test_create_table(repl);
+
+        test_insert(repl);
+
+        test_select(repl);
+
+        datanode.kill().unwrap();
+        datanode.wait().unwrap();
+    }
+
+    fn test_create_database(repl: &mut Repl) {
+        repl.send_line("CREATE DATABASE db;");
+        repl.read_expect("Affected Rows: 1");
+        repl.read_contains("Cost");
+    }
+
+    fn test_use_database(repl: &mut Repl) {
+        repl.send_line("USE db");
+        repl.read_expect("Total Rows: 0");
+        repl.read_contains("Cost");
+        repl.read_expect("Using db");
+    }
+
+    fn test_create_table(repl: &mut Repl) {
+        repl.send_line("CREATE TABLE t(x STRING, ts TIMESTAMP TIME INDEX);");
+        repl.read_expect("Affected Rows: 0");
+        repl.read_contains("Cost");
+    }
+
+    fn test_insert(repl: &mut Repl) {
+        repl.send_line("INSERT INTO t(x, ts) VALUES ('hello', 1676895812239);");
+        repl.read_expect("Affected Rows: 1");
+        repl.read_contains("Cost");
+    }
+
+    fn test_select(repl: &mut Repl) {
+        repl.send_line("SELECT * FROM t;");
+
+        repl.read_expect("+-------+-------------------------+");
+        repl.read_expect("| x     | ts                      |");
+        repl.read_expect("+-------+-------------------------+");
+        repl.read_expect("| hello | 2023-02-20T12:23:32.239 |");
+        repl.read_expect("+-------+-------------------------+");
+        repl.read_expect("Total Rows: 1");
+
+        repl.read_contains("Cost");
+    }
+}
--- a/src/common/base/src/bytes.rs
+++ b/src/common/base/src/bytes.rs
@@ -20,6 +20,12 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
 #[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
 pub struct Bytes(bytes::Bytes);

+impl From<Bytes> for bytes::Bytes {
+    fn from(value: Bytes) -> Self {
+        value.0
+    }
+}
+
 impl From<bytes::Bytes> for Bytes {
    fn from(bytes: bytes::Bytes) -> Bytes {
        Bytes(bytes)
--- a/src/common/catalog/Cargo.toml
+++ b/src/common/catalog/Cargo.toml
@@ -17,5 +17,4 @@ snafu = { version = "0.7", features = ["backtraces"] }

 [dev-dependencies]
 chrono.workspace = true
-tempdir = "0.3"
 tokio.workspace = true
--- a/src/common/error/src/status_code.rs
+++ b/src/common/error/src/status_code.rs
@@ -86,6 +86,34 @@ impl StatusCode {
    pub fn is_success(code: u32) -> bool {
        Self::Success as u32 == code
    }
+
+    pub fn is_retryable(&self) -> bool {
+        match self {
+            StatusCode::StorageUnavailable
+            | StatusCode::RuntimeResourcesExhausted
+            | StatusCode::Internal => true,
+
+            StatusCode::Success
+            | StatusCode::Unknown
+            | StatusCode::Unsupported
+            | StatusCode::Unexpected
+            | StatusCode::InvalidArguments
+            | StatusCode::InvalidSyntax
+            | StatusCode::PlanQuery
+            | StatusCode::EngineExecuteQuery
+            | StatusCode::TableAlreadyExists
+            | StatusCode::TableNotFound
+            | StatusCode::TableColumnNotFound
+            | StatusCode::TableColumnExists
+            | StatusCode::DatabaseNotFound
+            | StatusCode::UserNotFound
+            | StatusCode::UnsupportedPasswordType
+            | StatusCode::UserPasswordMismatch
+            | StatusCode::AuthHeaderNotFound
+            | StatusCode::InvalidAuthHeader
+            | StatusCode::AccessDenied => false,
+        }
+    }
 }

 impl fmt::Display for StatusCode {
--- a/src/common/function-macro/Cargo.toml
+++ b/src/common/function-macro/Cargo.toml
@@ -10,6 +10,7 @@ proc-macro = true
 [dependencies]
 quote = "1.0"
 syn = "1.0"
+proc-macro2 = "1.0"

 [dev-dependencies]
 arc-swap = "1.0"
--- a/src/common/function-macro/src/lib.rs
+++ b/src/common/function-macro/src/lib.rs
@@ -12,8 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+mod range_fn;
+
 use proc_macro::TokenStream;
 use quote::{quote, quote_spanned};
+use range_fn::process_range_fn;
 use syn::parse::Parser;
 use syn::spanned::Spanned;
 use syn::{parse_macro_input, DeriveInput, ItemStruct};
@@ -83,3 +86,31 @@ pub fn as_aggr_func_creator(_args: TokenStream, input: TokenStream) -> TokenStre
    }
    .into()
 }
+
+/// Attribute macro to convert an arithimetic function to a range function. The annotated function
+/// should accept servaral arrays as input and return a single value as output. This procedure
+/// macro can works on any number of input parameters. Return type can be either primitive type
+/// or wrapped in `Option`.
+///
+/// # Example
+/// Take `count_over_time()` in PromQL as an example:
+/// ```rust, ignore
+/// /// The count of all values in the specified interval.
+/// #[range_fn(
+///     name = "CountOverTime",
+///     ret = "Float64Array",
+///     display_name = "prom_count_over_time"
+/// )]
+/// pub fn count_over_time(_: &TimestampMillisecondArray, values: &Float64Array) -> f64 {
+///      values.len() as f64
+/// }
+/// ```
+///
+/// # Arguments
+/// - `name`: The name of the generated [ScalarUDF] struct.
+/// - `ret`: The return type of the generated UDF function.
+/// - `display_name`: The display name of the generated UDF function.
+#[proc_macro_attribute]
+pub fn range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
+    process_range_fn(args, input)
+}
--- a/src/common/function-macro/src/range_fn.rs
+++ b/src/common/function-macro/src/range_fn.rs
@@ -0,0 +1,230 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use syn::punctuated::Punctuated;
+use syn::spanned::Spanned;
+use syn::token::Comma;
+use syn::{
+    parse_macro_input, Attribute, AttributeArgs, FnArg, Ident, ItemFn, Meta, MetaNameValue,
+    NestedMeta, Signature, Type, TypeReference, Visibility,
+};
+
+/// Internal util macro to early return on error.
+macro_rules! ok {
+    ($item:expr) => {
+        match $item {
+            Ok(item) => item,
+            Err(e) => return e.into_compile_error().into(),
+        }
+    };
+}
+
+pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
+    // extract arg map
+    let arg_pairs = parse_macro_input!(args as AttributeArgs);
+    let arg_span = arg_pairs[0].span();
+    let arg_map = ok!(extract_arg_map(arg_pairs));
+
+    // decompose the fn block
+    let compute_fn = parse_macro_input!(input as ItemFn);
+    let ItemFn {
+        attrs,
+        vis,
+        sig,
+        block,
+    } = compute_fn;
+
+    // extract fn arg list
+    let Signature {
+        inputs,
+        ident: fn_name,
+        ..
+    } = &sig;
+    let arg_types = ok!(extract_input_types(inputs));
+
+    // build the struct and its impl block
+    let struct_code = build_struct(
+        attrs,
+        vis,
+        ok!(get_ident(&arg_map, "name", arg_span)),
+        ok!(get_ident(&arg_map, "display_name", arg_span)),
+    );
+    let calc_fn_code = build_calc_fn(
+        ok!(get_ident(&arg_map, "name", arg_span)),
+        arg_types,
+        fn_name.clone(),
+        ok!(get_ident(&arg_map, "ret", arg_span)),
+    );
+    // preserve this fn, but remove its `pub` modifier
+    let input_fn_code: TokenStream = quote! {
+        #sig { #block }
+    }
+    .into();
+
+    let mut result = TokenStream::new();
+    result.extend(struct_code);
+    result.extend(calc_fn_code);
+    result.extend(input_fn_code);
+    result
+}
+
+/// Extract a String <-> Ident map from the attribute args.
+fn extract_arg_map(args: Vec<NestedMeta>) -> Result<HashMap<String, Ident>, syn::Error> {
+    args.into_iter()
+        .map(|meta| {
+            if let NestedMeta::Meta(Meta::NameValue(MetaNameValue { path, lit, .. })) = meta {
+                let name = path.get_ident().unwrap().to_string();
+                let ident = match lit {
+                    syn::Lit::Str(lit_str) => lit_str.parse::<Ident>(),
+                    _ => Err(syn::Error::new(
+                        lit.span(),
+                        "Unexpected attribute format. Expected `name = \"value\"`",
+                    )),
+                }?;
+                Ok((name, ident))
+            } else {
+                Err(syn::Error::new(
+                    meta.span(),
+                    "Unexpected attribute format. Expected `name = \"value\"`",
+                ))
+            }
+        })
+        .collect::<Result<HashMap<String, Ident>, syn::Error>>()
+}
+
+/// Helper function to get an Ident from the previous arg map.
+fn get_ident(map: &HashMap<String, Ident>, key: &str, span: Span) -> Result<Ident, syn::Error> {
+    map.get(key)
+        .cloned()
+        .ok_or_else(|| syn::Error::new(span, format!("Expect attribute {key} but not found")))
+}
+
+/// Extract the argument list from the annotated function.
+fn extract_input_types(inputs: &Punctuated<FnArg, Comma>) -> Result<Vec<Type>, syn::Error> {
+    inputs
+        .iter()
+        .map(|arg| match arg {
+            FnArg::Receiver(receiver) => Err(syn::Error::new(receiver.span(), "expected bool")),
+            FnArg::Typed(pat_type) => Ok(*pat_type.ty.clone()),
+        })
+        .collect()
+}
+
+fn build_struct(
+    attrs: Vec<Attribute>,
+    vis: Visibility,
+    name: Ident,
+    display_name_ident: Ident,
+) -> TokenStream {
+    let display_name = display_name_ident.to_string();
+    quote! {
+        #(#attrs)*
+        #[derive(Debug)]
+        #vis struct #name {}
+
+        impl #name {
+            pub const fn name() -> &'static str {
+                #display_name
+            }
+
+            pub fn scalar_udf() -> ScalarUDF {
+                ScalarUDF {
+                    name: Self::name().to_string(),
+                    signature: Signature::new(
+                        TypeSignature::Exact(Self::input_type()),
+                        Volatility::Immutable,
+                    ),
+                    return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))),
+                    fun: Arc::new(Self::calc),
+                }
+            }
+
+            // TODO(ruihang): this should be parameterized
+            // time index column and value column
+            fn input_type() -> Vec<DataType> {
+                vec![
+                    RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
+                    RangeArray::convert_data_type(DataType::Float64),
+                ]
+            }
+
+            // TODO(ruihang): this should be parameterized
+            fn return_type() -> DataType {
+                DataType::Float64
+            }
+        }
+    }
+    .into()
+}
+
+fn build_calc_fn(
+    name: Ident,
+    param_types: Vec<Type>,
+    fn_name: Ident,
+    ret_type: Ident,
+) -> TokenStream {
+    let param_names = param_types
+        .iter()
+        .enumerate()
+        .map(|(i, ty)| Ident::new(&format!("param_{}", i), ty.span()))
+        .collect::<Vec<_>>();
+    let unref_param_types = param_types
+        .iter()
+        .map(|ty| {
+            if let Type::Reference(TypeReference { elem, .. }) = ty {
+                elem.as_ref().clone()
+            } else {
+                ty.clone()
+            }
+        })
+        .collect::<Vec<_>>();
+    let num_params = param_types.len();
+    let param_numbers = (0..num_params).collect::<Vec<_>>();
+    let range_array_names = param_names
+        .iter()
+        .map(|name| Ident::new(&format!("{}_range_array", name), name.span()))
+        .collect::<Vec<_>>();
+    let first_range_array_name = range_array_names.first().unwrap().clone();
+
+    quote! {
+        impl #name {
+            fn calc(input: &[ColumnarValue]) -> Result<ColumnarValue, DataFusionError> {
+                assert_eq!(input.len(), #num_params);
+
+                #( let #range_array_names = RangeArray::try_new(extract_array(&input[#param_numbers])?.data().clone().into())?; )*
+
+                // TODO(ruihang): add ensure!() 
+
+                let mut result_array = Vec::new();
+                for index in 0..#first_range_array_name.len(){
+                    #( let #param_names = #range_array_names.get(index).unwrap().as_any().downcast_ref::<#unref_param_types>().unwrap().clone(); )*
+
+                    // TODO(ruihang): add ensure!() to check length
+
+                    let result = #fn_name(#( &#param_names, )*);
+                    result_array.push(result);
+                }
+
+                let result = ColumnarValue::Array(Arc::new(#ret_type::from_iter(result_array)));
+                Ok(result)
+            }
+        }
+    }
+    .into()
+}
--- a/src/common/function-macro/tests/test_derive.rs
+++ b/src/common/function-macro/tests/test_derive.rs
@@ -20,6 +20,7 @@ use static_assertions::{assert_fields, assert_impl_all};
 struct Foo {}

 #[test]
+#[allow(clippy::extra_unused_type_parameters)]
 fn test_derive() {
    Foo::default();
    assert_fields!(Foo: input_types);
--- a/src/common/function/Cargo.toml
+++ b/src/common/function/Cargo.toml
@@ -19,7 +19,7 @@ num-traits = "0.2"
 once_cell = "1.10"
 paste = "1.0"
 snafu.workspace = true
-statrs = "0.15"
+statrs = "0.16"

 [dev-dependencies]
 ron = "0.7"
--- a/src/common/function/src/scalars/udf.rs
+++ b/src/common/function/src/scalars/udf.rs
@@ -14,9 +14,9 @@

 use std::sync::Arc;

-use common_query::error::{ExecuteFunctionSnafu, FromScalarValueSnafu};
+use common_query::error::FromScalarValueSnafu;
 use common_query::prelude::{
-    ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUdf, ScalarValue,
+    ColumnarValue, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUdf,
 };
 use datatypes::error::Error as DataTypeError;
 use datatypes::prelude::*;
@@ -54,16 +54,8 @@ pub fn create_udf(func: FunctionRef) -> ScalarUdf {
            .collect();

        let result = func_cloned.eval(func_ctx, &args.context(FromScalarValueSnafu)?);
-
-        let udf = if len.is_some() {
-            result.map(ColumnarValue::Vector)?
-        } else {
-            ScalarValue::try_from_array(&result?.to_arrow_array(), 0)
-                .map(ColumnarValue::Scalar)
-                .context(ExecuteFunctionSnafu)?
-        };
-
-        Ok(udf)
+        let udf_result = result.map(ColumnarValue::Vector)?;
+        Ok(udf_result)
    });

    ScalarUdf::new(func.name(), &func.signature(), &return_type, &fun)
--- a/src/common/grpc-expr/src/alter.rs
+++ b/src/common/grpc-expr/src/alter.rs
@@ -12,19 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::sync::Arc;
-
 use api::v1::alter_expr::Kind;
 use api::v1::{column_def, AlterExpr, CreateTableExpr, DropColumns, RenameTable};
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use datatypes::schema::{ColumnSchema, SchemaBuilder, SchemaRef};
+use datatypes::schema::{ColumnSchema, RawSchema};
 use snafu::{ensure, OptionExt, ResultExt};
 use table::metadata::TableId;
-use table::requests::{AddColumnRequest, AlterKind, AlterTableRequest, CreateTableRequest};
+use table::requests::{
+    AddColumnRequest, AlterKind, AlterTableRequest, CreateTableRequest, TableOptions,
+};

 use crate::error::{
-    ColumnNotFoundSnafu, CreateSchemaSnafu, InvalidColumnDefSnafu, MissingFieldSnafu,
-    MissingTimestampColumnSnafu, Result,
+    ColumnNotFoundSnafu, InvalidColumnDefSnafu, MissingFieldSnafu, MissingTimestampColumnSnafu,
+    Result, UnrecognizedTableOptionSnafu,
 };

 /// Convert an [`AlterExpr`] to an [`AlterTableRequest`]
@@ -92,7 +92,7 @@ pub fn alter_expr_to_request(expr: AlterExpr) -> Result<AlterTableRequest> {
    }
 }

-pub fn create_table_schema(expr: &CreateTableExpr) -> Result<SchemaRef> {
+pub fn create_table_schema(expr: &CreateTableExpr) -> Result<RawSchema> {
    let column_schemas = expr
        .column_defs
        .iter()
@@ -121,12 +121,7 @@ pub fn create_table_schema(expr: &CreateTableExpr) -> Result<SchemaRef> {
        })
        .collect::<Vec<_>>();

-    Ok(Arc::new(
-        SchemaBuilder::try_from(column_schemas)
-            .context(CreateSchemaSnafu)?
-            .build()
-            .context(CreateSchemaSnafu)?,
-    ))
+    Ok(RawSchema::new(column_schemas))
 }

 pub fn create_expr_to_request(
@@ -138,8 +133,11 @@ pub fn create_expr_to_request(
        .primary_keys
        .iter()
        .map(|key| {
+            // We do a linear search here.
            schema
-                .column_index_by_name(key)
+                .column_schemas
+                .iter()
+                .position(|column_schema| column_schema.name == *key)
                .context(ColumnNotFoundSnafu {
                    column_name: key,
                    table_name: &expr.table_name,
@@ -167,6 +165,8 @@ pub fn create_expr_to_request(
        expr.region_ids
    };

+    let table_options =
+        TableOptions::try_from(&expr.table_options).context(UnrecognizedTableOptionSnafu)?;
    Ok(CreateTableRequest {
        id: table_id,
        catalog_name,
@@ -177,7 +177,7 @@ pub fn create_expr_to_request(
        region_numbers: region_ids,
        primary_key_indices,
        create_if_not_exists: expr.create_if_not_exists,
-        table_options: expr.table_options,
+        table_options,
    })
 }

--- a/src/common/grpc-expr/src/error.rs
+++ b/src/common/grpc-expr/src/error.rs
@@ -40,12 +40,6 @@ pub enum Error {
        source: api::error::Error,
    },

-    #[snafu(display("Failed to create schema when creating table, source: {}", source))]
-    CreateSchema {
-        #[snafu(backtrace)]
-        source: datatypes::error::Error,
-    },
-
    #[snafu(display(
        "Duplicated timestamp column in gRPC requests, exists {}, duplicated: {}",
        exists,
@@ -90,6 +84,12 @@ pub enum Error {
        #[snafu(backtrace)]
        source: api::error::Error,
    },
+
+    #[snafu(display("Unrecognized table option: {}", source))]
+    UnrecognizedTableOption {
+        #[snafu(backtrace)]
+        source: table::error::Error,
+    },
 }

 pub type Result<T> = std::result::Result<T, Error>;
@@ -102,14 +102,15 @@ impl ErrorExt for Error {
                StatusCode::InvalidArguments
            }
            Error::ColumnDataType { .. } => StatusCode::Internal,
-            Error::CreateSchema { .. }
-            | Error::DuplicatedTimestampColumn { .. }
-            | Error::MissingTimestampColumn { .. } => StatusCode::InvalidArguments,
+            Error::DuplicatedTimestampColumn { .. } | Error::MissingTimestampColumn { .. } => {
+                StatusCode::InvalidArguments
+            }
            Error::InvalidColumnProto { .. } => StatusCode::InvalidArguments,
            Error::CreateVector { .. } => StatusCode::InvalidArguments,
            Error::MissingField { .. } => StatusCode::InvalidArguments,
            Error::ColumnDefaultConstraint { source, .. } => source.status_code(),
            Error::InvalidColumnDef { source, .. } => source.status_code(),
+            Error::UnrecognizedTableOption { .. } => StatusCode::InvalidArguments,
        }
    }
    fn backtrace_opt(&self) -> Option<&Backtrace> {
--- a/src/common/grpc-expr/src/insert.rs
+++ b/src/common/grpc-expr/src/insert.rs
@@ -26,6 +26,7 @@ use common_time::{Date, DateTime};
 use datatypes::data_type::{ConcreteDataType, DataType};
 use datatypes::prelude::{ValueRef, VectorRef};
 use datatypes::schema::SchemaRef;
+use datatypes::types::TimestampType;
 use datatypes::value::Value;
 use datatypes::vectors::MutableVector;
 use snafu::{ensure, OptionExt, ResultExt};
@@ -96,9 +97,7 @@ pub fn column_to_vector(column: &Column, rows: u32) -> Result<VectorRef> {

        for i in 0..rows {
            if let Some(true) = nulls_iter.next() {
-                vector
-                    .push_value_ref(ValueRef::Null)
-                    .context(CreateVectorSnafu)?;
+                vector.push_null();
            } else {
                let value_ref = values_iter
                    .next()
@@ -109,16 +108,12 @@ pub fn column_to_vector(column: &Column, rows: u32) -> Result<VectorRef> {
                        ),
                    })?;
                vector
-                    .push_value_ref(value_ref)
+                    .try_push_value_ref(value_ref)
                    .context(CreateVectorSnafu)?;
            }
        }
    } else {
-        (0..rows).try_for_each(|_| {
-            vector
-                .push_value_ref(ValueRef::Null)
-                .context(CreateVectorSnafu)
-        })?;
+        (0..rows).for_each(|_| vector.push_null());
    }
    Ok(vector.to_vector())
 }
@@ -324,7 +319,7 @@ fn add_values_to_builder(

        values.iter().try_for_each(|value| {
            builder
-                .push_value_ref(value.as_value_ref())
+                .try_push_value_ref(value.as_value_ref())
                .context(CreateVectorSnafu)
        })?;
    } else {
@@ -337,12 +332,10 @@ fn add_values_to_builder(
        let mut idx_of_values = 0;
        for idx in 0..row_count {
            match is_null(&null_mask, idx) {
-                Some(true) => builder
-                    .push_value_ref(ValueRef::Null)
-                    .context(CreateVectorSnafu)?,
+                Some(true) => builder.push_null(),
                _ => {
                    builder
-                        .push_value_ref(values[idx_of_values].as_value_ref())
+                        .try_push_value_ref(values[idx_of_values].as_value_ref())
                        .context(CreateVectorSnafu)?;
                    idx_of_values += 1
                }
@@ -422,13 +415,29 @@ fn convert_values(data_type: &ConcreteDataType, values: Values) -> Vec<Value> {
            .into_iter()
            .map(|v| Value::Date(v.into()))
            .collect(),
-        ConcreteDataType::Timestamp(_) => values
+        ConcreteDataType::Timestamp(TimestampType::Second(_)) => values
+            .ts_second_values
+            .into_iter()
+            .map(|v| Value::Timestamp(Timestamp::new_second(v)))
+            .collect(),
+        ConcreteDataType::Timestamp(TimestampType::Millisecond(_)) => values
            .ts_millisecond_values
            .into_iter()
            .map(|v| Value::Timestamp(Timestamp::new_millisecond(v)))
            .collect(),
-        ConcreteDataType::Null(_) => unreachable!(),
-        ConcreteDataType::List(_) => unreachable!(),
+        ConcreteDataType::Timestamp(TimestampType::Microsecond(_)) => values
+            .ts_microsecond_values
+            .into_iter()
+            .map(|v| Value::Timestamp(Timestamp::new_microsecond(v)))
+            .collect(),
+        ConcreteDataType::Timestamp(TimestampType::Nanosecond(_)) => values
+            .ts_nanosecond_values
+            .into_iter()
+            .map(|v| Value::Timestamp(Timestamp::new_nanosecond(v)))
+            .collect(),
+        ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
+            unreachable!()
+        }
    }
 }

@@ -451,6 +460,7 @@ mod tests {
    use common_time::timestamp::Timestamp;
    use datatypes::data_type::ConcreteDataType;
    use datatypes::schema::{ColumnSchema, SchemaBuilder, SchemaRef};
+    use datatypes::types::{TimestampMillisecondType, TimestampSecondType, TimestampType};
    use datatypes::value::Value;
    use snafu::ResultExt;
    use table::error::Result as TableResult;
@@ -654,6 +664,39 @@ mod tests {
        );
    }

+    #[test]
+    fn test_convert_timestamp_values() {
+        // second
+        let actual = convert_values(
+            &ConcreteDataType::Timestamp(TimestampType::Second(TimestampSecondType)),
+            Values {
+                ts_second_values: vec![1_i64, 2_i64, 3_i64],
+                ..Default::default()
+            },
+        );
+        let expect = vec![
+            Value::Timestamp(Timestamp::new_second(1_i64)),
+            Value::Timestamp(Timestamp::new_second(2_i64)),
+            Value::Timestamp(Timestamp::new_second(3_i64)),
+        ];
+        assert_eq!(expect, actual);
+
+        // millisecond
+        let actual = convert_values(
+            &ConcreteDataType::Timestamp(TimestampType::Millisecond(TimestampMillisecondType)),
+            Values {
+                ts_millisecond_values: vec![1_i64, 2_i64, 3_i64],
+                ..Default::default()
+            },
+        );
+        let expect = vec![
+            Value::Timestamp(Timestamp::new_millisecond(1_i64)),
+            Value::Timestamp(Timestamp::new_millisecond(2_i64)),
+            Value::Timestamp(Timestamp::new_millisecond(3_i64)),
+        ];
+        assert_eq!(expect, actual);
+    }
+
    #[test]
    fn test_is_null() {
        let null_mask = BitVec::from_slice(&[0b0000_0001, 0b0000_1000]);
--- a/src/common/grpc/Cargo.toml
+++ b/src/common/grpc/Cargo.toml
@@ -16,7 +16,7 @@ common-runtime = { path = "../runtime" }
 dashmap = "5.4"
 datafusion.workspace = true
 datatypes = { path = "../../datatypes" }
-flatbuffers = "22"
+flatbuffers = "23.1"
 futures = "0.3"
 prost.workspace = true
 snafu = { version = "0.7", features = ["backtraces"] }
--- a/src/common/grpc/src/channel_manager.rs
+++ b/src/common/grpc/src/channel_manager.rs
@@ -18,18 +18,20 @@ use std::time::Duration;

 use dashmap::mapref::entry::Entry;
 use dashmap::DashMap;
-use snafu::ResultExt;
-use tonic::transport::{Channel as InnerChannel, Endpoint, Uri};
+use snafu::{OptionExt, ResultExt};
+use tonic::transport::{
+    Certificate, Channel as InnerChannel, ClientTlsConfig, Endpoint, Identity, Uri,
+};
 use tower::make::MakeConnection;

-use crate::error;
-use crate::error::Result;
+use crate::error::{CreateChannelSnafu, InvalidConfigFilePathSnafu, InvalidTlsConfigSnafu, Result};

 const RECYCLE_CHANNEL_INTERVAL_SECS: u64 = 60;

 #[derive(Clone, Debug)]
 pub struct ChannelManager {
    config: ChannelConfig,
+    client_tls_config: Option<ClientTlsConfig>,
    pool: Arc<Pool>,
 }

@@ -52,7 +54,37 @@ impl ChannelManager {
            recycle_channel_in_loop(cloned_pool, RECYCLE_CHANNEL_INTERVAL_SECS).await;
        });

-        Self { config, pool }
+        Self {
+            config,
+            client_tls_config: None,
+            pool,
+        }
+    }
+
+    pub fn with_tls_config(config: ChannelConfig) -> Result<Self> {
+        let mut cm = Self::with_config(config.clone());
+
+        // setup tls
+        let path_config = config.client_tls.context(InvalidTlsConfigSnafu {
+            msg: "no config input",
+        })?;
+
+        let server_root_ca_cert = std::fs::read_to_string(path_config.server_ca_cert_path)
+            .context(InvalidConfigFilePathSnafu)?;
+        let server_root_ca_cert = Certificate::from_pem(server_root_ca_cert);
+        let client_cert = std::fs::read_to_string(path_config.client_cert_path)
+            .context(InvalidConfigFilePathSnafu)?;
+        let client_key = std::fs::read_to_string(path_config.client_key_path)
+            .context(InvalidConfigFilePathSnafu)?;
+        let client_identity = Identity::from_pem(client_cert, client_key);
+
+        cm.client_tls_config = Some(
+            ClientTlsConfig::new()
+                .ca_certificate(server_root_ca_cert)
+                .identity(client_identity),
+        );
+
+        Ok(cm)
    }

    pub fn config(&self) -> &ChannelConfig {
@@ -119,8 +151,7 @@ impl ChannelManager {
    }

    fn build_endpoint(&self, addr: &str) -> Result<Endpoint> {
-        let mut endpoint =
-            Endpoint::new(format!("http://{addr}")).context(error::CreateChannelSnafu)?;
+        let mut endpoint = Endpoint::new(format!("http://{addr}")).context(CreateChannelSnafu)?;

        if let Some(dur) = self.config.timeout {
            endpoint = endpoint.timeout(dur);
@@ -152,6 +183,12 @@ impl ChannelManager {
        if let Some(enabled) = self.config.http2_adaptive_window {
            endpoint = endpoint.http2_adaptive_window(enabled);
        }
+        if let Some(tls_config) = &self.client_tls_config {
+            endpoint = endpoint
+                .tls_config(tls_config.clone())
+                .context(CreateChannelSnafu)?;
+        }
+
        endpoint = endpoint
            .tcp_keepalive(self.config.tcp_keepalive)
            .tcp_nodelay(self.config.tcp_nodelay);
@@ -160,6 +197,13 @@ impl ChannelManager {
    }
 }

+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ClientTlsOption {
+    pub server_ca_cert_path: String,
+    pub client_cert_path: String,
+    pub client_key_path: String,
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ChannelConfig {
    pub timeout: Option<Duration>,
@@ -174,6 +218,7 @@ pub struct ChannelConfig {
    pub http2_adaptive_window: Option<bool>,
    pub tcp_keepalive: Option<Duration>,
    pub tcp_nodelay: bool,
+    pub client_tls: Option<ClientTlsOption>,
 }

 impl Default for ChannelConfig {
@@ -191,6 +236,7 @@ impl Default for ChannelConfig {
            http2_adaptive_window: None,
            tcp_keepalive: None,
            tcp_nodelay: true,
+            client_tls: None,
        }
    }
 }
@@ -307,6 +353,16 @@ impl ChannelConfig {
            ..self
        }
    }
+
+    /// Set the value of tls client auth.
+    ///
+    /// Disabled by default.
+    pub fn client_tls_config(self, client_tls_option: ClientTlsOption) -> Self {
+        Self {
+            client_tls: Some(client_tls_option),
+            ..self
+        }
+    }
 }

 #[derive(Debug)]
@@ -401,7 +457,11 @@ mod tests {
    async fn test_access_count() {
        let pool = Arc::new(Pool::default());
        let config = ChannelConfig::new();
-        let mgr = Arc::new(ChannelManager { pool, config });
+        let mgr = Arc::new(ChannelManager {
+            pool,
+            config,
+            client_tls_config: None,
+        });
        let addr = "test_uri";

        let mut joins = Vec::with_capacity(10);
@@ -443,6 +503,7 @@ mod tests {
                http2_adaptive_window: None,
                tcp_keepalive: None,
                tcp_nodelay: true,
+                client_tls: None,
            },
            default_cfg
        );
@@ -459,7 +520,12 @@ mod tests {
            .http2_keep_alive_while_idle(true)
            .http2_adaptive_window(true)
            .tcp_keepalive(Duration::from_secs(2))
-            .tcp_nodelay(false);
+            .tcp_nodelay(false)
+            .client_tls_config(ClientTlsOption {
+                server_ca_cert_path: "some_server_path".to_string(),
+                client_cert_path: "some_cert_path".to_string(),
+                client_key_path: "some_key_path".to_string(),
+            });

        assert_eq!(
            ChannelConfig {
@@ -475,6 +541,11 @@ mod tests {
                http2_adaptive_window: Some(true),
                tcp_keepalive: Some(Duration::from_secs(2)),
                tcp_nodelay: false,
+                client_tls: Some(ClientTlsOption {
+                    server_ca_cert_path: "some_server_path".to_string(),
+                    client_cert_path: "some_cert_path".to_string(),
+                    client_key_path: "some_key_path".to_string(),
+                }),
            },
            cfg
        );
@@ -496,7 +567,11 @@ mod tests {
            .http2_adaptive_window(true)
            .tcp_keepalive(Duration::from_secs(2))
            .tcp_nodelay(true);
-        let mgr = ChannelManager { pool, config };
+        let mgr = ChannelManager {
+            pool,
+            config,
+            client_tls_config: None,
+        };

        let res = mgr.build_endpoint("test_addr");

@@ -512,7 +587,11 @@ mod tests {
        let pool = Arc::new(pool);

        let config = ChannelConfig::new();
-        let mgr = ChannelManager { pool, config };
+        let mgr = ChannelManager {
+            pool,
+            config,
+            client_tls_config: None,
+        };

        let addr = "test_addr";
        let res = mgr.get(addr);
--- a/src/common/grpc/src/error.rs
+++ b/src/common/grpc/src/error.rs
@@ -13,6 +13,7 @@
 // limitations under the License.

 use std::any::Any;
+use std::io;

 use common_error::prelude::{ErrorExt, StatusCode};
 use snafu::{Backtrace, ErrorCompat, Snafu};
@@ -22,6 +23,15 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
+    #[snafu(display("Invalid client tls config, {}", msg))]
+    InvalidTlsConfig { msg: String },
+
+    #[snafu(display("Invalid config file path, {}", source))]
+    InvalidConfigFilePath {
+        source: io::Error,
+        backtrace: Backtrace,
+    },
+
    #[snafu(display("Missing required field in protobuf, field: {}", field))]
    MissingField { field: String, backtrace: Backtrace },

@@ -81,7 +91,9 @@ pub enum Error {
 impl ErrorExt for Error {
    fn status_code(&self) -> StatusCode {
        match self {
-            Error::MissingField { .. }
+            Error::InvalidTlsConfig { .. }
+            | Error::InvalidConfigFilePath { .. }
+            | Error::MissingField { .. }
            | Error::TypeMismatch { .. }
            | Error::InvalidFlightData { .. } => StatusCode::InvalidArguments,

--- a/src/common/grpc/src/flight.rs
+++ b/src/common/grpc/src/flight.rs
@@ -16,8 +16,9 @@ use std::collections::HashMap;
 use std::sync::Arc;

 use api::v1::{AffectedRows, FlightMetadata};
-use arrow_flight::utils::{flight_data_from_arrow_batch, flight_data_to_arrow_batch};
+use arrow_flight::utils::flight_data_to_arrow_batch;
 use arrow_flight::{FlightData, IpcMessage, SchemaAsIpc};
+use common_base::bytes::Bytes;
 use common_recordbatch::{RecordBatch, RecordBatches};
 use datatypes::arrow;
 use datatypes::arrow::datatypes::Schema as ArrowSchema;
@@ -39,38 +40,58 @@ pub enum FlightMessage {
    AffectedRows(usize),
 }

-#[derive(Default)]
 pub struct FlightEncoder {
    write_options: writer::IpcWriteOptions,
+    data_gen: writer::IpcDataGenerator,
+    dictionary_tracker: writer::DictionaryTracker,
+}
+
+impl Default for FlightEncoder {
+    fn default() -> Self {
+        Self {
+            write_options: writer::IpcWriteOptions::default(),
+            data_gen: writer::IpcDataGenerator::default(),
+            dictionary_tracker: writer::DictionaryTracker::new(false),
+        }
+    }
 }

 impl FlightEncoder {
-    pub fn encode(&self, flight_message: FlightMessage) -> FlightData {
+    pub fn encode(&mut self, flight_message: FlightMessage) -> FlightData {
        match flight_message {
            FlightMessage::Schema(schema) => {
                SchemaAsIpc::new(schema.arrow_schema(), &self.write_options).into()
            }
            FlightMessage::Recordbatch(recordbatch) => {
-                let (flight_dictionaries, flight_batch) = flight_data_from_arrow_batch(
-                    recordbatch.df_record_batch(),
-                    &self.write_options,
-                );
+                let (encoded_dictionaries, encoded_batch) = self
+                    .data_gen
+                    .encoded_batch(
+                        recordbatch.df_record_batch(),
+                        &mut self.dictionary_tracker,
+                        &self.write_options,
+                    )
+                    .expect("DictionaryTracker configured above to not fail on replacement");

                // TODO(LFC): Handle dictionary as FlightData here, when we supported Arrow's Dictionary DataType.
                // Currently we don't have a datatype corresponding to Arrow's Dictionary DataType,
                // so there won't be any "dictionaries" here. Assert to be sure about it, and
                // perform a "testing guard" in case we forgot to handle the possible "dictionaries"
                // here in the future.
-                debug_assert_eq!(flight_dictionaries.len(), 0);
+                debug_assert_eq!(encoded_dictionaries.len(), 0);

-                flight_batch
+                encoded_batch.into()
            }
            FlightMessage::AffectedRows(rows) => {
                let metadata = FlightMetadata {
                    affected_rows: Some(AffectedRows { value: rows as _ }),
                }
                .encode_to_vec();
-                FlightData::new(None, IpcMessage(build_none_flight_msg()), metadata, vec![])
+                FlightData::new(
+                    None,
+                    IpcMessage(build_none_flight_msg().into()),
+                    metadata,
+                    vec![],
+                )
            }
        }
    }
@@ -83,7 +104,8 @@ pub struct FlightDecoder {

 impl FlightDecoder {
    pub fn try_decode(&mut self, flight_data: FlightData) -> Result<FlightMessage> {
-        let message = root_as_message(flight_data.data_header.as_slice()).map_err(|e| {
+        let bytes = flight_data.data_header.slice(..);
+        let message = root_as_message(&bytes).map_err(|e| {
            InvalidFlightDataSnafu {
                reason: e.to_string(),
            }
@@ -91,7 +113,7 @@ impl FlightDecoder {
        })?;
        match message.header_type() {
            MessageHeader::NONE => {
-                let metadata = FlightMetadata::decode(flight_data.app_metadata.as_slice())
+                let metadata = FlightMetadata::decode(flight_data.app_metadata)
                    .context(DecodeFlightDataSnafu)?;
                if let Some(AffectedRows { value }) = metadata.affected_rows {
                    return Ok(FlightMessage::AffectedRows(value as _));
@@ -176,7 +198,7 @@ pub fn flight_messages_to_recordbatches(messages: Vec<FlightMessage>) -> Result<
    }
 }

-fn build_none_flight_msg() -> Vec<u8> {
+fn build_none_flight_msg() -> Bytes {
    let mut builder = FlatBufferBuilder::new();

    let mut message = arrow::ipc::MessageBuilder::new(&mut builder);
@@ -187,7 +209,7 @@ fn build_none_flight_msg() -> Vec<u8> {
    let data = message.finish();
    builder.finish(data, None);

-    builder.finished_data().to_vec()
+    builder.finished_data().into()
 }

 #[cfg(test)]
--- a/src/common/grpc/src/select.rs
+++ b/src/common/grpc/src/select.rs
@@ -67,7 +67,7 @@ macro_rules! convert_arrow_array_to_grpc_vals {
                    return Ok(vals);
                },
            )+
-            ConcreteDataType::Null(_) | ConcreteDataType::List(_) => unreachable!("Should not send {:?} in gRPC", $data_type),
+            ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => unreachable!("Should not send {:?} in gRPC", $data_type),
        }
    }};
 }
--- a/src/common/grpc/tests/mod.rs
+++ b/src/common/grpc/tests/mod.rs
@@ -0,0 +1,57 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_grpc::channel_manager::{ChannelConfig, ChannelManager, ClientTlsOption};
+
+#[tokio::test]
+async fn test_mtls_config() {
+    // test no config
+    let config = ChannelConfig::new();
+    let re = ChannelManager::with_tls_config(config);
+    assert!(re.is_err());
+
+    // test wrong file
+    let config = ChannelConfig::new().client_tls_config(ClientTlsOption {
+        server_ca_cert_path: "tests/tls/wrong_server.cert.pem".to_string(),
+        client_cert_path: "tests/tls/wrong_client.cert.pem".to_string(),
+        client_key_path: "tests/tls/wrong_client.key.pem".to_string(),
+    });
+
+    let re = ChannelManager::with_tls_config(config);
+    assert!(re.is_err());
+
+    // test corrupted file content
+    let config = ChannelConfig::new().client_tls_config(ClientTlsOption {
+        server_ca_cert_path: "tests/tls/server.cert.pem".to_string(),
+        client_cert_path: "tests/tls/client.cert.pem".to_string(),
+        client_key_path: "tests/tls/corrupted".to_string(),
+    });
+
+    let re = ChannelManager::with_tls_config(config);
+    assert!(re.is_ok());
+    let re = re.unwrap().get("127.0.0.1:0");
+    assert!(re.is_err());
+
+    // success
+    let config = ChannelConfig::new().client_tls_config(ClientTlsOption {
+        server_ca_cert_path: "tests/tls/server.cert.pem".to_string(),
+        client_cert_path: "tests/tls/client.cert.pem".to_string(),
+        client_key_path: "tests/tls/client.key.pem".to_string(),
+    });
+
+    let re = ChannelManager::with_tls_config(config);
+    assert!(re.is_ok());
+    let re = re.unwrap().get("127.0.0.1:0");
+    assert!(re.is_ok());
+}
--- a/src/common/grpc/tests/tls/client.cert.pem
+++ b/src/common/grpc/tests/tls/client.cert.pem
@@ -0,0 +1,36 @@
+-----BEGIN CERTIFICATE-----
+MIIGOzCCBCOgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBhzELMAkGA1UEBhMCSU4x
+EjAQBgNVBAgMCUthcm5hdGFrYTESMBAGA1UEBwwJQkFOR0FMT1JFMRUwEwYDVQQK
+DAxHb0xpbnV4Q2xvdWQxEjAQBgNVBAMMCWNhLXNlcnZlcjElMCMGCSqGSIb3DQEJ
+ARYWYWRtaW5AZ29saW51eGNsb3VkLmNvbTAeFw0yMzAyMTQxMTM4MDFaFw0yNzA4
+MjIxMTM4MDFaMHIxCzAJBgNVBAYTAklOMRIwEAYDVQQIDAlLYXJuYXRha2ExFTAT
+BgNVBAoMDEdvTGludXhDbG91ZDERMA8GA1UEAwwIc2VydmVyLTIxJTAjBgkqhkiG
+9w0BCQEWFmFkbWluQGdvbGludXhjbG91ZC5jb20wggIiMA0GCSqGSIb3DQEBAQUA
+A4ICDwAwggIKAoICAQDNPiXZFK1cDOevdU5628xqAZjHn2e86hD9ih0IHvQKbcAm
+a8fhFMQ+Gki+p2+Ga1fxHDi1+aUn00UjyLAxSMQVulpZWYHsRj3koyD9LyTvpDQk
+SwJhFNtL33WlqUMtjgVXoznjECfhc/hwKJ9BS0b5j21XzqYkSKTJNcxZmoNLJVvL
+dfbsWjLywSAHbcF1gs2w3IxruPQwyMXL1URjcwGRTtK+zk6QGxgyXsIEJDW4EZqR
+xXgmEz7jx7vfDLaYc8GoujTki2dkyTWQkdDrJ4/N7VWGOGjL60EJDOcQyCowDuAq
+sbB5C9OuhB59o2/wzeSeaY7qS5nLOufwiYmvc1S6kgi9emirxqFLmrcaJv8QPDEX
+6ufI8wSkCS/CX/IUNXPkSripU3zQcjorinAw3w9pGY1VNknz5AgDXrEAW17aZKsp
+QyLSyl87vG9dhjybdkc7QyBghTxweggYT1INY6dmj9ijIyU+9V64xOTb9dlbgLW/
+qAvZyeq2H9Z5aBwkG31n1b2rX0JEK+/NC+8PRs2tWq63EOB8hzh4mF9RKLcZC3zS
+9eJa1B0ugyy5fw8GGWA49H3rFoU2u7+Gazzdn5uD9sqLuVnzW1FREDhMHGd4VdRx
+vuhUp9jz9u0WDRr2Ix7N7Vd57mwhBPivUywg7QwZSTqlIrGVoQFPL4BjWwSSswID
+AQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMDMGCWCGSAGG
+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQgQ2VydGlmaWNhdGUwHQYD
+VR0OBBYEFI056bMc2jHoeOTUGBCpBGGY/UfQMB8GA1UdIwQYMBaAFKVZwpSJCPkN
+wGXyJX1sl2Pbby4FMA4GA1UdDwEB/wQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcD
+AgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBABHQ/EGnAFeIdzKTbaP3kaSd
+A3tCyjWVwo9eULXBjsMFFyf4NDw8bkrYdJos6rBpzi6R1PUb4UMc9CUF6ee9zbTK
+mDeusqwhDOLmYZot1aZbujMngpbMoQx5keSQ9Eg10npbYMl6Sq3qFbAST9l/hlDh
+Ue9KhfrAvrSobP0WWb/EpEXZMt2DafKpoz4nvtFpcOO5kbsQ+/eQfWHmR/k6sCYG
+UycFYCJCFQz2xG8wtbExg5iyaR3nE0LfqZwRxhIa4iSWlCecYc1XUJnOh8fIeop4
+9fD5k2wqvCEBAZiaKg2RYbaw6LIFkg7c99B4Gt5eez7Bs878T7lS+xl9wbzinzez
+WFIgsDYHYjmK8s5WXXWwT7UhqSA12FHOp8grqFllXV/dOPTFz+dq9Mn1VGgH6MS4
+Ls3r2LH5ycAz+gkoY2wlnF++ItpB2K3LTlqk+OvQZ1oXMq8u5F6XsM7Uirc7Da+9
+MEG1zBpGvA/iAd2kKd3APS+EuoytSt022bD7YDJ1isuxT5q2Hpa4p14BJHCgDKTZ
+vPYIdzCh05vwLwB28T8bh7s5OLOcRY9KmxVPkT0SYLOk11j5nZ1N/hQvGDxL60e2
+RBS3ADHkymIE55Xf1VLXcs17zR9fLV+5fiSQ40FLjcBEjhkvrzcDe3tVFsA/ty9h
+dBCSsexiXj/S5KwKtz/c
+-----END CERTIFICATE-----
--- a/src/common/grpc/tests/tls/client.key.pem
+++ b/src/common/grpc/tests/tls/client.key.pem
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEAzT4l2RStXAznr3VOetvMagGYx59nvOoQ/YodCB70Cm3AJmvH
+4RTEPhpIvqdvhmtX8Rw4tfmlJ9NFI8iwMUjEFbpaWVmB7EY95KMg/S8k76Q0JEsC
+YRTbS991palDLY4FV6M54xAn4XP4cCifQUtG+Y9tV86mJEikyTXMWZqDSyVby3X2
+7Foy8sEgB23BdYLNsNyMa7j0MMjFy9VEY3MBkU7Svs5OkBsYMl7CBCQ1uBGakcV4
+JhM+48e73wy2mHPBqLo05ItnZMk1kJHQ6yePze1Vhjhoy+tBCQznEMgqMA7gKrGw
+eQvTroQefaNv8M3knmmO6kuZyzrn8ImJr3NUupIIvXpoq8ahS5q3Gib/EDwxF+rn
+yPMEpAkvwl/yFDVz5Eq4qVN80HI6K4pwMN8PaRmNVTZJ8+QIA16xAFte2mSrKUMi
+0spfO7xvXYY8m3ZHO0MgYIU8cHoIGE9SDWOnZo/YoyMlPvVeuMTk2/XZW4C1v6gL
+2cnqth/WeWgcJBt9Z9W9q19CRCvvzQvvD0bNrVqutxDgfIc4eJhfUSi3GQt80vXi
+WtQdLoMsuX8PBhlgOPR96xaFNru/hms83Z+bg/bKi7lZ81tRURA4TBxneFXUcb7o
+VKfY8/btFg0a9iMeze1Xee5sIQT4r1MsIO0MGUk6pSKxlaEBTy+AY1sEkrMCAwEA
+AQKCAgEAw2jBZj5+k96hk/dPIkA1DlS43o7RmRcN2CdwXrQBzBAUW0BRDObVtP8X
+dZY647M+BozFHdUzPoizEk/YGQRb1QgZT2qd/ZQfB5mdJhGFzDf9gPR9rmrKJCH8
+hB50nGHUik0ZJyvRnKDqz/aNMgB28dJx26Efo/oaEoyLJGCtUpWeIUgOMZfrXB8t
+3ITOJZDFP/esJj/xFqWBVQGXXEw6GNwAYLRSLnftgL+hX4oOL1NrZBCrxSybuwkG
+wWX8T4gewQOQqmxjo5zCyANc8xc2nmyx+dmpRUWWJQTI1ryNFjaDjYKiL41oHIcj
+9KDwSkftvDlqXX5fThSmkeiRU5+t8UMj4+Bt7opCzIlwHtQe+95BqiXQ7bHfCjn7
+GvShZgHo45rDkfwWDz/pYhHQ2Wb9DkhEtwa0cu3mDMGc6BY+4yo+Vz6Rk1TypxQw
+LIa43WgVCRm66Mq65sObx7wkdxvolUE8j1Io3AHwgeBjV+gISV9srj2m/HnOmFFb
+16SKQEDEVoaci+v6DT8A7UOZH4sgYSbknHdjMy6c6UlYgd8UNqbY3h/ohZ2JOcPd
+8DqGUDGKbpS7OxWogxb9K++6SPSn86sPmUjzRPMgijVjU5pyK42DpZj1/RIe8Tml
+JXVqHuZvURK4Qi3ECQ09m9vQ9nS88HMRVJ7sFSca6HOFYSFyAfkCggEBAPN35hva
+OhbgQlFJrpo5YDYS5v7l7YjLbry6DaCR1CpYaKlTPkc4tiznCUHe1N41mR4qu2Tc
+4+m7GN9BZfLU8w/Jvrp7mAO7fZXZtIzTQrZQDbAZppUGBbGBoAOlLVxR4NrN2TSk
+49Ljj87UynhxeCv6RWx0F1p1/VIZertLELbSdb3C43pAsNSXzbkb7LtT9RXemyUL
+LBK4ugcXMSZrzHJK1Ct31LoGd9m+TEp/VW2aGMeWliuIticJx44OW4tlJ70qKrd0
+KezBZVMHPa3FqW7kdYwdlISoqZsE9OVPgLCQNVLhDO1YMaTl3WEHKTRBxTF70pvv
+zMkSRQGoU4ff7AUCggEBANfOkCsx2mRvJV+UYxW8R6510w2H1bNbNRbfTJAo8kld
+/7dXU4H3QrhUrCsSyc5ijm09q7I4+rc+uMxfT/R1mO5tq9AueCWhg85WV+NBR1FE
+Yg7MX+zblpHqUDQoTj9vvgwLyqvZ7k9NON42Zz+Tj2worICnVlDvahm/3NaItT9B
+oGhsEoJjYFK4Hq7RwosU+KPXkQBxWzrNLipo8jx0XFpPZVHSLIFs9eW25bnj/qxc
+toMgx4IsvEDlzS/oqfycCrDdKwqiW74w0Djb5TiJv+dYzl9GnN6istqbUTNZkJjn
+lkbmegrtfz3Yd1ORvjkNqHuANyuR+YnUSIsb0PV5eVcCggEAck5bgb4eQbk+SY3P
+ZOcFLb4IJ6ppsCzaq86qMTXmJ49kbAMCHUwZ89DwvrVQuZbucYRcgMlYU9ccoUzC
+AZVLHKF6Y3E9eJshJiaVJvzUuGWzV3djh1nReHpEVxHIzyw95lx42seDkvJ2BQRQ
+nuWfJv6Uc4u5nyYALfh6b86ZZUxALTx/slkG7HjtBDiBF54eVgsySd0J7yw9YrDX
+yZMY5JwPKu1SuZfp0xgOF3fa8t9DPQmNLZk88+0afK5u+m4ejyhp78GhIV/XI3kl
+0x0XJFIsggEtRm8tWfOkyrhd0geSkXvJpvEeNa4aFsDW7ormewoIYl/ehJSIQ3P0
+67kMxQKCAQA12iP7w2r+GQY4fazkJaG1lU1fWQAoy5/J31sZtj4PtNc1ByOdkPgj
+S23TKdMWH13vQK5xwOo/g/VVeotXM2lARjnTr2Tn7xAXE1DHMuj7DJdznehqELnY
+G6J8AXrVNas1ElQ24iEnxNtmCClnogjuMpApYpiVhcjyOACBwIeKC3Rd2mocA3Rr
+7+ooMcvcLRWGvSo/9AmR+NWGW73m/Bp3psxfyJS2j1wlQKi+5HgOxuv8eNeQUl1/
+zFiRlfulP8MjM22kL7O5GDE9nxHqM+Whc3W8LMDEhdEf4BY5PCZrIY9MjgLyayWP
+Z08PmZTgY9ohR3N8+eZNUJ3xqLVSLEftAoIBAQDF1K8lPXAs8e4V0oc9hq4GFLvi
+E0KC+8X1ShzvkVGV/3Kz1FJ0bwix/M3C5XSSNguxHI6CG2GprJlExp1qqwlvmGr2
+hHdfemvq6tF4qjXLgPXvgoWocBGNUvBXxFVuc0hOHgT/X3+GsPYtNvZb3fp+4Bm6
+ugUu05drqrHSOY5kUbU3jf/5KctnDFmOsSeOgGiI/JJWVcKJALpDkhazRL0nxfuW
+6xU6pZazhCAby2Qn+wn0xyi4bEZSNobiQTgOXOC0DA1uGD3XHctCMnSBtYtocQjq
+IFT2l3u4pEKpVQwuc4+yObWUT47oBxV6vFneXsnV89vd2SSUPuR8GIYYeA+/
+-----END RSA PRIVATE KEY-----
--- a/src/common/grpc/tests/tls/corrupted
+++ b/src/common/grpc/tests/tls/corrupted
@@ -0,0 +1,50 @@
+rWtZ7U3SoVAl6yMhfJsB
+LcEGbuCfgFxk2ADw0N1G
+byTKlrUgoRZeSc0cYHTf
+0XjbRCBtMV9yYaVJKPwi
+rGofQgFoc1lW0U5x2bnN
+O9nn9aDe5t5LAlGS81uX
+aBMvuzVjHbZKOlabXl4W
+ZJc06qngAcQWQUu8nAnR
+FLsjhoaTyuaDMY3OWJAx
+5Dt7YglND5uFAqYwRG9L
+agLGOCH8suwnXGYaPxjM
+Ysb5RANkpgcbSulLZiic
+4sLmpJomjokwZbctODVW
+pCLiQT3wWDJ7YjIePR6g
+P3Jlg0LDhbgSwXxgjjUR
+6qGRfcb8LFlVlT7O1ze2
+lFBNWzijkPeKyKmwpOSa
+oGCR2OUg71n0Tzt2a3ir
+WLijq0bL1Cetz24fv738
+L3MEAwezFBW38U4QilNz
+uza1bC3PgToermGSgKLx
+WMdgjZIszK4t6Rehelx8
+YpCJWVXTob3Gn4bMwWJO
+xpJ9qhvMBdD8iamheF4b
+bUm1YmHW4gPT1ujiqCmN
+I7hOFurjJ6zvXGETyfCn
+w23W8PNFWbqpHUKN59Bz
+HpbsIRDVVpEGxnoWmdjq
+58BUOxDdbTZxCKt0UqLD
+uUPOlW8bRhuC1tK1NL5u
+wq9ybcfwZ4jIHyYlHZ5M
+4t4zKLRG2DN6icHmctOW
+TzYp3np0OFsTlzCwkogM
+Os6SOvjU0Irq2Xo5wLvn
+1nN6FQwUxcw0H5rfQEZo
+NioHP0JdBv3HmIaQZs1n
+8lJWLVof1TBWtRUKmWmO
+79DcTURdzt28Vdn6F0K0
+UiG15bda4Pb81I9IE9ug
+iZkC7CE98aE6WQK9Ghlu
+dNXJTkUD3uVg6Tqi3957
+Hfa9xMclyrxsOvkGcudI
+QbcvG5Apom6nBWIGHRMQ
+68rn9eZEcq5mJLaiNmHr
+5AOtHddC5NVgQLgdmmKb
+gQlrcSXzxT6V6jzbxZ79
+xmulvmkeqG4kj6TAuJEg
+u9dCkExxv5tLSpF8hC08
+HHU4QE56UC97djO5EpmK
+g3rElyboRHlAYPWviWbm
--- a/src/common/grpc/tests/tls/server.cert.pem
+++ b/src/common/grpc/tests/tls/server.cert.pem
@@ -0,0 +1,40 @@
+-----BEGIN CERTIFICATE-----
+MIIG+jCCBOKgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCBhzELMAkGA1UEBhMCSU4x
+EjAQBgNVBAgMCUthcm5hdGFrYTESMBAGA1UEBwwJQkFOR0FMT1JFMRUwEwYDVQQK
+DAxHb0xpbnV4Q2xvdWQxEjAQBgNVBAMMCWNhLXNlcnZlcjElMCMGCSqGSIb3DQEJ
+ARYWYWRtaW5AZ29saW51eGNsb3VkLmNvbTAeFw0yMzAyMTQxMTM5NDBaFw0yNzA4
+MjIxMTM5NDBaMHAxCzAJBgNVBAYTAklOMRIwEAYDVQQIDAlLYXJuYXRha2ExFTAT
+BgNVBAoMDEdvTGludXhDbG91ZDEPMA0GA1UEAwwGc2VydmVyMSUwIwYJKoZIhvcN
+AQkBFhZhZG1pbkBnb2xpbnV4Y2xvdWQuY29tMIICIjANBgkqhkiG9w0BAQEFAAOC
+Ag8AMIICCgKCAgEAvVtxAoRjLRs3Ei4+CgzqJ2+bpc0sBdUm/4LM/D+0KbXxwD7w
+HP6GcKl/9zf9GJg56pVXxXMaerMDLS4Est25+mBgqcePC6utCBYrKA25pKbkFkxZ
+TPh9/R4RHGVJ3KHy9vc4VzqoV7XFMJFFUQ2fQywHZlXh6MNz0WPTIGaH7hvYoHbK
+I3NpPq8TjRuuV61XB0hK+RW0K6/5Yuj74h/mfheX1VIUOjGwKnTPccZQAlrKYjeW
+BZBS4YqahkTIaGLa06SdUSkuhL85rqAxWvhK9GIRlQLNYJOzg+E3jGyqf566xX60
+fxM6alLYf+ZzCwSBuDDj5f+j752gPLYUI82YL4xQ+AEHNR8U1uMvt0EzzFt7mSRe
+fobVr+Y2zpci+mo7kcQGOhenzGclsm+qXwMhYUnJcOYFZWtTJlFaaPreL4M3Dh+2
+pmKj23ZU6zcT3MYtE6phjCLJl0DsFIcOn+tSqMdpwB20EeQjo9bVJuw/HJrlpcnY
+U9aLsnm/4Ls5A0BQutZnxKBIJjpzp8VfK0WU8a4iKok3AS0z1/K+atNrgSUB9DCH
+0MvLqqQmM9TdLcZj7NSEfLyyFVwPRc5dt4CrNDL7JUpMzt36ezU83JU+nfqWDZsL
+2JOaE4gGLZDcA3cfP83/mYRaAnYW/9W4vEnIpa6subzq1aFOeY/3dKLTx8CAwEA
+AaOCAYUwggGBMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG
+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYD
+VR0OBBYEFLijeA+RFDQtuVeMUkaXqF7LF50GMIG8BgNVHSMEgbQwgbGAFKVZwpSJ
+CPkNwGXyJX1sl2Pbby4FoYGNpIGKMIGHMQswCQYDVQQGEwJJTjESMBAGA1UECAwJ
+S2FybmF0YWthMRIwEAYDVQQHDAlCQU5HQUxPUkUxFTATBgNVBAoMDEdvTGludXhD
+bG91ZDESMBAGA1UEAwwJY2Etc2VydmVyMSUwIwYJKoZIhvcNAQkBFhZhZG1pbkBn
+b2xpbnV4Y2xvdWQuY29tggkA7NvbvF8jodEwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud
+JQQMMAoGCCsGAQUFBwMBMCkGA1UdEQQiMCCHBMCoAHKHBAoAAg+CEnNlcnZlci5l
+eGFtcGxlLmNvbTANBgkqhkiG9w0BAQsFAAOCAgEAXvaS9+y5g2Kw/4EPsnhjpN1v
+CxXW0+UYSWOaxVJdEAjGQI/1m9LOiF9IHImmiwluJ/Bex1TzuaTCKmpluPwGvd9D
+Zgf0A5SmVqW4WTT4d2nSecxw4OICJ3j6ubKkvMVf9s+ZJwb+fMMUaSt80bWqp1TY
+XbZguv67PkBECPqVe6rgzXnTLwM3lE8EgG8VtM3IOy9a5SIEjm5L8SQ2I2hiytmE
+e4jR1fbZsB5NbBdfA3GFMKQEE2dIymkG3Bz71M3tZi1y4RnHtRKdrFtrIlgclrwd
+nVnQn/NiXUOOzsL2+vwSF32SSbiLvOxu63qO1YDBkKVChog3P/2f6xcJ23wkbHlL
+qaL2jvLo6ylvMPUYHf5ZWat5zayaGUMHYDKcbD4Dw7aY3M0tNgEHdqUqNePmKvmn
+luyXof3KmmLgWlcfBoX96a7hXDtxFyB2N4nzfQBXh+0VAlgqa+ZZhpdEqRQaWkkR
+MDBdsVJ9O3812IaNfMzpS1vb701GFDCM5Hcyw6a/v6Ln08NMhYut4saLi13kHilS
+Wq7wOAfW3rzxuhjOJJxsi0jJNI775q+a/BbbG/CPl826bXPGH43BdPV8mKwsX5HM
+wwDKf3otP/v7bxwJabfhv2EKUy+W1kkFW9FEZ919yTtfhSDrTNcrXtE7RkiAepfm
+95I025URIlhJGLGBUlA=
+-----END CERTIFICATE-----
--- a/src/common/mem-prof/Cargo.toml
+++ b/src/common/mem-prof/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "common-mem-prof"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+common-error = { path = "../error" }
+snafu.workspace = true
+tempfile = "3.4"
+tikv-jemalloc-ctl = { version = "0.5", features = ["use_std"] }
+tikv-jemallocator = "0.5"
+tokio.workspace = true
+
+[dependencies.tikv-jemalloc-sys]
+version = "0.5"
+features = ["stats", "profiling", "unprefixed_malloc_on_supported_platforms"]
+
+[profile.release]
+debug = true
--- a/src/common/mem-prof/README.md
+++ b/src/common/mem-prof/README.md
@@ -0,0 +1,50 @@
+# Profile memory usage of GreptimeDB
+
+This crate provides an easy approach to dump memory profiling info.
+
+## Prerequisites
+### jemalloc
+```bash
+# for macOS
+brew install jemalloc
+
+# for Ubuntu
+sudo apt install libjemalloc-dev
+```
+
+### [flamegraph](https://github.com/brendangregg/FlameGraph) 
+
+```bash
+curl https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl > ./flamegraph.pl 
+```
+
+### Build GreptimeDB with `mem-prof` feature.
+
+```bash
+cargo build --features=mem-prof
+```
+
+## Profiling
+
+Start GreptimeDB instance with environment variables:
+
+```bash
+MALLOC_CONF=prof:true,lg_prof_interval:28 ./target/debug/greptime standalone start
+```
+
+Dump memory profiling data through HTTP API:
+
+```bash
+curl localhost:4000/v1/prof/mem > greptime.hprof
+```
+
+You can periodically dump profiling data and compare them to find the delta memory usage.
+
+## Analyze profiling data with flamegraph
+
+To create flamegraph according to dumped profiling data:
+
+```bash
+jeprof --svg <path_to_greptimedb_binary> --base=<baseline_prof> <profile_data> > output.svg
+```
+
--- a/src/common/mem-prof/src/error.rs
+++ b/src/common/mem-prof/src/error.rs
@@ -0,0 +1,66 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::path::PathBuf;
+
+use common_error::prelude::{ErrorExt, StatusCode};
+use snafu::{Backtrace, Snafu};
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub))]
+pub enum Error {
+    #[snafu(display("Failed to read OPT_PROF"))]
+    ReadOptProf { source: tikv_jemalloc_ctl::Error },
+
+    #[snafu(display("Memory profiling is not enabled"))]
+    ProfilingNotEnabled,
+
+    #[snafu(display("Failed to build temp file from given path: {:?}", path))]
+    BuildTempPath { path: PathBuf, backtrace: Backtrace },
+
+    #[snafu(display("Failed to open temp file: {}", path))]
+    OpenTempFile {
+        path: String,
+        source: std::io::Error,
+    },
+
+    #[snafu(display("Failed to dump profiling data to temp file: {:?}", path))]
+    DumpProfileData {
+        path: PathBuf,
+        source: tikv_jemalloc_ctl::Error,
+    },
+}
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        match self {
+            Error::ReadOptProf { .. } => StatusCode::Internal,
+            Error::ProfilingNotEnabled => StatusCode::InvalidArguments,
+            Error::BuildTempPath { .. } => StatusCode::Internal,
+            Error::OpenTempFile { .. } => StatusCode::StorageUnavailable,
+            Error::DumpProfileData { .. } => StatusCode::StorageUnavailable,
+        }
+    }
+
+    fn backtrace_opt(&self) -> Option<&Backtrace> {
+        snafu::ErrorCompat::backtrace(self)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
--- a/src/common/mem-prof/src/lib.rs
+++ b/src/common/mem-prof/src/lib.rs
@@ -0,0 +1,74 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod error;
+
+use std::ffi::{c_char, CString};
+use std::path::PathBuf;
+
+use snafu::{ensure, ResultExt};
+use tokio::io::AsyncReadExt;
+
+use crate::error::{
+    BuildTempPathSnafu, DumpProfileDataSnafu, OpenTempFileSnafu, ProfilingNotEnabledSnafu,
+    ReadOptProfSnafu,
+};
+
+const PROF_DUMP: &[u8] = b"prof.dump\0";
+const OPT_PROF: &[u8] = b"opt.prof\0";
+
+pub async fn dump_profile() -> error::Result<Vec<u8>> {
+    ensure!(is_prof_enabled()?, ProfilingNotEnabledSnafu);
+    let tmp_path = tempfile::tempdir().map_err(|_| {
+        BuildTempPathSnafu {
+            path: std::env::temp_dir(),
+        }
+        .build()
+    })?;
+
+    let mut path_buf = PathBuf::from(tmp_path.path());
+    path_buf.push("greptimedb.hprof");
+
+    let path = path_buf
+        .to_str()
+        .ok_or_else(|| BuildTempPathSnafu { path: &path_buf }.build())?
+        .to_string();
+
+    let mut bytes = CString::new(path.as_str())
+        .map_err(|_| BuildTempPathSnafu { path: &path_buf }.build())?
+        .into_bytes_with_nul();
+
+    {
+        // #safety: we always expect a valid temp file path to write profiling data to.
+        let ptr = bytes.as_mut_ptr() as *mut c_char;
+        unsafe {
+            tikv_jemalloc_ctl::raw::write(PROF_DUMP, ptr)
+                .context(DumpProfileDataSnafu { path: path_buf })?
+        }
+    }
+
+    let mut f = tokio::fs::File::open(path.as_str())
+        .await
+        .context(OpenTempFileSnafu { path: &path })?;
+    let mut buf = vec![];
+    f.read_to_end(&mut buf)
+        .await
+        .context(OpenTempFileSnafu { path })?;
+    Ok(buf)
+}
+
+fn is_prof_enabled() -> error::Result<bool> {
+    // safety: OPT_PROF variable, if present, is always a boolean value.
+    Ok(unsafe { tikv_jemalloc_ctl::raw::read::<bool>(OPT_PROF).context(ReadOptProfSnafu)? })
+}
--- a/src/common/procedure/Cargo.toml
+++ b/src/common/procedure/Cargo.toml
@@ -13,10 +13,11 @@ futures.workspace = true
 object-store = { path = "../../object-store" }
 serde.workspace = true
 serde_json = "1.0"
+smallvec = "1"
 snafu.workspace = true
 tokio.workspace = true
 uuid.workspace = true

 [dev-dependencies]
+common-test-util = { path = "../test-util" }
 futures-util.workspace = true
-tempdir = "0.3"
--- a/src/common/procedure/src/error.rs
+++ b/src/common/procedure/src/error.rs
@@ -13,6 +13,7 @@
 // limitations under the License.

 use std::any::Any;
+use std::sync::Arc;

 use common_error::prelude::*;

@@ -69,6 +70,33 @@ pub enum Error {
        key: String,
        source: object_store::Error,
    },
+
+    #[snafu(display("Failed to deserialize from json, source: {}", source))]
+    FromJson {
+        source: serde_json::Error,
+        backtrace: Backtrace,
+    },
+
+    #[snafu(display("Procedure exec failed, source: {}", source))]
+    RetryLater {
+        #[snafu(backtrace)]
+        source: BoxedError,
+    },
+
+    #[snafu(display("Procedure panics, procedure_id: {}", procedure_id))]
+    ProcedurePanic { procedure_id: ProcedureId },
+
+    #[snafu(display("Failed to wait watcher, source: {}", source))]
+    WaitWatcher {
+        source: tokio::sync::watch::error::RecvError,
+        backtrace: Backtrace,
+    },
+
+    #[snafu(display("Failed to execute procedure, source: {}", source))]
+    ProcedureExec {
+        source: Arc<Error>,
+        backtrace: Backtrace,
+    },
 }

 pub type Result<T> = std::result::Result<T, Error>;
@@ -81,10 +109,15 @@ impl ErrorExt for Error {
            | Error::PutState { .. }
            | Error::DeleteState { .. }
            | Error::ListState { .. }
-            | Error::ReadState { .. } => StatusCode::Internal,
+            | Error::ReadState { .. }
+            | Error::FromJson { .. }
+            | Error::RetryLater { .. }
+            | Error::WaitWatcher { .. } => StatusCode::Internal,
            Error::LoaderConflict { .. } | Error::DuplicateProcedure { .. } => {
                StatusCode::InvalidArguments
            }
+            Error::ProcedurePanic { .. } => StatusCode::Unexpected,
+            Error::ProcedureExec { source, .. } => source.status_code(),
        }
    }

@@ -104,4 +137,26 @@ impl Error {
            source: BoxedError::new(err),
        }
    }
+
+    /// Creates a new [Error::RetryLater] error from source `err`.
+    pub fn retry_later<E: ErrorExt + Send + Sync + 'static>(err: E) -> Error {
+        Error::RetryLater {
+            source: BoxedError::new(err),
+        }
+    }
+
+    /// Determine whether it is a retry later type through [StatusCode]
+    pub fn is_retry_later(&self) -> bool {
+        matches!(self, Error::RetryLater { .. })
+    }
+
+    /// Creates a new [Error::RetryLater] or [Error::External] error from source `err` according
+    /// to its [StatusCode].
+    pub fn from_error_ext<E: ErrorExt + Send + Sync + 'static>(err: E) -> Self {
+        if err.status_code().is_retryable() {
+            Error::retry_later(err)
+        } else {
+            Error::external(err)
+        }
+    }
 }
--- a/src/common/procedure/src/lib.rs
+++ b/src/common/procedure/src/lib.rs
@@ -15,15 +15,14 @@
 //! Common traits and structures for the procedure framework.

 pub mod error;
-#[allow(dead_code)]
-mod local;
+pub mod local;
 mod procedure;
-// TODO(yingwen): Remove this attribute once ProcedureManager is implemented.
-#[allow(dead_code)]
 mod store;
+pub mod watcher;

 pub use crate::error::{Error, Result};
 pub use crate::procedure::{
-    BoxedProcedure, Context, LockKey, Procedure, ProcedureId, ProcedureManager,
+    BoxedProcedure, Context, ContextProvider, LockKey, Procedure, ProcedureId, ProcedureManager,
    ProcedureManagerRef, ProcedureState, ProcedureWithId, Status,
 };
+pub use crate::watcher::Watcher;
--- a/src/common/procedure/src/local.rs
+++ b/src/common/procedure/src/local.rs
@@ -22,6 +22,7 @@ use async_trait::async_trait;
 use common_telemetry::logging;
 use object_store::ObjectStore;
 use snafu::ensure;
+use tokio::sync::watch::{self, Receiver, Sender};
 use tokio::sync::Notify;

 use crate::error::{DuplicateProcedureSnafu, LoaderConflictSnafu, Result};
@@ -30,27 +31,10 @@ use crate::local::runner::Runner;
 use crate::procedure::BoxedProcedureLoader;
 use crate::store::{ObjectStateStore, ProcedureMessage, ProcedureStore, StateStoreRef};
 use crate::{
-    BoxedProcedure, LockKey, ProcedureId, ProcedureManager, ProcedureState, ProcedureWithId,
+    BoxedProcedure, ContextProvider, LockKey, ProcedureId, ProcedureManager, ProcedureState,
+    ProcedureWithId, Watcher,
 };

-/// Mutable metadata of a procedure during execution.
-#[derive(Debug)]
-struct ExecMeta {
-    /// Current procedure state.
-    state: ProcedureState,
-    /// Id of child procedures.
-    children: Vec<ProcedureId>,
-}
-
-impl Default for ExecMeta {
-    fn default() -> ExecMeta {
-        ExecMeta {
-            state: ProcedureState::Running,
-            children: Vec::new(),
-        }
-    }
-}
-
 /// Shared metadata of a procedure.
 ///
 /// # Note
@@ -70,44 +54,62 @@ pub(crate) struct ProcedureMeta {
    /// Notify to wait for subprocedures.
    child_notify: Notify,
    /// Lock required by this procedure.
-    lock_key: Option<LockKey>,
-    /// Mutable status during execution.
-    exec_meta: Mutex<ExecMeta>,
+    lock_key: LockKey,
+    /// Sender to notify the procedure state.
+    state_sender: Sender<ProcedureState>,
+    /// Receiver to watch the procedure state.
+    state_receiver: Receiver<ProcedureState>,
+    /// Id of child procedures.
+    children: Mutex<Vec<ProcedureId>>,
 }

 impl ProcedureMeta {
+    fn new(id: ProcedureId, parent_id: Option<ProcedureId>, lock_key: LockKey) -> ProcedureMeta {
+        let (state_sender, state_receiver) = watch::channel(ProcedureState::Running);
+        ProcedureMeta {
+            id,
+            lock_notify: Notify::new(),
+            parent_id,
+            child_notify: Notify::new(),
+            lock_key,
+            state_sender,
+            state_receiver,
+            children: Mutex::new(Vec::new()),
+        }
+    }
+
    /// Returns current [ProcedureState].
    fn state(&self) -> ProcedureState {
-        let meta = self.exec_meta.lock().unwrap();
-        meta.state.clone()
+        self.state_receiver.borrow().clone()
    }

    /// Update current [ProcedureState].
    fn set_state(&self, state: ProcedureState) {
-        let mut meta = self.exec_meta.lock().unwrap();
-        meta.state = state;
+        // Safety: ProcedureMeta also holds the receiver, so `send()` should never fail.
+        self.state_sender.send(state).unwrap();
    }

    /// Push `procedure_id` of the subprocedure to the metadata.
    fn push_child(&self, procedure_id: ProcedureId) {
-        let mut meta = self.exec_meta.lock().unwrap();
-        meta.children.push(procedure_id);
+        let mut children = self.children.lock().unwrap();
+        children.push(procedure_id);
    }

    /// Append subprocedures to given `buffer`.
    fn list_children(&self, buffer: &mut Vec<ProcedureId>) {
-        let meta = self.exec_meta.lock().unwrap();
-        buffer.extend_from_slice(&meta.children);
+        let children = self.children.lock().unwrap();
+        buffer.extend_from_slice(&children);
    }

    /// Returns the number of subprocedures.
    fn num_children(&self) -> usize {
-        self.exec_meta.lock().unwrap().children.len()
+        self.children.lock().unwrap().len()
    }
 }

 /// Reference counted pointer to [ProcedureMeta].
 type ProcedureMetaRef = Arc<ProcedureMeta>;
+
 /// Procedure loaded from store.
 struct LoadedProcedure {
    procedure: BoxedProcedure,
@@ -121,12 +123,17 @@ pub(crate) struct ManagerContext {
    loaders: Mutex<HashMap<String, BoxedProcedureLoader>>,
    lock_map: LockMap,
    procedures: RwLock<HashMap<ProcedureId, ProcedureMetaRef>>,
-    // TODO(yingwen): Now we never clean the messages. But when the root procedure is done, we
-    // should be able to remove the its message and all its child messages.
    /// Messages loaded from the procedure store.
    messages: Mutex<HashMap<ProcedureId, ProcedureMessage>>,
 }

+#[async_trait]
+impl ContextProvider for ManagerContext {
+    async fn procedure_state(&self, procedure_id: ProcedureId) -> Result<Option<ProcedureState>> {
+        Ok(self.state(procedure_id))
+    }
+}
+
 impl ManagerContext {
    /// Returns a new [ManagerContext].
    fn new() -> ManagerContext {
@@ -166,6 +173,14 @@ impl ManagerContext {
        procedures.get(&procedure_id).map(|meta| meta.state())
    }

+    /// Returns the [Watcher] of specific `procedure_id`.
+    fn watcher(&self, procedure_id: ProcedureId) -> Option<Watcher> {
+        let procedures = self.procedures.read().unwrap();
+        procedures
+            .get(&procedure_id)
+            .map(|meta| meta.state_receiver.clone())
+    }
+
    /// Notify a suspended parent procedure with specific `procedure_id` by its subprocedure.
    fn notify_by_subprocedure(&self, procedure_id: ProcedureId) {
        let procedures = self.procedures.read().unwrap();
@@ -176,9 +191,20 @@ impl ManagerContext {

    /// Load procedure with specific `procedure_id` from cached [ProcedureMessage]s.
    fn load_one_procedure(&self, procedure_id: ProcedureId) -> Option<LoadedProcedure> {
-        let messages = self.messages.lock().unwrap();
-        let message = messages.get(&procedure_id)?;
+        let message = {
+            let messages = self.messages.lock().unwrap();
+            messages.get(&procedure_id).cloned()?
+        };

+        self.load_one_procedure_from_message(procedure_id, &message)
+    }
+
+    /// Load procedure from specific [ProcedureMessage].
+    fn load_one_procedure_from_message(
+        &self,
+        procedure_id: ProcedureId,
+        message: &ProcedureMessage,
+    ) -> Option<LoadedProcedure> {
        let loaders = self.loaders.lock().unwrap();
        let loader = loaders.get(&message.type_name).or_else(|| {
            logging::error!(
@@ -264,7 +290,7 @@ impl ManagerContext {
 #[derive(Debug)]
 pub struct ManagerConfig {
    /// Object store
-    object_store: ObjectStore,
+    pub object_store: ObjectStore,
 }

 /// A [ProcedureManager] that maintains procedure states locally.
@@ -288,15 +314,8 @@ impl LocalManager {
        procedure_id: ProcedureId,
        step: u32,
        procedure: BoxedProcedure,
-    ) -> Result<()> {
-        let meta = Arc::new(ProcedureMeta {
-            id: procedure_id,
-            lock_notify: Notify::new(),
-            parent_id: None,
-            child_notify: Notify::new(),
-            lock_key: procedure.lock_key(),
-            exec_meta: Mutex::new(ExecMeta::default()),
-        });
+    ) -> Result<Watcher> {
+        let meta = Arc::new(ProcedureMeta::new(procedure_id, None, procedure.lock_key()));
        let runner = Runner {
            meta: meta.clone(),
            procedure,
@@ -305,6 +324,8 @@ impl LocalManager {
            store: ProcedureStore::new(self.state_store.clone()),
        };

+        let watcher = meta.state_receiver.clone();
+
        // Inserts meta into the manager before actually spawnd the runner.
        ensure!(
            self.manager_ctx.try_insert_procedure(meta),
@@ -313,10 +334,10 @@ impl LocalManager {

        common_runtime::spawn_bg(async move {
            // Run the root procedure.
-            let _ = runner.run().await;
+            runner.run().await;
        });

-        Ok(())
+        Ok(watcher)
    }
 }

@@ -331,57 +352,85 @@ impl ProcedureManager for LocalManager {
        Ok(())
    }

-    async fn submit(&self, procedure: ProcedureWithId) -> Result<()> {
+    async fn submit(&self, procedure: ProcedureWithId) -> Result<Watcher> {
        let procedure_id = procedure.id;
        ensure!(
            !self.manager_ctx.contains_procedure(procedure_id),
            DuplicateProcedureSnafu { procedure_id }
        );

-        self.submit_root(procedure.id, 0, procedure.procedure)?;
+        self.submit_root(procedure.id, 0, procedure.procedure)
+    }
+
+    async fn recover(&self) -> Result<()> {
+        logging::info!("LocalManager start to recover");
+
+        let procedure_store = ProcedureStore::new(self.state_store.clone());
+        let messages = procedure_store.load_messages().await?;
+
+        for (procedure_id, message) in &messages {
+            if message.parent_id.is_none() {
+                // This is the root procedure. We only submit the root procedure as it will
+                // submit sub-procedures to the manager.
+                let Some(loaded_procedure) = self.manager_ctx.load_one_procedure_from_message(*procedure_id, message) else {
+                    // Try to load other procedures.
+                    continue;
+                };
+
+                logging::info!(
+                    "Recover root procedure {}-{}, step: {}",
+                    loaded_procedure.procedure.type_name(),
+                    procedure_id,
+                    loaded_procedure.step
+                );
+
+                if let Err(e) = self.submit_root(
+                    *procedure_id,
+                    loaded_procedure.step,
+                    loaded_procedure.procedure,
+                ) {
+                    logging::error!(e; "Failed to recover procedure {}", procedure_id);
+                }
+            }
+        }

        Ok(())
    }

-    async fn recover(&self) -> Result<()> {
-        todo!("Recover procedure and messages")
-    }
-
    async fn procedure_state(&self, procedure_id: ProcedureId) -> Result<Option<ProcedureState>> {
        Ok(self.manager_ctx.state(procedure_id))
    }
+
+    fn procedure_watcher(&self, procedure_id: ProcedureId) -> Option<Watcher> {
+        self.manager_ctx.watcher(procedure_id)
+    }
 }

 /// Create a new [ProcedureMeta] for test purpose.
 #[cfg(test)]
 mod test_util {
-    use object_store::services::fs::Builder;
-    use tempdir::TempDir;
+    use common_test_util::temp_dir::TempDir;
+    use object_store::services::Fs as Builder;
+    use object_store::ObjectStoreBuilder;

    use super::*;

    pub(crate) fn procedure_meta_for_test() -> ProcedureMeta {
-        ProcedureMeta {
-            id: ProcedureId::random(),
-            lock_notify: Notify::new(),
-            parent_id: None,
-            child_notify: Notify::new(),
-            lock_key: None,
-            exec_meta: Mutex::new(ExecMeta::default()),
-        }
+        ProcedureMeta::new(ProcedureId::random(), None, LockKey::default())
    }

    pub(crate) fn new_object_store(dir: &TempDir) -> ObjectStore {
        let store_dir = dir.path().to_str().unwrap();
        let accessor = Builder::default().root(store_dir).build().unwrap();
-        ObjectStore::new(accessor)
+        ObjectStore::new(accessor).finish()
    }
 }

 #[cfg(test)]
 mod tests {
-    use serde::{Deserialize, Serialize};
-    use tempdir::TempDir;
+    use common_error::mock::MockError;
+    use common_error::prelude::StatusCode;
+    use common_test_util::temp_dir::create_temp_dir;

    use super::*;
    use crate::error::Error;
@@ -398,9 +447,9 @@ mod tests {
        assert!(ctx.try_insert_procedure(meta.clone()));
        assert!(ctx.contains_procedure(meta.id));

-        assert_eq!(ProcedureState::Running, ctx.state(meta.id).unwrap());
+        assert!(ctx.state(meta.id).unwrap().is_running());
        meta.set_state(ProcedureState::Done);
-        assert_eq!(ProcedureState::Done, ctx.state(meta.id).unwrap());
+        assert!(ctx.state(meta.id).unwrap().is_done());
    }

    #[test]
@@ -447,100 +496,133 @@ mod tests {
        assert_eq!(expect, ctx.procedures_in_tree(&root));
    }

+    #[derive(Debug)]
+    struct ProcedureToLoad {
+        content: String,
+        lock_key: LockKey,
+    }
+
+    #[async_trait]
+    impl Procedure for ProcedureToLoad {
+        fn type_name(&self) -> &str {
+            "ProcedureToLoad"
+        }
+
+        async fn execute(&mut self, _ctx: &Context) -> Result<Status> {
+            Ok(Status::Done)
+        }
+
+        fn dump(&self) -> Result<String> {
+            Ok(self.content.clone())
+        }
+
+        fn lock_key(&self) -> LockKey {
+            self.lock_key.clone()
+        }
+    }
+
+    impl ProcedureToLoad {
+        fn new(content: &str) -> ProcedureToLoad {
+            ProcedureToLoad {
+                content: content.to_string(),
+                lock_key: LockKey::default(),
+            }
+        }
+
+        fn loader() -> BoxedProcedureLoader {
+            let f = |json: &str| {
+                let procedure = ProcedureToLoad::new(json);
+                Ok(Box::new(procedure) as _)
+            };
+            Box::new(f)
+        }
+    }
+
    #[test]
    fn test_register_loader() {
-        let dir = TempDir::new("register").unwrap();
+        let dir = create_temp_dir("register");
        let config = ManagerConfig {
            object_store: test_util::new_object_store(&dir),
        };
        let manager = LocalManager::new(config);

-        #[derive(Debug, Serialize, Deserialize)]
-        struct MockData {
-            id: u32,
-            content: String,
-        }
-
-        #[derive(Debug)]
-        struct ProcedureToLoad {
-            data: MockData,
-        }
-
-        #[async_trait]
-        impl Procedure for ProcedureToLoad {
-            fn type_name(&self) -> &str {
-                "ProcedureToLoad"
-            }
-
-            async fn execute(&mut self, _ctx: &Context) -> Result<Status> {
-                unimplemented!()
-            }
-
-            fn dump(&self) -> Result<String> {
-                Ok(serde_json::to_string(&self.data).unwrap())
-            }
-
-            fn lock_key(&self) -> Option<LockKey> {
-                None
-            }
-        }
-
-        let loader = |json: &str| {
-            let data = serde_json::from_str(json).unwrap();
-            let procedure = ProcedureToLoad { data };
-            Ok(Box::new(procedure) as _)
-        };
        manager
-            .register_loader("ProcedureToLoad", Box::new(loader))
+            .register_loader("ProcedureToLoad", ProcedureToLoad::loader())
            .unwrap();
        // Register duplicate loader.
        let err = manager
-            .register_loader("ProcedureToLoad", Box::new(loader))
+            .register_loader("ProcedureToLoad", ProcedureToLoad::loader())
            .unwrap_err();
        assert!(matches!(err, Error::LoaderConflict { .. }), "{err}");
    }

+    #[tokio::test]
+    async fn test_recover() {
+        let dir = create_temp_dir("recover");
+        let object_store = test_util::new_object_store(&dir);
+        let config = ManagerConfig {
+            object_store: object_store.clone(),
+        };
+        let manager = LocalManager::new(config);
+
+        manager
+            .register_loader("ProcedureToLoad", ProcedureToLoad::loader())
+            .unwrap();
+
+        // Prepare data
+        let procedure_store = ProcedureStore::from(object_store.clone());
+        let root: BoxedProcedure = Box::new(ProcedureToLoad::new("test recover manager"));
+        let root_id = ProcedureId::random();
+        // Prepare data for the root procedure.
+        for step in 0..3 {
+            procedure_store
+                .store_procedure(root_id, step, &root, None)
+                .await
+                .unwrap();
+        }
+
+        let child: BoxedProcedure = Box::new(ProcedureToLoad::new("a child procedure"));
+        let child_id = ProcedureId::random();
+        // Prepare data for the child procedure
+        for step in 0..2 {
+            procedure_store
+                .store_procedure(child_id, step, &child, Some(root_id))
+                .await
+                .unwrap();
+        }
+
+        // Recover the manager
+        manager.recover().await.unwrap();
+
+        // The manager should submit the root procedure.
+        assert!(manager.procedure_state(root_id).await.unwrap().is_some());
+        // Since the mocked root procedure actually doesn't submit subprocedures, so there is no
+        // related state.
+        assert!(manager.procedure_state(child_id).await.unwrap().is_none());
+    }
+
    #[tokio::test]
    async fn test_submit_procedure() {
-        let dir = TempDir::new("submit").unwrap();
+        let dir = create_temp_dir("submit");
        let config = ManagerConfig {
            object_store: test_util::new_object_store(&dir),
        };
        let manager = LocalManager::new(config);

-        #[derive(Debug)]
-        struct MockProcedure {}
-
-        #[async_trait]
-        impl Procedure for MockProcedure {
-            fn type_name(&self) -> &str {
-                "MockProcedure"
-            }
-
-            async fn execute(&mut self, _ctx: &Context) -> Result<Status> {
-                unimplemented!()
-            }
-
-            fn dump(&self) -> Result<String> {
-                unimplemented!()
-            }
-
-            fn lock_key(&self) -> Option<LockKey> {
-                Some(LockKey::new("test.submit"))
-            }
-        }
-
        let procedure_id = ProcedureId::random();
        assert!(manager
            .procedure_state(procedure_id)
            .await
            .unwrap()
            .is_none());
+        assert!(manager.procedure_watcher(procedure_id).is_none());

+        let mut procedure = ProcedureToLoad::new("submit");
+        procedure.lock_key = LockKey::single("test.submit");
        manager
            .submit(ProcedureWithId {
                id: procedure_id,
-                procedure: Box::new(MockProcedure {}),
+                procedure: Box::new(procedure),
            })
            .await
            .unwrap();
@@ -549,15 +631,77 @@ mod tests {
            .await
            .unwrap()
            .is_some());
+        // Wait for the procedure done.
+        let mut watcher = manager.procedure_watcher(procedure_id).unwrap();
+        watcher.changed().await.unwrap();
+        assert!(watcher.borrow().is_done());

        // Try to submit procedure with same id again.
        let err = manager
            .submit(ProcedureWithId {
                id: procedure_id,
-                procedure: Box::new(MockProcedure {}),
+                procedure: Box::new(ProcedureToLoad::new("submit")),
            })
            .await
            .unwrap_err();
        assert!(matches!(err, Error::DuplicateProcedure { .. }), "{err}");
    }
+
+    #[tokio::test]
+    async fn test_state_changed_on_err() {
+        let dir = create_temp_dir("on_err");
+        let config = ManagerConfig {
+            object_store: test_util::new_object_store(&dir),
+        };
+        let manager = LocalManager::new(config);
+
+        #[derive(Debug)]
+        struct MockProcedure {
+            panic: bool,
+        }
+
+        #[async_trait]
+        impl Procedure for MockProcedure {
+            fn type_name(&self) -> &str {
+                "MockProcedure"
+            }
+
+            async fn execute(&mut self, _ctx: &Context) -> Result<Status> {
+                if self.panic {
+                    // Test the runner can set the state to failed even the procedure
+                    // panics.
+                    panic!();
+                } else {
+                    Err(Error::external(MockError::new(StatusCode::Unexpected)))
+                }
+            }
+
+            fn dump(&self) -> Result<String> {
+                Ok(String::new())
+            }
+
+            fn lock_key(&self) -> LockKey {
+                LockKey::single("test.submit")
+            }
+        }
+
+        let check_procedure = |procedure| {
+            async {
+                let procedure_id = ProcedureId::random();
+                let mut watcher = manager
+                    .submit(ProcedureWithId {
+                        id: procedure_id,
+                        procedure: Box::new(procedure),
+                    })
+                    .await
+                    .unwrap();
+                // Wait for the notification.
+                watcher.changed().await.unwrap();
+                assert!(watcher.borrow().is_failed());
+            }
+        };
+
+        check_procedure(MockProcedure { panic: false }).await;
+        check_procedure(MockProcedure { panic: true }).await;
+    }
 }
--- a/src/common/procedure/src/local/runner.rs
+++ b/src/common/procedure/src/local/runner.rs
@@ -12,15 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use std::time::Duration;

 use common_telemetry::logging;
-use tokio::sync::Notify;
 use tokio::time;

-use crate::error::{Error, Result};
-use crate::local::{ExecMeta, ManagerContext, ProcedureMeta, ProcedureMetaRef};
+use crate::error::{ProcedurePanicSnafu, Result};
+use crate::local::{ManagerContext, ProcedureMeta, ProcedureMetaRef};
 use crate::store::ProcedureStore;
 use crate::{BoxedProcedure, Context, ProcedureId, ProcedureState, ProcedureWithId, Status};

@@ -31,7 +30,7 @@ enum ExecResult {
    Continue,
    Done,
    RetryLater,
-    Failed(Error),
+    Failed,
 }

 #[cfg(test)]
@@ -49,7 +48,57 @@ impl ExecResult {
    }

    fn is_failed(&self) -> bool {
-        matches!(self, ExecResult::Failed(_))
+        matches!(self, ExecResult::Failed)
+    }
+}
+
+/// A guard to cleanup procedure state.
+struct ProcedureGuard {
+    meta: ProcedureMetaRef,
+    manager_ctx: Arc<ManagerContext>,
+    finish: bool,
+}
+
+impl ProcedureGuard {
+    /// Returns a new [ProcedureGuard].
+    fn new(meta: ProcedureMetaRef, manager_ctx: Arc<ManagerContext>) -> ProcedureGuard {
+        ProcedureGuard {
+            meta,
+            manager_ctx,
+            finish: false,
+        }
+    }
+
+    /// The procedure is finished successfully.
+    fn finish(mut self) {
+        self.finish = true;
+    }
+}
+
+impl Drop for ProcedureGuard {
+    fn drop(&mut self) {
+        if !self.finish {
+            logging::error!("Procedure {} exits unexpectedly", self.meta.id);
+
+            // Set state to failed. This is useful in test as runtime may not abort when the runner task panics.
+            // See https://github.com/tokio-rs/tokio/issues/2002 .
+            // We set set_panic_hook() in the application's main function. But our tests don't have this panic hook.
+            let err = ProcedurePanicSnafu {
+                procedure_id: self.meta.id,
+            }
+            .build();
+            self.meta.set_state(ProcedureState::failed(Arc::new(err)));
+        }
+
+        // Notify parent procedure.
+        if let Some(parent_id) = self.meta.parent_id {
+            self.manager_ctx.notify_by_subprocedure(parent_id);
+        }
+
+        // Release lock in reverse order.
+        for key in self.meta.lock_key.keys_to_unlock() {
+            self.manager_ctx.lock_map.release_lock(key, self.meta.id);
+        }
    }
 }

@@ -64,45 +113,37 @@ pub(crate) struct Runner {

 impl Runner {
    /// Run the procedure.
-    pub(crate) async fn run(mut self) -> Result<()> {
+    pub(crate) async fn run(mut self) {
+        // Ensure we can update the procedure state.
+        let guard = ProcedureGuard::new(self.meta.clone(), self.manager_ctx.clone());
+
        logging::info!(
            "Runner {}-{} starts",
            self.procedure.type_name(),
            self.meta.id
        );
-        // We use the lock key in ProcedureMeta as it considers locks inherited from
-        // its parent.
-        let lock_key = self.meta.lock_key.clone();

-        // TODO(yingwen):
-        // 1. Support multiple lock keys;
-        // 2. Detect recursive locking (and deadlock) if possible. Maybe we could detect
+        // TODO(yingwen): Detect recursive locking (and deadlock) if possible. Maybe we could detect
        // recursive locking by adding a root procedure id to the meta.
-        // Acquire lock if necessary.
-        if let Some(key) = &lock_key {
+        for key in self.meta.lock_key.keys_to_lock() {
+            // Acquire lock for each key.
            self.manager_ctx
                .lock_map
-                .acquire_lock(key.key(), self.meta.clone())
+                .acquire_lock(key, self.meta.clone())
                .await;
        }

-        let mut result = Ok(());
        // Execute the procedure. We need to release the lock whenever the the execution
        // is successful or fail.
-        if let Err(e) = self.execute_procedure_in_loop().await {
-            result = Err(e);
-        }
+        self.execute_procedure_in_loop().await;

-        // Notify parent procedure.
-        if let Some(parent_id) = self.meta.parent_id {
-            self.manager_ctx.notify_by_subprocedure(parent_id);
-        }
+        // We can't remove the metadata of the procedure now as users and its parent might
+        // need to query its state.
+        // TODO(yingwen): 1. Add TTL to the metadata; 2. Only keep state in the procedure store
+        // so we don't need to always store the metadata in memory after the procedure is done.

-        if let Some(key) = &lock_key {
-            self.manager_ctx
-                .lock_map
-                .release_lock(key.key(), self.meta.id);
-        }
+        // Release locks and notify parent procedure.
+        guard.finish();

        // If this is the root procedure, clean up message cache.
        if self.meta.parent_id.is_none() {
@@ -110,33 +151,26 @@ impl Runner {
            self.manager_ctx.remove_messages(&procedure_ids);
        }

-        // We can't remove the metadata of the procedure now as users and its parent might
-        // need to query its state.
-        // TODO(yingwen): 1. Add TTL to the metadata; 2. Only keep state in the procedure store
-        // so we don't need to always store the metadata in memory after the procedure is done.
-
        logging::info!(
            "Runner {}-{} exits",
            self.procedure.type_name(),
            self.meta.id
        );
-
-        result
    }

-    async fn execute_procedure_in_loop(&mut self) -> Result<()> {
+    async fn execute_procedure_in_loop(&mut self) {
        let ctx = Context {
            procedure_id: self.meta.id,
+            provider: self.manager_ctx.clone(),
        };

        loop {
            match self.execute_once(&ctx).await {
                ExecResult::Continue => (),
-                ExecResult::Done => return Ok(()),
+                ExecResult::Done | ExecResult::Failed => return,
                ExecResult::RetryLater => {
                    self.wait_on_err().await;
                }
-                ExecResult::Failed(e) => return Err(e),
            }
        }
    }
@@ -176,19 +210,24 @@ impl Runner {
            Err(e) => {
                logging::error!(
                    e;
-                    "Failed to execute procedure {}-{}",
+                    "Failed to execute procedure {}-{}, retry: {}",
                    self.procedure.type_name(),
-                    self.meta.id
+                    self.meta.id,
+                    e.is_retry_later(),
                );

-                self.meta.set_state(ProcedureState::Failed);
+                if e.is_retry_later() {
+                    return ExecResult::RetryLater;
+                }
+
+                self.meta.set_state(ProcedureState::failed(Arc::new(e)));

                // Write rollback key so we can skip this procedure while recovering procedures.
                if self.rollback_procedure().await.is_err() {
                    return ExecResult::RetryLater;
                }

-                ExecResult::Failed(e)
+                ExecResult::Failed
            }
        }
    }
@@ -212,14 +251,11 @@ impl Runner {
            step = loaded_procedure.step;
        }

-        let meta = Arc::new(ProcedureMeta {
-            id: procedure_id,
-            lock_notify: Notify::new(),
-            parent_id: Some(self.meta.id),
-            child_notify: Notify::new(),
-            lock_key: procedure.lock_key(),
-            exec_meta: Mutex::new(ExecMeta::default()),
-        });
+        let meta = Arc::new(ProcedureMeta::new(
+            procedure_id,
+            Some(self.meta.id),
+            procedure.lock_key(),
+        ));
        let runner = Runner {
            meta: meta.clone(),
            procedure,
@@ -261,7 +297,7 @@ impl Runner {
                self.procedure.type_name(),
                self.meta.id,
                subprocedure.procedure.type_name(),
-                subprocedure.id
+                subprocedure.id,
            );

            self.submit_subprocedure(subprocedure.id, subprocedure.procedure);
@@ -343,7 +379,7 @@ impl Runner {
        logging::info!(
            "Procedure {}-{} done",
            self.procedure.type_name(),
-            self.meta.id
+            self.meta.id,
        );

        // Mark the state of this procedure to done.
@@ -359,15 +395,14 @@ mod tests {
    use common_error::ext::PlainError;
    use common_error::mock::MockError;
    use common_error::prelude::StatusCode;
+    use common_test_util::temp_dir::create_temp_dir;
    use futures_util::future::BoxFuture;
    use futures_util::{FutureExt, TryStreamExt};
    use object_store::ObjectStore;
-    use tempdir::TempDir;

    use super::*;
    use crate::local::test_util;
-    use crate::store::ObjectStateStore;
-    use crate::{LockKey, Procedure};
+    use crate::{ContextProvider, Error, LockKey, Procedure};

    const ROOT_ID: &str = "9f805a1f-05f7-490c-9f91-bd56e3cc54c1";

@@ -385,12 +420,6 @@ mod tests {
        }
    }

-    fn new_procedure_store(object_store: ObjectStore) -> ProcedureStore {
-        let state_store = ObjectStateStore::new(object_store);
-
-        ProcedureStore::new(Arc::new(state_store))
-    }
-
    async fn check_files(object_store: &ObjectStore, procedure_id: ProcedureId, files: &[&str]) {
        let dir = format!("{procedure_id}/");
        let object = object_store.object(&dir);
@@ -404,10 +433,29 @@ mod tests {
        assert_eq!(files, files_in_dir);
    }

+    fn context_without_provider(procedure_id: ProcedureId) -> Context {
+        struct MockProvider;
+
+        #[async_trait]
+        impl ContextProvider for MockProvider {
+            async fn procedure_state(
+                &self,
+                _procedure_id: ProcedureId,
+            ) -> Result<Option<ProcedureState>> {
+                unimplemented!()
+            }
+        }
+
+        Context {
+            procedure_id,
+            provider: Arc::new(MockProvider),
+        }
+    }
+
    #[derive(Debug)]
    struct ProcedureAdapter<F> {
        data: String,
-        lock_key: Option<LockKey>,
+        lock_key: LockKey,
        exec_fn: F,
    }

@@ -424,14 +472,14 @@ mod tests {
    #[async_trait]
    impl<F> Procedure for ProcedureAdapter<F>
    where
-        F: FnMut() -> BoxFuture<'static, Result<Status>> + Send + Sync,
+        F: FnMut(Context) -> BoxFuture<'static, Result<Status>> + Send + Sync,
    {
        fn type_name(&self) -> &str {
            "ProcedureAdapter"
        }

-        async fn execute(&mut self, _ctx: &Context) -> Result<Status> {
-            let f = (self.exec_fn)();
+        async fn execute(&mut self, ctx: &Context) -> Result<Status> {
+            let f = (self.exec_fn)(ctx.clone());
            f.await
        }

@@ -439,14 +487,14 @@ mod tests {
            Ok(self.data.clone())
        }

-        fn lock_key(&self) -> Option<LockKey> {
+        fn lock_key(&self) -> LockKey {
            self.lock_key.clone()
        }
    }

    async fn execute_once_normal(persist: bool, first_files: &[&str], second_files: &[&str]) {
        let mut times = 0;
-        let exec_fn = move || {
+        let exec_fn = move |_| {
            times += 1;
            async move {
                if times == 1 {
@@ -459,17 +507,15 @@ mod tests {
        };
        let normal = ProcedureAdapter {
            data: "normal".to_string(),
-            lock_key: Some(LockKey::new("catalog.schema.table")),
+            lock_key: LockKey::single("catalog.schema.table"),
            exec_fn,
        };

-        let dir = TempDir::new("normal").unwrap();
+        let dir = create_temp_dir("normal");
        let meta = normal.new_meta(ROOT_ID);
-        let ctx = Context {
-            procedure_id: meta.id,
-        };
+        let ctx = context_without_provider(meta.id);
        let object_store = test_util::new_object_store(&dir);
-        let procedure_store = new_procedure_store(object_store.clone());
+        let procedure_store = ProcedureStore::from(object_store.clone());
        let mut runner = new_runner(meta, Box::new(normal), procedure_store);

        let res = runner.execute_once(&ctx).await;
@@ -498,7 +544,7 @@ mod tests {

    #[tokio::test]
    async fn test_on_suspend_empty() {
-        let exec_fn = move || {
+        let exec_fn = move |_| {
            async move {
                Ok(Status::Suspended {
                    subprocedures: Vec::new(),
@@ -509,26 +555,24 @@ mod tests {
        };
        let suspend = ProcedureAdapter {
            data: "suspend".to_string(),
-            lock_key: Some(LockKey::new("catalog.schema.table")),
+            lock_key: LockKey::single("catalog.schema.table"),
            exec_fn,
        };

-        let dir = TempDir::new("suspend").unwrap();
+        let dir = create_temp_dir("suspend");
        let meta = suspend.new_meta(ROOT_ID);
-        let ctx = Context {
-            procedure_id: meta.id,
-        };
+        let ctx = context_without_provider(meta.id);
        let object_store = test_util::new_object_store(&dir);
-        let procedure_store = new_procedure_store(object_store.clone());
+        let procedure_store = ProcedureStore::from(object_store.clone());
        let mut runner = new_runner(meta, Box::new(suspend), procedure_store);

        let res = runner.execute_once(&ctx).await;
        assert!(res.is_continue(), "{res:?}");
    }

-    fn new_child_procedure(procedure_id: ProcedureId, key: &str) -> ProcedureWithId {
+    fn new_child_procedure(procedure_id: ProcedureId, keys: &[&str]) -> ProcedureWithId {
        let mut times = 0;
-        let exec_fn = move || {
+        let exec_fn = move |_| {
            times += 1;
            async move {
                if times == 1 {
@@ -542,7 +586,7 @@ mod tests {
        };
        let child = ProcedureAdapter {
            data: "child".to_string(),
-            lock_key: Some(LockKey::new(key)),
+            lock_key: LockKey::new(keys.iter().map(|k| k.to_string())),
            exec_fn,
        };

@@ -557,16 +601,18 @@ mod tests {
        let mut times = 0;
        let children_ids = [ProcedureId::random(), ProcedureId::random()];
        let keys = [
-            "catalog.schema.table.region-0",
-            "catalog.schema.table.region-1",
+            &[
+                "catalog.schema.table.region-0",
+                "catalog.schema.table.region-1",
+            ],
+            &[
+                "catalog.schema.table.region-2",
+                "catalog.schema.table.region-3",
+            ],
        ];
-        let manager_ctx = Arc::new(ManagerContext::new());

-        let ctx_in_fn = manager_ctx.clone();
-        let exec_fn = move || {
+        let exec_fn = move |ctx: Context| {
            times += 1;
-            let ctx_in_future = ctx_in_fn.clone();
-
            async move {
                if times == 1 {
                    // Submit subprocedures.
@@ -574,15 +620,25 @@ mod tests {
                        subprocedures: children_ids
                            .into_iter()
                            .zip(keys)
-                            .map(|(id, key)| new_child_procedure(id, key))
+                            .map(|(id, key_slice)| new_child_procedure(id, key_slice))
                            .collect(),
                        persist: true,
                    })
                } else {
                    // Wait for subprocedures.
-                    let all_child_done = children_ids
-                        .iter()
-                        .all(|id| ctx_in_future.state(*id) == Some(ProcedureState::Done));
+                    let mut all_child_done = true;
+                    for id in children_ids {
+                        let is_not_done = ctx
+                            .provider
+                            .procedure_state(id)
+                            .await
+                            .unwrap()
+                            .map(|s| !s.is_done())
+                            .unwrap_or(true);
+                        if is_not_done {
+                            all_child_done = false;
+                        }
+                    }
                    if all_child_done {
                        Ok(Status::Done)
                    } else {
@@ -598,23 +654,24 @@ mod tests {
        };
        let parent = ProcedureAdapter {
            data: "parent".to_string(),
-            lock_key: Some(LockKey::new("catalog.schema.table")),
+            lock_key: LockKey::single("catalog.schema.table"),
            exec_fn,
        };

-        let dir = TempDir::new("parent").unwrap();
+        let dir = create_temp_dir("parent");
        let meta = parent.new_meta(ROOT_ID);
        let procedure_id = meta.id;
-        // Manually add this procedure to the manager ctx.
-        assert!(manager_ctx.try_insert_procedure(meta.clone()));

        let object_store = test_util::new_object_store(&dir);
-        let procedure_store = new_procedure_store(object_store.clone());
-        let mut runner = new_runner(meta, Box::new(parent), procedure_store);
+        let procedure_store = ProcedureStore::from(object_store.clone());
+        let mut runner = new_runner(meta.clone(), Box::new(parent), procedure_store);
+        let manager_ctx = Arc::new(ManagerContext::new());
+        // Manually add this procedure to the manager ctx.
+        assert!(manager_ctx.try_insert_procedure(meta));
        // Replace the manager ctx.
        runner.manager_ctx = manager_ctx;

-        runner.run().await.unwrap();
+        runner.run().await;

        // Check files on store.
        for child_id in children_ids {
@@ -636,49 +693,82 @@ mod tests {
    #[tokio::test]
    async fn test_execute_on_error() {
        let exec_fn =
-            || async { Err(Error::external(MockError::new(StatusCode::Unexpected))) }.boxed();
+            |_| async { Err(Error::external(MockError::new(StatusCode::Unexpected))) }.boxed();
        let fail = ProcedureAdapter {
            data: "fail".to_string(),
-            lock_key: Some(LockKey::new("catalog.schema.table")),
+            lock_key: LockKey::single("catalog.schema.table"),
            exec_fn,
        };

-        let dir = TempDir::new("fail").unwrap();
+        let dir = create_temp_dir("fail");
        let meta = fail.new_meta(ROOT_ID);
-        let ctx = Context {
-            procedure_id: meta.id,
-        };
+        let ctx = context_without_provider(meta.id);
        let object_store = test_util::new_object_store(&dir);
-        let procedure_store = new_procedure_store(object_store.clone());
+        let procedure_store = ProcedureStore::from(object_store.clone());
        let mut runner = new_runner(meta.clone(), Box::new(fail), procedure_store);

        let res = runner.execute_once(&ctx).await;
        assert!(res.is_failed(), "{res:?}");
-        assert_eq!(ProcedureState::Failed, meta.state());
+        assert!(meta.state().is_failed());
        check_files(&object_store, ctx.procedure_id, &["0000000000.rollback"]).await;
    }

+    #[tokio::test]
+    async fn test_execute_on_retry_later_error() {
+        let mut times = 0;
+
+        let exec_fn = move |_| {
+            times += 1;
+            async move {
+                if times == 1 {
+                    Err(Error::retry_later(MockError::new(StatusCode::Unexpected)))
+                } else {
+                    Ok(Status::Done)
+                }
+            }
+            .boxed()
+        };
+
+        let retry_later = ProcedureAdapter {
+            data: "retry_later".to_string(),
+            lock_key: LockKey::single("catalog.schema.table"),
+            exec_fn,
+        };
+
+        let dir = create_temp_dir("retry_later");
+        let meta = retry_later.new_meta(ROOT_ID);
+        let ctx = context_without_provider(meta.id);
+        let object_store = test_util::new_object_store(&dir);
+        let procedure_store = ProcedureStore::from(object_store.clone());
+        let mut runner = new_runner(meta.clone(), Box::new(retry_later), procedure_store);
+
+        let res = runner.execute_once(&ctx).await;
+        assert!(res.is_retry_later(), "{res:?}");
+        assert!(meta.state().is_running());
+
+        let res = runner.execute_once(&ctx).await;
+        assert!(res.is_done(), "{res:?}");
+        assert!(meta.state().is_done());
+        check_files(&object_store, ctx.procedure_id, &["0000000000.commit"]).await;
+    }
+
    #[tokio::test]
    async fn test_child_error() {
        let mut times = 0;
-        let manager_ctx = Arc::new(ManagerContext::new());
        let child_id = ProcedureId::random();

-        let ctx_in_fn = manager_ctx.clone();
-        let exec_fn = move || {
+        let exec_fn = move |ctx: Context| {
            times += 1;
-            let ctx_in_future = ctx_in_fn.clone();
-
            async move {
                if times == 1 {
                    // Submit subprocedures.
-                    let exec_fn = || {
+                    let exec_fn = |_| {
                        async { Err(Error::external(MockError::new(StatusCode::Unexpected))) }
                            .boxed()
                    };
                    let fail = ProcedureAdapter {
                        data: "fail".to_string(),
-                        lock_key: Some(LockKey::new("catalog.schema.table.region-0")),
+                        lock_key: LockKey::single("catalog.schema.table.region-0"),
                        exec_fn,
                    };

@@ -691,10 +781,11 @@ mod tests {
                    })
                } else {
                    // Wait for subprocedures.
-                    logging::info!("child state is {:?}", ctx_in_future.state(child_id));
-                    if ctx_in_future.state(child_id) == Some(ProcedureState::Failed) {
+                    let state = ctx.provider.procedure_state(child_id).await.unwrap();
+                    let is_failed = state.map(|s| s.is_failed()).unwrap_or(false);
+                    if is_failed {
                        // The parent procedure to abort itself if child procedure is failed.
-                        Err(Error::external(PlainError::new(
+                        Err(Error::from_error_ext(PlainError::new(
                            "subprocedure failed".to_string(),
                            StatusCode::Unexpected,
                        )))
@@ -711,23 +802,26 @@ mod tests {
        };
        let parent = ProcedureAdapter {
            data: "parent".to_string(),
-            lock_key: Some(LockKey::new("catalog.schema.table")),
+            lock_key: LockKey::single("catalog.schema.table"),
            exec_fn,
        };

-        let dir = TempDir::new("child_err").unwrap();
+        let dir = create_temp_dir("child_err");
        let meta = parent.new_meta(ROOT_ID);
-        // Manually add this procedure to the manager ctx.
-        assert!(manager_ctx.try_insert_procedure(meta.clone()));

        let object_store = test_util::new_object_store(&dir);
-        let procedure_store = new_procedure_store(object_store.clone());
-        let mut runner = new_runner(meta, Box::new(parent), procedure_store);
+        let procedure_store = ProcedureStore::from(object_store.clone());
+        let mut runner = new_runner(meta.clone(), Box::new(parent), procedure_store);
+
+        let manager_ctx = Arc::new(ManagerContext::new());
+        // Manually add this procedure to the manager ctx.
+        assert!(manager_ctx.try_insert_procedure(meta.clone()));
        // Replace the manager ctx.
        runner.manager_ctx = manager_ctx;

        // Run the runer and execute the procedure.
-        let err = runner.run().await.unwrap_err();
-        assert!(err.to_string().contains("subprocedure failed"), "{err}");
+        runner.run().await;
+        let err = meta.state().error().unwrap().to_string();
+        assert!(err.contains("subprocedure failed"), "{err}");
    }
 }
--- a/src/common/procedure/src/procedure.rs
+++ b/src/common/procedure/src/procedure.rs
@@ -18,10 +18,12 @@ use std::sync::Arc;

 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
+use smallvec::{smallvec, SmallVec};
 use snafu::{ResultExt, Snafu};
 use uuid::Uuid;

-use crate::error::Result;
+use crate::error::{Error, Result};
+use crate::watcher::Watcher;

 /// Procedure execution status.
 #[derive(Debug)]
@@ -58,11 +60,23 @@ impl Status {
    }
 }

+/// [ContextProvider] provides information about procedures in the [ProcedureManager].
+#[async_trait]
+pub trait ContextProvider: Send + Sync {
+    /// Query the procedure state.
+    async fn procedure_state(&self, procedure_id: ProcedureId) -> Result<Option<ProcedureState>>;
+}
+
+/// Reference-counted pointer to [ContextProvider].
+pub type ContextProviderRef = Arc<dyn ContextProvider>;
+
 /// Procedure execution context.
-#[derive(Debug)]
+#[derive(Clone)]
 pub struct Context {
    /// Id of the procedure.
    pub procedure_id: ProcedureId,
+    /// [ProcedureManager] context provider.
+    pub provider: ContextProviderRef,
 }

 /// A `Procedure` represents an operation or a set of operations to be performed step-by-step.
@@ -79,25 +93,41 @@ pub trait Procedure: Send + Sync {
    /// Dump the state of the procedure to a string.
    fn dump(&self) -> Result<String>;

-    /// Returns the [LockKey] if this procedure needs to acquire lock.
-    fn lock_key(&self) -> Option<LockKey>;
+    /// Returns the [LockKey] that this procedure needs to acquire.
+    fn lock_key(&self) -> LockKey;
 }

-/// A key to identify the lock.
-// We might hold multiple keys in this struct. When there are multiple keys, we need to sort the
-// keys lock all the keys in order to avoid dead lock.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct LockKey(String);
+/// Keys to identify required locks.
+///
+/// [LockKey] always sorts keys lexicographically so that they can be acquired
+/// in the same order.
+// Most procedures should only acquire 1 ~ 2 locks so we use smallvec to hold keys.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct LockKey(SmallVec<[String; 2]>);

 impl LockKey {
-    /// Returns a new [LockKey].
-    pub fn new(key: impl Into<String>) -> LockKey {
-        LockKey(key.into())
+    /// Returns a new [LockKey] with only one key.
+    pub fn single(key: impl Into<String>) -> LockKey {
+        LockKey(smallvec![key.into()])
    }

-    /// Returns the lock key.
-    pub fn key(&self) -> &str {
-        &self.0
+    /// Returns a new [LockKey] with keys from specific `iter`.
+    pub fn new(iter: impl IntoIterator<Item = String>) -> LockKey {
+        let mut vec: SmallVec<_> = iter.into_iter().collect();
+        vec.sort();
+        // Dedup keys to avoid acquiring the same key multiple times.
+        vec.dedup();
+        LockKey(vec)
+    }
+
+    /// Returns the keys to lock.
+    pub fn keys_to_lock(&self) -> impl Iterator<Item = &String> {
+        self.0.iter()
+    }
+
+    /// Returns the keys to unlock.
+    pub fn keys_to_unlock(&self) -> impl Iterator<Item = &String> {
+        self.0.iter().rev()
    }
 }

@@ -168,16 +198,46 @@ impl FromStr for ProcedureId {
 /// Loader to recover the [Procedure] instance from serialized data.
 pub type BoxedProcedureLoader = Box<dyn Fn(&str) -> Result<BoxedProcedure> + Send>;

-// TODO(yingwen): Find a way to return the error message if the procedure is failed.
 /// State of a submitted procedure.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Default, Clone)]
 pub enum ProcedureState {
    /// The procedure is running.
+    #[default]
    Running,
    /// The procedure is finished.
    Done,
    /// The procedure is failed and cannot proceed anymore.
-    Failed,
+    Failed { error: Arc<Error> },
+}
+
+impl ProcedureState {
+    /// Returns a [ProcedureState] with failed state.
+    pub fn failed(error: Arc<Error>) -> ProcedureState {
+        ProcedureState::Failed { error }
+    }
+
+    /// Returns true if the procedure state is running.
+    pub fn is_running(&self) -> bool {
+        matches!(self, ProcedureState::Running)
+    }
+
+    /// Returns true if the procedure state is done.
+    pub fn is_done(&self) -> bool {
+        matches!(self, ProcedureState::Done)
+    }
+
+    /// Returns true if the procedure state failed.
+    pub fn is_failed(&self) -> bool {
+        matches!(self, ProcedureState::Failed { .. })
+    }
+
+    /// Returns the error.
+    pub fn error(&self) -> Option<&Arc<Error>> {
+        match self {
+            ProcedureState::Failed { error } => Some(error),
+            _ => None,
+        }
+    }
 }

 // TODO(yingwen): Shutdown
@@ -188,7 +248,9 @@ pub trait ProcedureManager: Send + Sync + 'static {
    fn register_loader(&self, name: &str, loader: BoxedProcedureLoader) -> Result<()>;

    /// Submits a procedure to execute.
-    async fn submit(&self, procedure: ProcedureWithId) -> Result<()>;
+    ///
+    /// Returns a [Watcher] to watch the created procedure.
+    async fn submit(&self, procedure: ProcedureWithId) -> Result<Watcher>;

    /// Recovers unfinished procedures and reruns them.
    ///
@@ -199,6 +261,9 @@ pub trait ProcedureManager: Send + Sync + 'static {
    ///
    /// Returns `Ok(None)` if the procedure doesn't exist.
    async fn procedure_state(&self, procedure_id: ProcedureId) -> Result<Option<ProcedureState>>;
+
+    /// Returns a [Watcher] to watch [ProcedureState] of specific procedure.
+    fn procedure_watcher(&self, procedure_id: ProcedureId) -> Option<Watcher>;
 }

 /// Ref-counted pointer to the [ProcedureManager].
@@ -206,6 +271,9 @@ pub type ProcedureManagerRef = Arc<dyn ProcedureManager>;

 #[cfg(test)]
 mod tests {
+    use common_error::mock::MockError;
+    use common_error::prelude::StatusCode;
+
    use super::*;

    #[test]
@@ -235,8 +303,21 @@ mod tests {
    #[test]
    fn test_lock_key() {
        let entity = "catalog.schema.my_table";
-        let key = LockKey::new(entity);
-        assert_eq!(entity, key.key());
+        let key = LockKey::single(entity);
+        assert_eq!(vec![entity], key.keys_to_lock().collect::<Vec<_>>());
+        assert_eq!(vec![entity], key.keys_to_unlock().collect::<Vec<_>>());
+
+        let key = LockKey::new([
+            "b".to_string(),
+            "c".to_string(),
+            "a".to_string(),
+            "c".to_string(),
+        ]);
+        assert_eq!(vec!["a", "b", "c"], key.keys_to_lock().collect::<Vec<_>>());
+        assert_eq!(
+            vec!["c", "b", "a"],
+            key.keys_to_unlock().collect::<Vec<_>>()
+        );
    }

    #[test]
@@ -260,4 +341,17 @@ mod tests {
        let parsed = serde_json::from_str(&json).unwrap();
        assert_eq!(id, parsed);
    }
+
+    #[test]
+    fn test_procedure_state() {
+        assert!(ProcedureState::Running.is_running());
+        assert!(ProcedureState::Running.error().is_none());
+        assert!(ProcedureState::Done.is_done());
+
+        let state = ProcedureState::failed(Arc::new(Error::external(MockError::new(
+            StatusCode::Unexpected,
+        ))));
+        assert!(state.is_failed());
+        assert!(state.error().is_some());
+    }
 }
--- a/src/common/procedure/src/store.rs
+++ b/src/common/procedure/src/store.rs
@@ -14,9 +14,11 @@

 use std::collections::HashMap;
 use std::fmt;
+use std::sync::Arc;

 use common_telemetry::logging;
 use futures::TryStreamExt;
+use object_store::ObjectStore;
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;

@@ -27,7 +29,7 @@ use crate::{BoxedProcedure, ProcedureId};
 mod state_store;

 /// Serialized data of a procedure.
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ProcedureMessage {
    /// Type name of the procedure. The procedure framework also use the type name to
    /// find a loader to load the procedure.
@@ -115,7 +117,7 @@ impl ProcedureStore {
    }

    /// Load uncommitted procedures from the storage.
-    async fn load_messages(&self) -> Result<HashMap<ProcedureId, ProcedureMessage>> {
+    pub(crate) async fn load_messages(&self) -> Result<HashMap<ProcedureId, ProcedureMessage>> {
        let mut messages = HashMap::new();
        // Track the key-value pair by procedure id.
        let mut procedure_key_values: HashMap<_, (ParsedKey, Vec<u8>)> = HashMap::new();
@@ -163,6 +165,14 @@ impl ProcedureStore {
    }
 }

+impl From<ObjectStore> for ProcedureStore {
+    fn from(store: ObjectStore) -> ProcedureStore {
+        let state_store = ObjectStateStore::new(store);
+
+        ProcedureStore::new(Arc::new(state_store))
+    }
+}
+
 /// Suffix type of the key.
 #[derive(Debug, PartialEq, Eq)]
 enum KeyType {
@@ -235,12 +245,10 @@ impl ParsedKey {

 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
    use async_trait::async_trait;
-    use object_store::services::fs::Builder;
-    use object_store::ObjectStore;
-    use tempdir::TempDir;
+    use common_test_util::temp_dir::{create_temp_dir, TempDir};
+    use object_store::services::Fs as Builder;
+    use object_store::ObjectStoreBuilder;

    use super::*;
    use crate::{Context, LockKey, Procedure, Status};
@@ -248,10 +256,9 @@ mod tests {
    fn procedure_store_for_test(dir: &TempDir) -> ProcedureStore {
        let store_dir = dir.path().to_str().unwrap();
        let accessor = Builder::default().root(store_dir).build().unwrap();
-        let object_store = ObjectStore::new(accessor);
-        let state_store = ObjectStateStore::new(object_store);
+        let object_store = ObjectStore::new(accessor).finish();

-        ProcedureStore::new(Arc::new(state_store))
+        ProcedureStore::from(object_store)
    }

    #[test]
@@ -359,14 +366,14 @@ mod tests {
            Ok(self.data.clone())
        }

-        fn lock_key(&self) -> Option<LockKey> {
-            None
+        fn lock_key(&self) -> LockKey {
+            LockKey::default()
        }
    }

    #[tokio::test]
    async fn test_store_procedure() {
-        let dir = TempDir::new("store_procedure").unwrap();
+        let dir = create_temp_dir("store_procedure");
        let store = procedure_store_for_test(&dir);

        let procedure_id = ProcedureId::random();
@@ -391,7 +398,7 @@ mod tests {

    #[tokio::test]
    async fn test_commit_procedure() {
-        let dir = TempDir::new("commit_procedure").unwrap();
+        let dir = create_temp_dir("commit_procedure");
        let store = procedure_store_for_test(&dir);

        let procedure_id = ProcedureId::random();
@@ -409,7 +416,7 @@ mod tests {

    #[tokio::test]
    async fn test_rollback_procedure() {
-        let dir = TempDir::new("rollback_procedure").unwrap();
+        let dir = create_temp_dir("rollback_procedure");
        let store = procedure_store_for_test(&dir);

        let procedure_id = ProcedureId::random();
@@ -427,7 +434,7 @@ mod tests {

    #[tokio::test]
    async fn test_load_messages() {
-        let dir = TempDir::new("load_messages").unwrap();
+        let dir = create_temp_dir("load_messages");
        let store = procedure_store_for_test(&dir);

        // store 3 steps
--- a/src/common/procedure/src/store/state_store.rs
+++ b/src/common/procedure/src/store/state_store.rs
@@ -20,9 +20,7 @@ use futures::{Stream, TryStreamExt};
 use object_store::{ObjectMode, ObjectStore};
 use snafu::ResultExt;

-use crate::error::{
-    DeleteStateSnafu, Error, ListStateSnafu, PutStateSnafu, ReadStateSnafu, Result,
-};
+use crate::error::{DeleteStateSnafu, Error, PutStateSnafu, Result};

 /// Key value from state store.
 type KeyValue = (String, Vec<u8>);
@@ -72,22 +70,23 @@ impl StateStore for ObjectStateStore {

    async fn walk_top_down(&self, path: &str) -> Result<KeyValueStream> {
        let path_string = path.to_string();
-        let op = self.store.batch();
-        // Note that there is no guarantee about the order between files and dirs
-        // at the same level.
-        // See https://docs.rs/opendal/0.25.2/opendal/raw/struct.TopDownWalker.html#note
-        let stream = op
-            .walk_top_down(path)
-            .context(ListStateSnafu { path })?
-            .map_err(move |e| Error::ListState {
+
+        let lister = self
+            .store
+            .object(path)
+            .scan()
+            .await
+            .map_err(|e| Error::ListState {
                path: path_string.clone(),
                source: e,
-            })
+            })?;
+
+        let stream = lister
            .try_filter_map(|entry| async move {
                let key = entry.path();
-                let key_value = match entry.mode().await.context(ReadStateSnafu { key })? {
+                let key_value = match entry.mode().await? {
                    ObjectMode::FILE => {
-                        let value = entry.read().await.context(ReadStateSnafu { key })?;
+                        let value = entry.read().await?;

                        Some((key.to_string(), value))
                    }
@@ -95,6 +94,10 @@ impl StateStore for ObjectStateStore {
                };

                Ok(key_value)
+            })
+            .map_err(move |e| Error::ListState {
+                path: path_string.clone(),
+                source: e,
            });

        Ok(Box::pin(stream))
@@ -112,17 +115,18 @@ impl StateStore for ObjectStateStore {

 #[cfg(test)]
 mod tests {
-    use object_store::services::fs::Builder;
-    use tempdir::TempDir;
+    use common_test_util::temp_dir::create_temp_dir;
+    use object_store::services::Fs as Builder;
+    use object_store::ObjectStoreBuilder;

    use super::*;

    #[tokio::test]
    async fn test_object_state_store() {
-        let dir = TempDir::new("state_store").unwrap();
+        let dir = create_temp_dir("state_store");
        let store_dir = dir.path().to_str().unwrap();
        let accessor = Builder::default().root(store_dir).build().unwrap();
-        let object_store = ObjectStore::new(accessor);
+        let object_store = ObjectStore::new(accessor).finish();
        let state_store = ObjectStateStore::new(object_store);

        let data: Vec<_> = state_store
--- a/src/common/procedure/src/watcher.rs
+++ b/src/common/procedure/src/watcher.rs
@@ -0,0 +1,38 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use snafu::ResultExt;
+use tokio::sync::watch::Receiver;
+
+use crate::error::{ProcedureExecSnafu, Result, WaitWatcherSnafu};
+use crate::procedure::ProcedureState;
+
+/// Watcher to watch procedure state.
+pub type Watcher = Receiver<ProcedureState>;
+
+/// Wait the [Watcher] until the [ProcedureState] is done.
+pub async fn wait(watcher: &mut Watcher) -> Result<()> {
+    loop {
+        watcher.changed().await.context(WaitWatcherSnafu)?;
+        match &*watcher.borrow() {
+            ProcedureState::Running => (),
+            ProcedureState::Done => {
+                return Ok(());
+            }
+            ProcedureState::Failed { error } => {
+                return Err(error.clone()).context(ProcedureExecSnafu);
+            }
+        }
+    }
+}
--- a/src/common/query/Cargo.toml
+++ b/src/common/query/Cargo.toml
@@ -14,7 +14,7 @@ datafusion-common.workspace = true
 datafusion-expr.workspace = true
 datatypes = { path = "../../datatypes" }
 snafu.workspace = true
-statrs = "0.15"
+statrs = "0.16"

 [dev-dependencies]
 common-base = { path = "../base" }
--- a/src/common/query/src/error.rs
+++ b/src/common/query/src/error.rs
@@ -239,7 +239,6 @@ impl From<BoxedError> for Error {

 #[cfg(test)]
 mod tests {
-    use datatypes::arrow::error::ArrowError;
    use snafu::GenerateImplicitData;

    use super::*;
@@ -286,7 +285,7 @@ mod tests {
    fn test_convert_df_recordbatch_stream_error() {
        let result: std::result::Result<i32, common_recordbatch::error::Error> =
            Err(common_recordbatch::error::Error::PollStream {
-                source: ArrowError::DivideByZero,
+                source: DataFusionError::Internal("blabla".to_string()),
                backtrace: Backtrace::generate(),
            });
        let error = result
--- a/src/common/query/src/physical_plan.rs
+++ b/src/common/query/src/physical_plan.rs
@@ -315,7 +315,11 @@ mod test {
                .unwrap()
                .build()
                .unwrap();
-        let physical_plan = ctx.create_physical_plan(&logical_plan).await.unwrap();
+        let physical_plan = ctx
+            .state()
+            .create_physical_plan(&logical_plan)
+            .await
+            .unwrap();
        let df_recordbatches = collect(physical_plan, Arc::new(TaskContext::from(&ctx)))
            .await
            .unwrap();
--- a/src/common/recordbatch/src/adapter.rs
+++ b/src/common/recordbatch/src/adapter.rs
@@ -18,9 +18,9 @@ use std::sync::Arc;
 use std::task::{Context, Poll};

 use datafusion::arrow::datatypes::SchemaRef as DfSchemaRef;
+use datafusion::error::Result as DfResult;
 use datafusion::physical_plan::RecordBatchStream as DfRecordBatchStream;
 use datafusion_common::DataFusionError;
-use datatypes::arrow::error::{ArrowError, Result as ArrowResult};
 use datatypes::schema::{Schema, SchemaRef};
 use futures::ready;
 use snafu::ResultExt;
@@ -57,14 +57,14 @@ impl DfRecordBatchStream for DfRecordBatchStreamAdapter {
 }

 impl Stream for DfRecordBatchStreamAdapter {
-    type Item = ArrowResult<DfRecordBatch>;
+    type Item = DfResult<DfRecordBatch>;

    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        match Pin::new(&mut self.stream).poll_next(cx) {
            Poll::Pending => Poll::Pending,
            Poll::Ready(Some(recordbatch)) => match recordbatch {
                Ok(recordbatch) => Poll::Ready(Some(Ok(recordbatch.into_df_record_batch()))),
-                Err(e) => Poll::Ready(Some(Err(ArrowError::ExternalError(Box::new(e))))),
+                Err(e) => Poll::Ready(Some(Err(DataFusionError::External(Box::new(e))))),
            },
            Poll::Ready(None) => Poll::Ready(None),
        }
@@ -242,12 +242,12 @@ mod test {
        )]));
        let batch1 = RecordBatch::new(
            schema.clone(),
-            vec![Arc::new(Int32Vector::from_slice(&[1])) as _],
+            vec![Arc::new(Int32Vector::from_slice([1])) as _],
        )
        .unwrap();
        let batch2 = RecordBatch::new(
            schema.clone(),
-            vec![Arc::new(Int32Vector::from_slice(&[2])) as _],
+            vec![Arc::new(Int32Vector::from_slice([2])) as _],
        )
        .unwrap();

--- a/src/common/recordbatch/src/error.rs
+++ b/src/common/recordbatch/src/error.rs
@@ -55,7 +55,7 @@ pub enum Error {

    #[snafu(display("Failed to poll stream, source: {}", source))]
    PollStream {
-        source: datatypes::arrow::error::ArrowError,
+        source: datafusion::error::DataFusionError,
        backtrace: Backtrace,
    },

--- a/src/common/recordbatch/src/lib.rs
+++ b/src/common/recordbatch/src/lib.rs
@@ -204,7 +204,7 @@ mod tests {
        );
        assert!(result.is_err());

-        let v: VectorRef = Arc::new(Int32Vector::from_slice(&[1, 2]));
+        let v: VectorRef = Arc::new(Int32Vector::from_slice([1, 2]));
        let expected = vec![RecordBatch::new(schema.clone(), vec![v.clone()]).unwrap()];
        let r = RecordBatches::try_from_columns(schema, vec![v]).unwrap();
        assert_eq!(r.take(), expected);
@@ -216,7 +216,7 @@ mod tests {
        let column_b = ColumnSchema::new("b", ConcreteDataType::string_datatype(), false);
        let column_c = ColumnSchema::new("c", ConcreteDataType::boolean_datatype(), false);

-        let va: VectorRef = Arc::new(Int32Vector::from_slice(&[1, 2]));
+        let va: VectorRef = Arc::new(Int32Vector::from_slice([1, 2]));
        let vb: VectorRef = Arc::new(StringVector::from(vec!["hello", "world"]));
        let vc: VectorRef = Arc::new(BooleanVector::from(vec![true, false]));

@@ -255,11 +255,11 @@ mod tests {
        let column_b = ColumnSchema::new("b", ConcreteDataType::string_datatype(), false);
        let schema = Arc::new(Schema::new(vec![column_a, column_b]));

-        let va1: VectorRef = Arc::new(Int32Vector::from_slice(&[1, 2]));
+        let va1: VectorRef = Arc::new(Int32Vector::from_slice([1, 2]));
        let vb1: VectorRef = Arc::new(StringVector::from(vec!["a", "b"]));
        let batch1 = RecordBatch::new(schema.clone(), vec![va1, vb1]).unwrap();

-        let va2: VectorRef = Arc::new(Int32Vector::from_slice(&[3, 4, 5]));
+        let va2: VectorRef = Arc::new(Int32Vector::from_slice([3, 4, 5]));
        let vb2: VectorRef = Arc::new(StringVector::from(vec!["c", "d", "e"]));
        let batch2 = RecordBatch::new(schema.clone(), vec![va2, vb2]).unwrap();

--- a/src/common/recordbatch/src/recordbatch.rs
+++ b/src/common/recordbatch/src/recordbatch.rs
@@ -189,8 +189,8 @@ mod tests {
        ]));
        let schema = Arc::new(Schema::try_from(arrow_schema).unwrap());

-        let c1 = Arc::new(UInt32Vector::from_slice(&[1, 2, 3]));
-        let c2 = Arc::new(UInt32Vector::from_slice(&[4, 5, 6]));
+        let c1 = Arc::new(UInt32Vector::from_slice([1, 2, 3]));
+        let c2 = Arc::new(UInt32Vector::from_slice([4, 5, 6]));
        let columns: Vec<VectorRef> = vec![c1, c2];

        let batch = RecordBatch::new(schema.clone(), columns.clone()).unwrap();
@@ -222,7 +222,7 @@ mod tests {
        let schema = Arc::new(Schema::try_new(column_schemas).unwrap());

        let numbers: Vec<u32> = (0..10).collect();
-        let columns = vec![Arc::new(UInt32Vector::from_slice(&numbers)) as VectorRef];
+        let columns = vec![Arc::new(UInt32Vector::from_slice(numbers)) as VectorRef];
        let batch = RecordBatch::new(schema, columns).unwrap();

        let output = serde_json::to_string(&batch).unwrap();
--- a/src/common/substrait/Cargo.toml
+++ b/src/common/substrait/Cargo.toml
@@ -5,6 +5,8 @@ edition.workspace = true
 license.workspace = true

 [dependencies]
+async-recursion = "1.0"
+async-trait.workspace = true
 bytes = "1.1"
 catalog = { path = "../../catalog" }
 common-catalog = { path = "../catalog" }
@@ -14,13 +16,14 @@ datafusion.workspace = true
 datafusion-expr.workspace = true
 datatypes = { path = "../../datatypes" }
 futures = "0.3"
-prost = "0.9"
+prost.workspace = true
+session = { path = "../../session" }
 snafu.workspace = true
 table = { path = "../../table" }

 [dependencies.substrait_proto]
 package = "substrait"
-version = "0.2"
+version = "0.4"

 [dev-dependencies]
 datatypes = { path = "../../datatypes" }
--- a/src/common/substrait/src/context.rs
+++ b/src/common/substrait/src/context.rs
@@ -15,10 +15,10 @@
 use std::collections::HashMap;

 use datafusion::common::DFSchemaRef;
-use substrait_proto::protobuf::extensions::simple_extension_declaration::{
+use substrait_proto::proto::extensions::simple_extension_declaration::{
    ExtensionFunction, MappingType,
 };
-use substrait_proto::protobuf::extensions::SimpleExtensionDeclaration;
+use substrait_proto::proto::extensions::SimpleExtensionDeclaration;

 #[derive(Default)]
 pub struct ConvertorContext {
--- a/Show More
+++ b/Show More