chore: add deprecate develop branch warning

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

.cargo/config.toml

@@ -12,9 +12,4 @@ rustflags = [
     "-Wclippy::print_stdout",
     "-Wclippy::print_stderr",
     "-Wclippy::implicit_clone",
-
-    # It seems clippy has made a false positive decision here when upgrading rust toolchain to
-    # nightly-2023-08-07, we do need it to be borrowed mutably.
-    # Allow it for now; try disallow it when the toolchain is upgraded in the future.
-    "-Aclippy::needless_pass_by_ref_mut",
 ]

.github/actions/build-greptime-binary/action.yml

@@ -40,9 +40,11 @@ runs:
     - name: Upload artifacts
       uses: ./.github/actions/upload-artifacts
       if: ${{ inputs.build-android-artifacts == 'false' }}
+      env:
+        PROFILE_TARGET: ${{ inputs.cargo-profile == 'dev' && 'debug' || inputs.cargo-profile }}
       with:
         artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: ./target/${{ inputs.cargo-profile }}/greptime
+        target-file: ./target/$PROFILE_TARGET/greptime
         version: ${{ inputs.version }}
         working-dir: ${{ inputs.working-dir }}
 

.github/actions/publish-github-release/action.yml

@@ -31,10 +31,12 @@ runs:
           echo "prerelease=false" >> $GITHUB_ENV
           echo "makeLatest=true" >> $GITHUB_ENV
           echo "generateReleaseNotes=false" >> $GITHUB_ENV
+          echo "omitBody=true" >> $GITHUB_ENV
         else
           echo "prerelease=true" >> $GITHUB_ENV
           echo "makeLatest=false" >> $GITHUB_ENV
           echo "generateReleaseNotes=true" >> $GITHUB_ENV
+          echo "omitBody=false" >> $GITHUB_ENV
         fi
 
     - name: Publish release
@@ -45,6 +47,7 @@ runs:
         makeLatest: ${{ env.makeLatest }}
         tag: ${{ inputs.version }}
         generateReleaseNotes: ${{ env.generateReleaseNotes }}
+        omitBody: ${{ env.omitBody }} # omitBody is true when the release is a official release.
         allowUpdates: true
         artifacts: |
           **/greptime-*/*

.github/actions/upload-artifacts/action.yml

@@ -22,7 +22,7 @@ runs:
       shell: bash
       run: |
         mkdir -p ${{ inputs.artifacts-dir }} && \
-        mv ${{ inputs.target-file }} ${{ inputs.artifacts-dir }}
+        cp ${{ inputs.target-file }} ${{ inputs.artifacts-dir }}
 
     # The compressed artifacts will use the following layout:
     # greptime-linux-amd64-pyo3-v0.3.0sha256sum

.github/doc-label-config.yml

@@ -0,0 +1,4 @@
+Doc not needed:
+    - '- \[x\]  This PR does not require documentation updates.'
+Doc update required:
+    - '- \[ \]  This PR does not require documentation updates.'

.github/pull_request_template.md

@@ -15,5 +15,6 @@ Please explain IN DETAIL what the changes are in this PR and why they are needed
 
 - [ ]  I have written the necessary rustdoc comments.
 - [ ]  I have added the necessary unit tests and integration tests.
+- [ ]  This PR does not require documentation updates.
 
 ## Refer to a related PR or issue link (optional)

.github/scripts/deploy-greptimedb.sh

@@ -107,12 +107,9 @@ function deploy_greptimedb_cluster_with_s3_storage() {
     --set storage.s3.bucket="$AWS_CI_TEST_BUCKET" \
     --set storage.s3.region="$AWS_REGION" \
     --set storage.s3.root="$DATA_ROOT" \
-    --set storage.s3.secretName=s3-credentials \
     --set storage.credentials.secretName=s3-credentials \
-    --set storage.credentials.secretCreation.enabled=true \
-    --set storage.credentials.secretCreation.enableEncryption=false \
-    --set storage.credentials.secretCreation.data.access-key-id="$AWS_ACCESS_KEY_ID" \
-    --set storage.credentials.secretCreation.data.secret-access-key="$AWS_SECRET_ACCESS_KEY"
+    --set storage.credentials.accessKeyId="$AWS_ACCESS_KEY_ID" \
+    --set storage.credentials.secretAccessKey="$AWS_SECRET_ACCESS_KEY"
 
   # Wait for greptimedb cluster to be ready.
   while true; do

.github/workflows/apidoc.yml

@@ -13,7 +13,7 @@ on:
 name: Build API docs
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-08-07
+  RUST_TOOLCHAIN: nightly-2023-12-19
 
 jobs:
   apidoc:

.github/workflows/dev-build.yml

@@ -55,10 +55,18 @@ on:
         description: Build and push images to DockerHub and ACR
         required: false
         default: true
+      cargo_profile:
+        type: choice
+        description: The cargo profile to use in building GreptimeDB.
+        default: nightly
+        options:
+          - dev
+          - release
+          - nightly
 
 # Use env variables to control all the release process.
 env:
-  CARGO_PROFILE: nightly
+  CARGO_PROFILE: ${{ inputs.cargo_profile }}
 
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
   DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}

.github/workflows/develop.yml

@@ -29,7 +29,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-08-07
+  RUST_TOOLCHAIN: nightly-2023-12-19
 
 jobs:
   typos:
@@ -42,7 +42,10 @@ jobs:
   check:
     name: Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ windows-latest-8-cores, ubuntu-20.04 ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -161,15 +164,21 @@ jobs:
         uses: Swatinem/rust-cache@v2
       - name: Install latest nextest release
         uses: taiki-e/install-action@nextest
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
       - name: Install Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
       - name: Install PyArrow Package
         run: pip install pyarrow
-      - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@cargo-llvm-cov
-      - name: Collect coverage data
+      - name: Setup etcd server
+        working-directory: tests-integration/fixtures/etcd
+        run: docker compose -f docker-compose-standalone.yml up -d --wait
+      - name: Setup kafka server
+        working-directory: tests-integration/fixtures/kafka
+        run: docker compose -f docker-compose-standalone.yml up -d --wait
+      - name: Run nextest cases
         run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
         env:
           CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
@@ -179,6 +188,8 @@ jobs:
           GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
           GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
           GT_S3_REGION: ${{ secrets.S3_REGION }}
+          GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
+          GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
           UNITTEST_LOG_DIR: "__unittest_logs"
       - name: Codecov upload
         uses: codecov/codecov-action@v2

.github/workflows/doc-label.yml

@@ -0,0 +1,20 @@
+name: "PR Doc Labeler"
+on:
+  pull_request_target:
+    types: [opened, edited, synchronize, ready_for_review, auto_merge_enabled, labeled, unlabeled]
+
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  triage:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: github/issue-labeler@v3.3
+      with:
+        configuration-path: .github/doc-label-config.yml
+        enable-versioned-regex: false
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        sync-labels: 1

.github/workflows/nightly-ci.yml

@@ -12,11 +12,12 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-08-07
+  RUST_TOOLCHAIN: nightly-2023-12-19
 
 jobs:
   sqlness:
     name: Sqlness Test
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -51,6 +52,7 @@ jobs:
           retention-days: 3
 
   test-on-windows:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: windows-latest-8-cores
     timeout-minutes: 60
     steps:

.github/workflows/nightly-funtional-tests.yml

@@ -9,6 +9,7 @@ on:
 jobs:
   sqlness-test:
     name: Run sqlness test
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout

.github/workflows/release.yml

@@ -82,7 +82,7 @@ on:
 # Use env variables to control all the release process.
 env:
   # The arguments of building greptime.
-  RUST_TOOLCHAIN: nightly-2023-08-07
+  RUST_TOOLCHAIN: nightly-2023-12-19
   CARGO_PROFILE: nightly
 
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
@@ -91,7 +91,7 @@ env:
   # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
   # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.5.0
+  NEXT_RELEASE_VERSION: v0.6.0
 
 jobs:
   allocate-runners:

.github/workflows/size-label.yml

@@ -1,11 +1,14 @@
 name: size-labeler
 
-on: [pull_request]
+on: [pull_request_target]
 
 jobs:
   labeler:
     runs-on: ubuntu-latest
     name: Label the PR size
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
       - uses: codelytv/pr-size-labeler@v1
         with:
@@ -18,9 +21,5 @@ jobs:
           l_max_size: '1000'
           xl_label: 'Size: XL'
           fail_if_xl: 'false'
-          message_if_xl: >
-            This PR exceeds the recommended size of 1000 lines.
-            Please make sure you are NOT addressing multiple issues with one PR.
-            Note this PR might be rejected due to its size.
-          github_api_url: 'api.github.com'
+          message_if_xl: ""
           files_to_ignore: 'Cargo.lock'

.github/workflows/user-doc-label-checker.yml

@@ -0,0 +1,19 @@
+name: Check user doc labels
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - labeled
+      - unlabeled
+
+jobs:
+
+  check_labels:
+    name: Check doc labels
+    runs-on: ubuntu-latest
+    steps:
+      - uses: docker://agilepathway/pull-request-label-checker:latest
+        with:
+          one_of: Doc update required,Doc not needed
+          repo_token: ${{ secrets.GITHUB_TOKEN }}

Cargo.toml

@@ -49,20 +49,21 @@ members = [
     "src/servers",
     "src/session",
     "src/sql",
-    "src/storage",
     "src/store-api",
     "src/table",
+    "src/index",
     "tests-integration",
     "tests/runner",
 ]
 resolver = "2"
 
 [workspace.package]
-version = "0.4.3"
+version = "0.5.0"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
+ahash = { version = "0.8", features = ["compile-time-rng"] }
 aquamarine = "0.3"
 arrow = { version = "47.0" }
 arrow-array = "47.0"
@@ -72,7 +73,11 @@ async-stream = "0.3"
 async-trait = "0.1"
 base64 = "0.21"
 bigdecimal = "0.4.2"
+bitflags = "2.4.1"
+bytemuck = "1.12"
+bytes = { version = "1.5", features = ["serde"] }
 chrono = { version = "0.4", features = ["serde"] }
+dashmap = "5.4"
 datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
 datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
 datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
@@ -82,13 +87,15 @@ datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev =
 datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
 derive_builder = "0.12"
 etcd-client = "0.12"
+fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "7eb2e78be7a104d2582fbea0bcb1e019407da702" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "a31ea166fc015ea7ff111ac94e26c3a5d64364d2" }
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
+mockall = "0.11.4"
 moka = "0.12"
 once_cell = "1.18"
 opentelemetry-proto = { git = "https://github.com/waynexia/opentelemetry-rust.git", rev = "33841b38dda79b15f2024952be5f32533325ca02", features = [
@@ -104,25 +111,28 @@ prost = "0.12"
 raft-engine = { git = "https://github.com/tikv/raft-engine.git", rev = "22dfb426cd994602b57725ef080287d3e53db479" }
 rand = "0.8"
 regex = "1.8"
+regex-automata = { version = "0.1", features = ["transducer"] }
 reqwest = { version = "0.11", default-features = false, features = [
     "json",
     "rustls-tls-native-roots",
     "stream",
 ] }
+rskafka = "0.5"
 rust_decimal = "1.33"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 smallvec = "1"
 snafu = "0.7"
 # on branch v0.38.x
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "0fbae07d0c46dc18e3381c406d8b9b8abef6b1fd", features = [
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "6a93567ae38d42be5c8d08b13c8ff4dde26502ef", features = [
     "visitor",
 ] }
 strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
 tokio = { version = "1.28", features = ["full"] }
+tokio-stream = { version = "0.1" }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
-toml = "0.7"
+toml = "0.8.8"
 tonic = { version = "0.10", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
 
@@ -162,7 +172,7 @@ frontend = { path = "src/frontend" }
 log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
 meta-srv = { path = "src/meta-srv" }
-mito = { path = "src/mito" }
+metric-engine = { path = "src/metric-engine" }
 mito2 = { path = "src/mito2" }
 object-store = { path = "src/object-store" }
 operator = { path = "src/operator" }
@@ -174,7 +184,6 @@ script = { path = "src/script" }
 servers = { path = "src/servers" }
 session = { path = "src/session" }
 sql = { path = "src/sql" }
-storage = { path = "src/storage" }
 store-api = { path = "src/store-api" }
 substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }
@@ -184,7 +193,7 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git"
 rev = "abbd357c1e193cd270ea65ee7652334a150b628f"
 
 [profile.release]
-debug = true
+debug = 1
 
 [profile.nightly]
 inherits = "release"

README.md

@@ -27,6 +27,9 @@
     <a href="https://greptime.com/slack"><img src="https://img.shields.io/badge/slack-GreptimeDB-0abd59?logo=slack" alt="slack" /></a>
 </p>
 
+> [!WARNING]  
+> Our default branch has changed from `develop` to `main` (issue [#3025](https://github.com/GreptimeTeam/greptimedb/issues/3025)). Please update your local repository to use the `main` branch.
+
 ## What is GreptimeDB
 
 GreptimeDB is an open-source time-series database with a special focus on
@@ -100,7 +103,7 @@ Please see the online document site for more installation options and [operation
 
 ### Get started
 
-Read the [complete getting started guide](https://docs.greptime.com/getting-started/try-out-greptimedb) on our [official document site](https://docs.greptime.com/).
+Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview) on our [official document site](https://docs.greptime.com/).
 
 To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients/overview).
 
@@ -135,6 +138,7 @@ To write and query data, GreptimeDB is compatible with multiple [protocols and c
 - [GreptimeDB Java Client](https://github.com/GreptimeTeam/greptimedb-client-java)
 - [GreptimeDB Python Client](https://github.com/GreptimeTeam/greptimedb-client-py) (WIP)
 - [GreptimeDB Rust Client](https://github.com/GreptimeTeam/greptimedb-client-rust)
+- [GreptimeDB JavaScript Client](https://github.com/GreptimeTeam/greptime-js-sdk)
 
 ## Project Status
 

benchmarks/src/bin/nyc-taxi.rs

@@ -152,6 +152,7 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
                 .unwrap_or_default(),
             datatype: datatype.into(),
             semantic_type: semantic_type as i32,
+            ..Default::default()
         };
         columns.push(column);
     }
@@ -266,6 +267,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Tag as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "tpep_pickup_datetime".to_string(),
@@ -274,6 +276,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Timestamp as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "tpep_dropoff_datetime".to_string(),
@@ -282,6 +285,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "passenger_count".to_string(),
@@ -290,6 +294,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "trip_distance".to_string(),
@@ -298,6 +303,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "RatecodeID".to_string(),
@@ -306,6 +312,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "store_and_fwd_flag".to_string(),
@@ -314,6 +321,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "PULocationID".to_string(),
@@ -322,6 +330,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "DOLocationID".to_string(),
@@ -330,6 +339,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "payment_type".to_string(),
@@ -338,6 +348,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "fare_amount".to_string(),
@@ -346,6 +357,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "extra".to_string(),
@@ -354,6 +366,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "mta_tax".to_string(),
@@ -362,6 +375,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "tip_amount".to_string(),
@@ -370,6 +384,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "tolls_amount".to_string(),
@@ -378,6 +393,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "improvement_surcharge".to_string(),
@@ -386,6 +402,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "total_amount".to_string(),
@@ -394,6 +411,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "congestion_surcharge".to_string(),
@@ -402,6 +420,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "airport_fee".to_string(),
@@ -410,6 +429,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
         ],
         time_index: "tpep_pickup_datetime".to_string(),

config/datanode.example.toml

@@ -12,6 +12,10 @@ rpc_runtime_size = 8
 # It will block the datanode start if it can't receive leases in the heartbeat from metasrv.
 require_lease_before_startup = false
 
+# Initialize all regions in the background during the startup.
+# By default, it provides services after all regions have been initialized.
+initialize_region_in_background = false
+
 [heartbeat]
 # Interval for sending heartbeat messages to the Metasrv, 3 seconds by default.
 interval = "3s"
@@ -29,9 +33,15 @@ connect_timeout = "1s"
 # `TCP_NODELAY` option for accepted connections, true by default.
 tcp_nodelay = true
 
-# WAL options, see `standalone.example.toml`.
+# WAL options.
+# Currently, users are expected to choose the wal through the provider field. 
+# When a wal provider is chose, the user should comment out all other wal config 
+# except those corresponding to the chosen one.
 [wal]
 # WAL data directory
+provider = "raft_engine"
+
+# Raft-engine wal options, see `standalone.example.toml`.
 # dir = "/tmp/greptimedb/wal"
 file_size = "256MB"
 purge_threshold = "4GB"
@@ -39,10 +49,21 @@ purge_interval = "10m"
 read_batch_size = 128
 sync_write = false
 
+# Kafka wal options, see `standalone.example.toml`.
+# broker_endpoints = ["127.0.0.1:9092"]
+# max_batch_size = "4MB"
+# linger = "200ms"
+# produce_record_timeout = "100ms"
+# backoff_init = "500ms"
+# backoff_max = "10s"
+# backoff_base = 2
+# backoff_deadline = "5mins"
+
 # Storage options, see `standalone.example.toml`.
 [storage]
 # The working home directory.
 data_home = "/tmp/greptimedb/"
+# Storage type.
 type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
@@ -53,32 +74,11 @@ type = "File"
 # The local file cache capacity in bytes.
 # cache_capacity = "256MB"
 
-# Compaction options, see `standalone.example.toml`.
-[storage.compaction]
-max_inflight_tasks = 4
-max_files_in_level0 = 8
-max_purge_tasks = 32
-
-# Storage manifest options
-[storage.manifest]
-# Region checkpoint actions margin.
-# Create a checkpoint every <checkpoint_margin> actions.
-checkpoint_margin = 10
-# Region manifest logs and checkpoints gc execution duration
-gc_duration = '10m'
-
-# Storage flush options
-[storage.flush]
-# Max inflight flush tasks.
-max_flush_tasks = 8
-# Default write buffer size for a region.
-region_write_buffer_size = "32MB"
-# Interval to check whether a region needs flush.
-picker_schedule_interval = "5m"
-# Interval to auto flush a region if it has not flushed yet.
-auto_flush_interval = "1h"
-# Global write buffer size for all regions.
-global_write_buffer_size = "1GB"
+# Custom storage options
+#[[storage.providers]]
+#type = "S3"
+#[[storage.providers]]
+#type = "Gcs"
 
 # Mito engine options
 [[region_engine]]
@@ -91,8 +91,8 @@ worker_channel_size = 128
 worker_request_batch_size = 64
 # Number of meta action updated to trigger a new checkpoint for the manifest
 manifest_checkpoint_distance = 10
-# Manifest compression type
-manifest_compress_type = "Uncompressed"
+# Whether to compress manifest and checkpoint file by gzip (default false).
+compress_manifest = false
 # Max number of running background jobs
 max_background_jobs = 4
 # Interval to auto flush a region if it has not flushed yet.
@@ -105,13 +105,35 @@ global_write_buffer_reject_size = "2GB"
 sst_meta_cache_size = "128MB"
 # Cache size for vectors and arrow arrays (default 512MB). Setting it to 0 to disable the cache.
 vector_cache_size = "512MB"
+# Cache size for pages of SST row groups (default 512MB). Setting it to 0 to disable the cache.
+page_cache_size = "512MB"
 # Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
+# Parallelism to scan a region (default: 1/4 of cpu cores).
+# - 0: using the default value (1/4 of cpu cores).
+# - 1: scan in current thread.
+# - n: scan in parallelism n.
+scan_parallelism = 0
+# Capacity of the channel to send data from parallel scan tasks to the main task (default 32).
+parallel_scan_channel_size = 32
 
-
-# Log options
+# Log options, see `standalone.example.toml`
 # [logging]
-# Specify logs directory.
 # dir = "/tmp/greptimedb/logs"
-# Specify the log level [info | debug | error | warn]
 # level = "info"
+
+# Datanode export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
+# [export_metrics]
+# whether enable export metrics, default is false
+# enable = false
+# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metrics
+# write_interval = "30s"
+# HTTP headers of Prometheus remote-write carry
+# headers = {}

config/frontend.example.toml

@@ -1,5 +1,7 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
+# The default timezone of the server
+# default_timezone = "UTC"
 
 [heartbeat]
 # Interval for sending heartbeat task to the Metasrv, 5 seconds by default.
@@ -77,3 +79,19 @@ tcp_nodelay = true
 timeout = "10s"
 connect_timeout = "10s"
 tcp_nodelay = true
+
+# Frontend export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
+# [export_metrics]
+# whether enable export metrics, default is false
+# enable = false
+# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metrics
+# write_interval = "30s"
+# HTTP headers of Prometheus remote-write carry
+# headers = {}

config/metasrv.example.toml

@@ -7,14 +7,16 @@ server_addr = "127.0.0.1:3002"
 # Etcd server address, "127.0.0.1:2379" by default.
 store_addr = "127.0.0.1:2379"
 # Datanode selector type.
-# - "LeaseBased" (default value).
-# - "LoadBased"
-# For details, please see "https://docs.greptime.com/developer-guide/meta/selector".
-selector = "LeaseBased"
+# - "lease_based" (default value).
+# - "load_based"
+# For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector".
+selector = "lease_based"
 # Store data in memory, false by default.
 use_memory_store = false
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
+# If it's not empty, the metasrv will store all data with this key prefix.
+store_key_prefix = ""
 
 # Log options, see `standalone.example.toml`
 # [logging]
@@ -42,3 +44,53 @@ first_heartbeat_estimate = "1000ms"
 # timeout = "10s"
 # connect_timeout = "10s"
 # tcp_nodelay = true
+
+[wal]
+# Available wal providers:
+# - "raft_engine" (default)
+# - "kafka"
+provider = "raft_engine"
+
+# There're none raft-engine wal config since meta srv only involves in remote wal currently.
+
+# Kafka wal config.
+# The broker endpoints of the Kafka cluster. ["127.0.0.1:9092"] by default.
+# broker_endpoints = ["127.0.0.1:9092"]
+# Number of topics to be created upon start.
+# num_topics = 64
+# Topic selector type.
+# Available selector types: 
+# - "round_robin" (default)
+# selector_type = "round_robin"
+# A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
+# topic_name_prefix = "greptimedb_wal_topic"
+# Number of partitions per topic.
+# num_partitions = 1
+# Expected number of replicas of each partition.
+# replication_factor = 1
+# Above which a topic creation operation will be cancelled.
+# create_topic_timeout = "30s"
+# The initial backoff for kafka clients.
+# backoff_init = "500ms"
+# The maximum backoff for kafka clients.
+# backoff_max = "10s"
+# Exponential backoff rate, i.e. next backoff = base * current backoff.
+# backoff_base = 2
+# Stop reconnecting if the total wait time reaches the deadline. If this config is missing, the reconnecting won't terminate.
+# backoff_deadline = "5mins"
+
+# Metasrv export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
+# [export_metrics]
+# whether enable export metrics, default is false
+# enable = false
+# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metrics
+# write_interval = "30s"
+# HTTP headers of Prometheus remote-write carry
+# headers = {}

config/standalone.example.toml

@@ -2,6 +2,8 @@
 mode = "standalone"
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
+# The default timezone of the server
+# default_timezone = "UTC"
 
 # HTTP server options.
 [http]
@@ -80,8 +82,49 @@ enable = true
 # Whether to enable Prometheus remote write and read in HTTP API, true by default.
 enable = true
 
-# WAL options.
 [wal]
+# Available wal providers:
+# - "raft_engine" (default)
+# - "kafka"
+provider = "raft_engine"
+
+# There're none raft-engine wal config since meta srv only involves in remote wal currently.
+
+# Kafka wal options.
+# The broker endpoints of the Kafka cluster. ["127.0.0.1:9092"] by default.
+# broker_endpoints = ["127.0.0.1:9092"]
+
+# Number of topics to be created upon start.
+# num_topics = 64
+# Topic selector type.
+# Available selector types: 
+# - "round_robin" (default)
+# selector_type = "round_robin"
+# A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
+# topic_name_prefix = "greptimedb_wal_topic"
+# Number of partitions per topic.
+# num_partitions = 1
+# Expected number of replicas of each partition.
+# replication_factor = 1
+
+# The maximum log size a kafka batch producer could buffer.
+# max_batch_size = "4MB"
+# The linger duration of a kafka batch producer.
+# linger = "200ms"
+# The maximum amount of time (in milliseconds) to wait for Kafka records to be returned.
+# produce_record_timeout = "100ms"
+# Above which a topic creation operation will be cancelled.
+# create_topic_timeout = "30s"
+
+# The initial backoff for kafka clients.
+# backoff_init = "500ms"
+# The maximum backoff for kafka clients.
+# backoff_max = "10s"
+# Exponential backoff rate, i.e. next backoff = base * current backoff.
+# backoff_base = 2
+# Stop reconnecting if the total wait time reaches the deadline. If this config is missing, the reconnecting won't terminate.
+# backoff_deadline = "5mins"
+
 # WAL data directory
 # dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
@@ -122,35 +165,48 @@ type = "File"
 # The local file cache capacity in bytes.
 # cache_capacity = "256MB"
 
-# Compaction options.
-[storage.compaction]
-# Max task number that can concurrently run.
-max_inflight_tasks = 4
-# Max files in level 0 to trigger compaction.
-max_files_in_level0 = 8
-# Max task number for SST purge task after compaction.
-max_purge_tasks = 32
+# Custom storage options
+#[[storage.providers]]
+#type = "S3"
+#[[storage.providers]]
+#type = "Gcs"
 
-# Storage manifest options
-[storage.manifest]
-# Region checkpoint actions margin.
-# Create a checkpoint every <checkpoint_margin> actions.
-checkpoint_margin = 10
-# Region manifest logs and checkpoints gc execution duration
-gc_duration = '10m'
-
-# Storage flush options
-[storage.flush]
-# Max inflight flush tasks.
-max_flush_tasks = 8
-# Default write buffer size for a region.
-region_write_buffer_size = "32MB"
-# Interval to check whether a region needs flush.
-picker_schedule_interval = "5m"
+# Mito engine options
+[[region_engine]]
+[region_engine.mito]
+# Number of region workers
+num_workers = 8
+# Request channel size of each worker
+worker_channel_size = 128
+# Max batch size for a worker to handle requests
+worker_request_batch_size = 64
+# Number of meta action updated to trigger a new checkpoint for the manifest
+manifest_checkpoint_distance = 10
+# Whether to compress manifest and checkpoint file by gzip (default false).
+compress_manifest = false
+# Max number of running background jobs
+max_background_jobs = 4
 # Interval to auto flush a region if it has not flushed yet.
 auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
+# Global write buffer size threshold to reject write requests (default 2G).
+global_write_buffer_reject_size = "2GB"
+# Cache size for SST metadata (default 128MB). Setting it to 0 to disable the cache.
+sst_meta_cache_size = "128MB"
+# Cache size for vectors and arrow arrays (default 512MB). Setting it to 0 to disable the cache.
+vector_cache_size = "512MB"
+# Cache size for pages of SST row groups (default 512MB). Setting it to 0 to disable the cache.
+page_cache_size = "512MB"
+# Buffer size for SST writing.
+sst_write_buffer_size = "8MB"
+# Parallelism to scan a region (default: 1/4 of cpu cores).
+# - 0: using the default value (1/4 of cpu cores).
+# - 1: scan in current thread.
+# - n: scan in parallelism n.
+scan_parallelism = 0
+# Capacity of the channel to send data from parallel scan tasks to the main task (default 32).
+parallel_scan_channel_size = 32
 
 # Log options
 # [logging]
@@ -158,3 +214,27 @@ global_write_buffer_size = "1GB"
 # dir = "/tmp/greptimedb/logs"
 # Specify the log level [info | debug | error | warn]
 # level = "info"
+# whether enable tracing, default is false
+# enable_otlp_tracing = false
+# tracing exporter endpoint with format `ip:port`, we use grpc oltp as exporter, default endpoint is `localhost:4317`
+# otlp_endpoint = "localhost:4317"
+# The percentage of tracing will be sampled and exported. Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1. ratio > 1 are treated as 1. Fractions < 0 are treated as 0
+# tracing_sample_ratio = 1.0
+# Whether to append logs to stdout. Defaults to true.
+# append_stdout = true
+
+# Standalone export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
+# [export_metrics]
+# whether enable export metrics, default is false
+# enable = false
+# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metrics
+# write_interval = "30s"
+# HTTP headers of Prometheus remote-write carry
+# headers = {}

docker/dev-builder/centos/Dockerfile

@@ -26,4 +26,5 @@ ARG RUST_TOOLCHAIN
 RUN rustup toolchain install ${RUST_TOOLCHAIN}
 
 # Install nextest.
-RUN cargo install cargo-nextest --locked
+RUN cargo install cargo-binstall --locked
+RUN cargo binstall cargo-nextest --no-confirm

docker/dev-builder/ubuntu/Dockerfile

@@ -43,4 +43,5 @@ ARG RUST_TOOLCHAIN
 RUN rustup toolchain install ${RUST_TOOLCHAIN}
 
 # Install nextest.
-RUN cargo install cargo-nextest --locked
+RUN cargo install cargo-binstall --locked
+RUN cargo binstall cargo-nextest --no-confirm

docker/dev-builder/ubuntu/Dockerfile-18.10

@@ -0,0 +1,48 @@
+# Use the legacy glibc 2.28.
+FROM ubuntu:18.10
+
+ENV LANG en_US.utf8
+WORKDIR /greptimedb
+
+# Use old-releases.ubuntu.com to avoid 404s: https://help.ubuntu.com/community/EOLUpgrades.
+RUN echo "deb http://old-releases.ubuntu.com/ubuntu/ cosmic main restricted universe multiverse\n\
+deb http://old-releases.ubuntu.com/ubuntu/ cosmic-updates main restricted universe multiverse\n\
+deb http://old-releases.ubuntu.com/ubuntu/ cosmic-security main restricted universe multiverse" > /etc/apt/sources.list
+
+# Install dependencies.
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    libssl-dev \
+    tzdata \
+    curl \
+    ca-certificates \
+    git \
+    build-essential \
+    unzip \
+    pkg-config
+
+# Install protoc.
+ENV PROTOC_VERSION=25.1
+RUN if [ "$(uname -m)" = "x86_64" ]; then \
+        PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-x86_64.zip; \
+    elif [ "$(uname -m)" = "aarch64" ]; then \
+        PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-aarch_64.zip; \
+    else \
+        echo "Unsupported architecture"; exit 1; \
+    fi && \
+    curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/${PROTOC_ZIP} && \
+    unzip -o ${PROTOC_ZIP} -d /usr/local bin/protoc && \
+    unzip -o ${PROTOC_ZIP} -d /usr/local 'include/*' && \
+    rm -f ${PROTOC_ZIP}
+
+# Install Rust.
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+
+# Install Rust toolchains.
+ARG RUST_TOOLCHAIN
+RUN rustup toolchain install ${RUST_TOOLCHAIN}
+
+# Install nextest.
+RUN cargo install cargo-binstall --locked
+RUN cargo binstall cargo-nextest --no-confirm

docs/rfcs/2023-12-22-enclose-column-id.md

@@ -0,0 +1,44 @@
+---
+Feature Name: Enclose Column Id
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/2982
+Date: 2023-12-22
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+# Summary
+This RFC proposes to enclose the usage of `ColumnId` into the region engine only.
+
+# Motivation
+`ColumnId` is an identifier for columns. It's assigned by meta server, stored in `TableInfo` and `RegionMetadata` and used in region engine to distinguish columns.
+
+At present, Both Frontend, Datanode and Metasrv are aware of `ColumnId` but it's only used in region engine. Thus this RFC proposes to remove it from Frontend (mainly used in `TableInfo`) and Metasrv.
+
+# Details
+
+`ColumnId` is used widely on both read and write paths. Removing it from Frontend and Metasrv implies several things:
+
+- A column may have different column id in different regions.
+- A column is identified by its name in all components.
+- Column order in the region engine is not restricted, i.e., no need to be in the same order with table info.
+
+The first thing doesn't matter IMO. This concept doesn't exist anymore outside of region server, and each region is autonomous and independent -- the only guarantee it should hold is those columns exist. But if we consider region repartition, where the SST file would be re-assign to different regions, things would become a bit more complicated. A possible solution is store the relation between name and ColumnId in the manifest, but it's out of the scope of this RFC. We can likely give a workaround by introducing a indirection mapping layer of different version of partitions.
+
+And more importantly, we can still assume columns have the same column ids across regions. We have procedure to maintain consistency between regions and the region engine should ensure alterations are idempotent. So it is possible that region repartition doesn't need to consider column ids or other region metadata in the future.
+
+Users write and query column by their names, not by ColumnId or something else. The second point also means to change the column reference in ScanRequest from index to name. This change can hugely alleviate the misuse of the column index, which has given us many surprises.
+
+And for the last one, column order only matters in table info. This order is used in user-faced table structure operation, like add column, describe column or as the default order of INSERT clause. None of them is connected with the order in storage.
+
+# Drawback
+Firstly, this is a breaking change. Delivering this change requires a full upgrade of the cluster. Secondly, this change may introduce some performance regression. For example, we have to pass the full table name in the `ScanRequest` instead of the `ColumnId`. But this influence is very limited, since the column index is only used in the region engine.
+
+# Alternatives
+
+There are two alternatives from the perspective of "what can be used as the column identifier":
+
+- Index of column to the table schema
+- `ColumnId` of that column
+
+The first one is what we are using now. By choosing this way, it's required to keep the column order in the region engine the same as the table info. This is not hard to achieve, but it's a bit annoying. And things become tricky when there is internal column or different schemas like those stored in file format. And this is the initial purpose of this RFC, which is trying to decouple the table schema and region schema.
+
+The second one, in other hand, requires the `ColumnId` should be identical in all regions and `TableInfo`. It has the same drawback with the previous alternative, that the `TableInfo` and `RegionMetadata` are tighted together. Another point is that the `ColumnId` is assigned by the Metasrv, who doesn't need it but have to maintain it. And this also limits the functionality of `ColumnId`, by taking the ability of assigning it from concrete region engine.

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-08-07"
+channel = "nightly-2023-12-19"

src/api/Cargo.toml

@@ -6,11 +6,13 @@ license.workspace = true
 
 [dependencies]
 common-base.workspace = true
+common-decimal.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 common-time.workspace = true
 datatypes.workspace = true
 greptime-proto.workspace = true
+paste = "1.0"
 prost.workspace = true
 snafu.workspace = true
 tonic.workspace = true

src/api/src/helper.rs

@@ -15,6 +15,8 @@
 use std::sync::Arc;
 
 use common_base::BitVec;
+use common_decimal::decimal128::{DECIMAL128_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION};
+use common_decimal::Decimal128;
 use common_time::interval::IntervalUnit;
 use common_time::time::Time;
 use common_time::timestamp::TimeUnit;
@@ -26,47 +28,71 @@ use datatypes::types::{
 };
 use datatypes::value::{OrderedF32, OrderedF64, Value};
 use datatypes::vectors::{
-    BinaryVector, BooleanVector, DateTimeVector, DateVector, DurationMicrosecondVector,
-    DurationMillisecondVector, DurationNanosecondVector, DurationSecondVector, Float32Vector,
-    Float64Vector, Int32Vector, Int64Vector, IntervalDayTimeVector, IntervalMonthDayNanoVector,
-    IntervalYearMonthVector, PrimitiveVector, StringVector, TimeMicrosecondVector,
-    TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector,
-    TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, UInt32Vector,
-    UInt64Vector, VectorRef,
+    BinaryVector, BooleanVector, DateTimeVector, DateVector, Decimal128Vector,
+    DurationMicrosecondVector, DurationMillisecondVector, DurationNanosecondVector,
+    DurationSecondVector, Float32Vector, Float64Vector, Int32Vector, Int64Vector,
+    IntervalDayTimeVector, IntervalMonthDayNanoVector, IntervalYearMonthVector, PrimitiveVector,
+    StringVector, TimeMicrosecondVector, TimeMillisecondVector, TimeNanosecondVector,
+    TimeSecondVector, TimestampMicrosecondVector, TimestampMillisecondVector,
+    TimestampNanosecondVector, TimestampSecondVector, UInt32Vector, UInt64Vector, VectorRef,
 };
+use greptime_proto::v1;
+use greptime_proto::v1::column_data_type_extension::TypeExt;
 use greptime_proto::v1::ddl_request::Expr;
 use greptime_proto::v1::greptime_request::Request;
 use greptime_proto::v1::query_request::Query;
 use greptime_proto::v1::value::ValueData;
-use greptime_proto::v1::{self, DdlRequest, IntervalMonthDayNano, QueryRequest, Row, SemanticType};
+use greptime_proto::v1::{
+    ColumnDataTypeExtension, DdlRequest, DecimalTypeExtension, QueryRequest, Row, SemanticType,
+};
+use paste::paste;
 use snafu::prelude::*;
 
 use crate::error::{self, Result};
 use crate::v1::column::Values;
 use crate::v1::{Column, ColumnDataType, Value as GrpcValue};
 
-#[derive(Debug, PartialEq, Eq)]
-pub struct ColumnDataTypeWrapper(ColumnDataType);
+/// ColumnDataTypeWrapper is a wrapper of ColumnDataType and ColumnDataTypeExtension.
+/// It could be used to convert with ConcreteDataType.
+#[derive(Debug, PartialEq)]
+pub struct ColumnDataTypeWrapper {
+    datatype: ColumnDataType,
+    datatype_ext: Option<ColumnDataTypeExtension>,
+}
 
 impl ColumnDataTypeWrapper {
-    pub fn try_new(datatype: i32) -> Result<Self> {
+    /// Try to create a ColumnDataTypeWrapper from i32(ColumnDataType) and ColumnDataTypeExtension.
+    pub fn try_new(datatype: i32, datatype_ext: Option<ColumnDataTypeExtension>) -> Result<Self> {
         let datatype = ColumnDataType::try_from(datatype)
             .context(error::UnknownColumnDataTypeSnafu { datatype })?;
-        Ok(Self(datatype))
+        Ok(Self {
+            datatype,
+            datatype_ext,
+        })
     }
 
-    pub fn new(datatype: ColumnDataType) -> Self {
-        Self(datatype)
+    /// Create a ColumnDataTypeWrapper from ColumnDataType and ColumnDataTypeExtension.
+    pub fn new(datatype: ColumnDataType, datatype_ext: Option<ColumnDataTypeExtension>) -> Self {
+        Self {
+            datatype,
+            datatype_ext,
+        }
     }
 
+    /// Get the ColumnDataType.
     pub fn datatype(&self) -> ColumnDataType {
-        self.0
+        self.datatype
+    }
+
+    /// Get a tuple of ColumnDataType and ColumnDataTypeExtension.
+    pub fn to_parts(&self) -> (ColumnDataType, Option<ColumnDataTypeExtension>) {
+        (self.datatype, self.datatype_ext.clone())
     }
 }
 
 impl From<ColumnDataTypeWrapper> for ConcreteDataType {
-    fn from(datatype: ColumnDataTypeWrapper) -> Self {
-        match datatype.0 {
+    fn from(datatype_wrapper: ColumnDataTypeWrapper) -> Self {
+        match datatype_wrapper.datatype {
             ColumnDataType::Boolean => ConcreteDataType::boolean_datatype(),
             ColumnDataType::Int8 => ConcreteDataType::int8_datatype(),
             ColumnDataType::Int16 => ConcreteDataType::int16_datatype(),
@@ -109,6 +135,100 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
                 ConcreteDataType::duration_microsecond_datatype()
             }
             ColumnDataType::DurationNanosecond => ConcreteDataType::duration_nanosecond_datatype(),
+            ColumnDataType::Decimal128 => {
+                if let Some(TypeExt::DecimalType(d)) = datatype_wrapper
+                    .datatype_ext
+                    .as_ref()
+                    .and_then(|datatype_ext| datatype_ext.type_ext.as_ref())
+                {
+                    ConcreteDataType::decimal128_datatype(d.precision as u8, d.scale as i8)
+                } else {
+                    ConcreteDataType::decimal128_default_datatype()
+                }
+            }
+        }
+    }
+}
+
+/// This macro is used to generate datatype functions
+/// with lower style for ColumnDataTypeWrapper.
+///
+///
+/// For example: we can use `ColumnDataTypeWrapper::int8_datatype()`,
+/// to get a ColumnDataTypeWrapper with datatype `ColumnDataType::Int8`.
+macro_rules! impl_column_type_functions {
+    ($($Type: ident), +) => {
+        paste! {
+            impl ColumnDataTypeWrapper {
+                $(
+                    pub fn [<$Type:lower _datatype>]() -> ColumnDataTypeWrapper {
+                        ColumnDataTypeWrapper {
+                            datatype: ColumnDataType::$Type,
+                            datatype_ext: None,
+                        }
+                    }
+                )+
+            }
+        }
+    }
+}
+
+/// This macro is used to generate datatype functions
+/// with snake style for ColumnDataTypeWrapper.
+///
+///
+/// For example: we can use `ColumnDataTypeWrapper::duration_second_datatype()`,
+/// to get a ColumnDataTypeWrapper with datatype `ColumnDataType::DurationSecond`.
+macro_rules! impl_column_type_functions_with_snake {
+    ($($TypeName: ident), +) => {
+        paste!{
+            impl ColumnDataTypeWrapper {
+                $(
+                    pub fn [<$TypeName:snake _datatype>]() -> ColumnDataTypeWrapper {
+                        ColumnDataTypeWrapper {
+                            datatype: ColumnDataType::$TypeName,
+                            datatype_ext: None,
+                        }
+                    }
+                )+
+            }
+        }
+    };
+}
+
+impl_column_type_functions!(
+    Boolean, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, Float32, Float64, Binary,
+    Date, Datetime, String
+);
+
+impl_column_type_functions_with_snake!(
+    TimestampSecond,
+    TimestampMillisecond,
+    TimestampMicrosecond,
+    TimestampNanosecond,
+    TimeSecond,
+    TimeMillisecond,
+    TimeMicrosecond,
+    TimeNanosecond,
+    IntervalYearMonth,
+    IntervalDayTime,
+    IntervalMonthDayNano,
+    DurationSecond,
+    DurationMillisecond,
+    DurationMicrosecond,
+    DurationNanosecond
+);
+
+impl ColumnDataTypeWrapper {
+    pub fn decimal128_datatype(precision: i32, scale: i32) -> Self {
+        ColumnDataTypeWrapper {
+            datatype: ColumnDataType::Decimal128,
+            datatype_ext: Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::DecimalType(DecimalTypeExtension {
+                    precision,
+                    scale,
+                })),
+            }),
         }
     }
 }
@@ -117,7 +237,7 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
     type Error = error::Error;
 
     fn try_from(datatype: ConcreteDataType) -> Result<Self> {
-        let datatype = ColumnDataTypeWrapper(match datatype {
+        let column_datatype = match datatype {
             ConcreteDataType::Boolean(_) => ColumnDataType::Boolean,
             ConcreteDataType::Int8(_) => ColumnDataType::Int8,
             ConcreteDataType::Int16(_) => ColumnDataType::Int16,
@@ -156,14 +276,30 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
                 DurationType::Microsecond(_) => ColumnDataType::DurationMicrosecond,
                 DurationType::Nanosecond(_) => ColumnDataType::DurationNanosecond,
             },
+            ConcreteDataType::Decimal128(_) => ColumnDataType::Decimal128,
             ConcreteDataType::Null(_)
             | ConcreteDataType::List(_)
-            | ConcreteDataType::Dictionary(_)
-            | ConcreteDataType::Decimal128(_) => {
+            | ConcreteDataType::Dictionary(_) => {
                 return error::IntoColumnDataTypeSnafu { from: datatype }.fail()
             }
-        });
-        Ok(datatype)
+        };
+        let datatype_extension = match column_datatype {
+            ColumnDataType::Decimal128 => {
+                datatype
+                    .as_decimal128()
+                    .map(|decimal_type| ColumnDataTypeExtension {
+                        type_ext: Some(TypeExt::DecimalType(DecimalTypeExtension {
+                            precision: decimal_type.precision() as i32,
+                            scale: decimal_type.scale() as i32,
+                        })),
+                    })
+            }
+            _ => None,
+        };
+        Ok(Self {
+            datatype: column_datatype,
+            datatype_ext: datatype_extension,
+        })
     }
 }
 
@@ -289,6 +425,10 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
             duration_nanosecond_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
+        ColumnDataType::Decimal128 => Values {
+            decimal128_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
     }
 }
 
@@ -342,7 +482,8 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
             TimeUnit::Microsecond => values.duration_microsecond_values.push(val.value()),
             TimeUnit::Nanosecond => values.duration_nanosecond_values.push(val.value()),
         },
-        Value::List(_) | Value::Decimal128(_) => unreachable!(),
+        Value::Decimal128(val) => values.decimal128_values.push(convert_to_pb_decimal128(val)),
+        Value::List(_) => unreachable!(),
     });
     column.null_mask = null_mask.into_vec();
 }
@@ -382,17 +523,26 @@ fn ddl_request_type(request: &DdlRequest) -> &'static str {
 }
 
 /// Converts an i128 value to google protobuf type [IntervalMonthDayNano].
-pub fn convert_i128_to_interval(v: i128) -> IntervalMonthDayNano {
+pub fn convert_i128_to_interval(v: i128) -> v1::IntervalMonthDayNano {
     let interval = Interval::from_i128(v);
     let (months, days, nanoseconds) = interval.to_month_day_nano();
-    IntervalMonthDayNano {
+    v1::IntervalMonthDayNano {
         months,
         days,
         nanoseconds,
     }
 }
 
-pub fn pb_value_to_value_ref(value: &v1::Value) -> ValueRef {
+/// Convert common decimal128 to grpc decimal128 without precision and scale.
+pub fn convert_to_pb_decimal128(v: Decimal128) -> v1::Decimal128 {
+    let (hi, lo) = v.split_value();
+    v1::Decimal128 { hi, lo }
+}
+
+pub fn pb_value_to_value_ref<'a>(
+    value: &'a v1::Value,
+    datatype_ext: &'a Option<ColumnDataTypeExtension>,
+) -> ValueRef<'a> {
     let Some(value) = &value.value_data else {
         return ValueRef::Null;
     };
@@ -427,9 +577,9 @@ pub fn pb_value_to_value_ref(value: &v1::Value) -> ValueRef {
         ValueData::TimeMillisecondValue(t) => ValueRef::Time(Time::new_millisecond(*t)),
         ValueData::TimeMicrosecondValue(t) => ValueRef::Time(Time::new_microsecond(*t)),
         ValueData::TimeNanosecondValue(t) => ValueRef::Time(Time::new_nanosecond(*t)),
-        ValueData::IntervalYearMonthValues(v) => ValueRef::Interval(Interval::from_i32(*v)),
-        ValueData::IntervalDayTimeValues(v) => ValueRef::Interval(Interval::from_i64(*v)),
-        ValueData::IntervalMonthDayNanoValues(v) => {
+        ValueData::IntervalYearMonthValue(v) => ValueRef::Interval(Interval::from_i32(*v)),
+        ValueData::IntervalDayTimeValue(v) => ValueRef::Interval(Interval::from_i64(*v)),
+        ValueData::IntervalMonthDayNanoValue(v) => {
             let interval = Interval::from_month_day_nano(v.months, v.days, v.nanoseconds);
             ValueRef::Interval(interval)
         }
@@ -437,6 +587,28 @@ pub fn pb_value_to_value_ref(value: &v1::Value) -> ValueRef {
         ValueData::DurationMillisecondValue(v) => ValueRef::Duration(Duration::new_millisecond(*v)),
         ValueData::DurationMicrosecondValue(v) => ValueRef::Duration(Duration::new_microsecond(*v)),
         ValueData::DurationNanosecondValue(v) => ValueRef::Duration(Duration::new_nanosecond(*v)),
+        ValueData::Decimal128Value(v) => {
+            // get precision and scale from datatype_extension
+            if let Some(TypeExt::DecimalType(d)) = datatype_ext
+                .as_ref()
+                .and_then(|column_ext| column_ext.type_ext.as_ref())
+            {
+                ValueRef::Decimal128(Decimal128::from_value_precision_scale(
+                    v.hi,
+                    v.lo,
+                    d.precision as u8,
+                    d.scale as i8,
+                ))
+            } else {
+                // If the precision and scale are not set, use the default value.
+                ValueRef::Decimal128(Decimal128::from_value_precision_scale(
+                    v.hi,
+                    v.lo,
+                    DECIMAL128_MAX_PRECISION,
+                    DECIMAL128_DEFAULT_SCALE,
+                ))
+            }
+        }
     }
 }
 
@@ -523,10 +695,12 @@ pub fn pb_values_to_vector_ref(data_type: &ConcreteDataType, values: Values) ->
                 values.duration_nanosecond_values,
             )),
         },
-        ConcreteDataType::Null(_)
-        | ConcreteDataType::List(_)
-        | ConcreteDataType::Dictionary(_)
-        | ConcreteDataType::Decimal128(_) => {
+        ConcreteDataType::Decimal128(d) => Arc::new(Decimal128Vector::from_values(
+            values.decimal128_values.iter().map(|x| {
+                Decimal128::from_value_precision_scale(x.hi, x.lo, d.precision(), d.scale()).into()
+            }),
+        )),
+        ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
             unreachable!()
         }
     }
@@ -696,10 +870,19 @@ pub fn pb_values_to_values(data_type: &ConcreteDataType, values: Values) -> Vec<
             .into_iter()
             .map(|v| Value::Duration(Duration::new_nanosecond(v)))
             .collect(),
-        ConcreteDataType::Null(_)
-        | ConcreteDataType::List(_)
-        | ConcreteDataType::Dictionary(_)
-        | ConcreteDataType::Decimal128(_) => {
+        ConcreteDataType::Decimal128(d) => values
+            .decimal128_values
+            .into_iter()
+            .map(|v| {
+                Value::Decimal128(Decimal128::from_value_precision_scale(
+                    v.hi,
+                    v.lo,
+                    d.precision(),
+                    d.scale(),
+                ))
+            })
+            .collect(),
+        ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
             unreachable!()
         }
     }
@@ -711,12 +894,14 @@ pub fn is_semantic_type_eq(type_value: i32, semantic_type: SemanticType) -> bool
 }
 
 /// Returns true if the pb type value is valid.
-pub fn is_column_type_value_eq(type_value: i32, expect_type: &ConcreteDataType) -> bool {
-    let Ok(column_type) = ColumnDataType::try_from(type_value) else {
-        return false;
-    };
-
-    is_column_type_eq(column_type, expect_type)
+pub fn is_column_type_value_eq(
+    type_value: i32,
+    type_extension: Option<ColumnDataTypeExtension>,
+    expect_type: &ConcreteDataType,
+) -> bool {
+    ColumnDataTypeWrapper::try_new(type_value, type_extension)
+        .map(|wrapper| ConcreteDataType::from(wrapper) == *expect_type)
+        .unwrap_or(false)
 }
 
 /// Convert value into proto's value.
@@ -798,13 +983,13 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
         },
         Value::Interval(v) => match v.unit() {
             IntervalUnit::YearMonth => v1::Value {
-                value_data: Some(ValueData::IntervalYearMonthValues(v.to_i32())),
+                value_data: Some(ValueData::IntervalYearMonthValue(v.to_i32())),
             },
             IntervalUnit::DayTime => v1::Value {
-                value_data: Some(ValueData::IntervalDayTimeValues(v.to_i64())),
+                value_data: Some(ValueData::IntervalDayTimeValue(v.to_i64())),
             },
             IntervalUnit::MonthDayNano => v1::Value {
-                value_data: Some(ValueData::IntervalMonthDayNanoValues(
+                value_data: Some(ValueData::IntervalMonthDayNanoValue(
                     convert_i128_to_interval(v.to_i128()),
                 )),
             },
@@ -823,13 +1008,16 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
                 value_data: Some(ValueData::DurationNanosecondValue(v.value())),
             },
         },
-        Value::List(_) | Value::Decimal128(_) => return None,
+        Value::Decimal128(v) => v1::Value {
+            value_data: Some(ValueData::Decimal128Value(convert_to_pb_decimal128(v))),
+        },
+        Value::List(_) => return None,
     };
 
     Some(proto_value)
 }
 
-/// Returns the [ColumnDataType] of the value.
+/// Returns the [ColumnDataTypeWrapper] of the value.
 ///
 /// If value is null, returns `None`.
 pub fn proto_value_type(value: &v1::Value) -> Option<ColumnDataType> {
@@ -857,73 +1045,18 @@ pub fn proto_value_type(value: &v1::Value) -> Option<ColumnDataType> {
         ValueData::TimeMillisecondValue(_) => ColumnDataType::TimeMillisecond,
         ValueData::TimeMicrosecondValue(_) => ColumnDataType::TimeMicrosecond,
         ValueData::TimeNanosecondValue(_) => ColumnDataType::TimeNanosecond,
-        ValueData::IntervalYearMonthValues(_) => ColumnDataType::IntervalYearMonth,
-        ValueData::IntervalDayTimeValues(_) => ColumnDataType::IntervalDayTime,
-        ValueData::IntervalMonthDayNanoValues(_) => ColumnDataType::IntervalMonthDayNano,
+        ValueData::IntervalYearMonthValue(_) => ColumnDataType::IntervalYearMonth,
+        ValueData::IntervalDayTimeValue(_) => ColumnDataType::IntervalDayTime,
+        ValueData::IntervalMonthDayNanoValue(_) => ColumnDataType::IntervalMonthDayNano,
         ValueData::DurationSecondValue(_) => ColumnDataType::DurationSecond,
         ValueData::DurationMillisecondValue(_) => ColumnDataType::DurationMillisecond,
         ValueData::DurationMicrosecondValue(_) => ColumnDataType::DurationMicrosecond,
         ValueData::DurationNanosecondValue(_) => ColumnDataType::DurationNanosecond,
+        ValueData::Decimal128Value(_) => ColumnDataType::Decimal128,
     };
     Some(value_type)
 }
 
-/// Convert [ConcreteDataType] to [ColumnDataType].
-pub fn to_column_data_type(data_type: &ConcreteDataType) -> Option<ColumnDataType> {
-    let column_data_type = match data_type {
-        ConcreteDataType::Boolean(_) => ColumnDataType::Boolean,
-        ConcreteDataType::Int8(_) => ColumnDataType::Int8,
-        ConcreteDataType::Int16(_) => ColumnDataType::Int16,
-        ConcreteDataType::Int32(_) => ColumnDataType::Int32,
-        ConcreteDataType::Int64(_) => ColumnDataType::Int64,
-        ConcreteDataType::UInt8(_) => ColumnDataType::Uint8,
-        ConcreteDataType::UInt16(_) => ColumnDataType::Uint16,
-        ConcreteDataType::UInt32(_) => ColumnDataType::Uint32,
-        ConcreteDataType::UInt64(_) => ColumnDataType::Uint64,
-        ConcreteDataType::Float32(_) => ColumnDataType::Float32,
-        ConcreteDataType::Float64(_) => ColumnDataType::Float64,
-        ConcreteDataType::Binary(_) => ColumnDataType::Binary,
-        ConcreteDataType::String(_) => ColumnDataType::String,
-        ConcreteDataType::Date(_) => ColumnDataType::Date,
-        ConcreteDataType::DateTime(_) => ColumnDataType::Datetime,
-        ConcreteDataType::Timestamp(TimestampType::Second(_)) => ColumnDataType::TimestampSecond,
-        ConcreteDataType::Timestamp(TimestampType::Millisecond(_)) => {
-            ColumnDataType::TimestampMillisecond
-        }
-        ConcreteDataType::Timestamp(TimestampType::Microsecond(_)) => {
-            ColumnDataType::TimestampMicrosecond
-        }
-        ConcreteDataType::Timestamp(TimestampType::Nanosecond(_)) => {
-            ColumnDataType::TimestampNanosecond
-        }
-        ConcreteDataType::Time(TimeType::Second(_)) => ColumnDataType::TimeSecond,
-        ConcreteDataType::Time(TimeType::Millisecond(_)) => ColumnDataType::TimeMillisecond,
-        ConcreteDataType::Time(TimeType::Microsecond(_)) => ColumnDataType::TimeMicrosecond,
-        ConcreteDataType::Time(TimeType::Nanosecond(_)) => ColumnDataType::TimeNanosecond,
-        ConcreteDataType::Duration(DurationType::Second(_)) => ColumnDataType::DurationSecond,
-        ConcreteDataType::Duration(DurationType::Millisecond(_)) => {
-            ColumnDataType::DurationMillisecond
-        }
-        ConcreteDataType::Duration(DurationType::Microsecond(_)) => {
-            ColumnDataType::DurationMicrosecond
-        }
-        ConcreteDataType::Duration(DurationType::Nanosecond(_)) => {
-            ColumnDataType::DurationNanosecond
-        }
-        ConcreteDataType::Interval(IntervalType::YearMonth(_)) => ColumnDataType::IntervalYearMonth,
-        ConcreteDataType::Interval(IntervalType::MonthDayNano(_)) => {
-            ColumnDataType::IntervalMonthDayNano
-        }
-        ConcreteDataType::Interval(IntervalType::DayTime(_)) => ColumnDataType::IntervalDayTime,
-        ConcreteDataType::Null(_)
-        | ConcreteDataType::List(_)
-        | ConcreteDataType::Dictionary(_)
-        | ConcreteDataType::Decimal128(_) => return None,
-    };
-
-    Some(column_data_type)
-}
-
 pub fn vectors_to_rows<'a>(
     columns: impl Iterator<Item = &'a VectorRef>,
     row_count: usize,
@@ -970,10 +1103,10 @@ pub fn value_to_grpc_value(value: Value) -> GrpcValue {
                 TimeUnit::Nanosecond => ValueData::TimeNanosecondValue(v.value()),
             }),
             Value::Interval(v) => Some(match v.unit() {
-                IntervalUnit::YearMonth => ValueData::IntervalYearMonthValues(v.to_i32()),
-                IntervalUnit::DayTime => ValueData::IntervalDayTimeValues(v.to_i64()),
+                IntervalUnit::YearMonth => ValueData::IntervalYearMonthValue(v.to_i32()),
+                IntervalUnit::DayTime => ValueData::IntervalDayTimeValue(v.to_i64()),
                 IntervalUnit::MonthDayNano => {
-                    ValueData::IntervalMonthDayNanoValues(convert_i128_to_interval(v.to_i128()))
+                    ValueData::IntervalMonthDayNanoValue(convert_i128_to_interval(v.to_i128()))
                 }
             }),
             Value::Duration(v) => Some(match v.unit() {
@@ -982,20 +1115,12 @@ pub fn value_to_grpc_value(value: Value) -> GrpcValue {
                 TimeUnit::Microsecond => ValueData::DurationMicrosecondValue(v.value()),
                 TimeUnit::Nanosecond => ValueData::DurationNanosecondValue(v.value()),
             }),
-            Value::List(_) | Value::Decimal128(_) => unreachable!(),
+            Value::Decimal128(v) => Some(ValueData::Decimal128Value(convert_to_pb_decimal128(v))),
+            Value::List(_) => unreachable!(),
         },
     }
 }
 
-/// Returns true if the column type is equal to expected type.
-fn is_column_type_eq(column_type: ColumnDataType, expect_type: &ConcreteDataType) -> bool {
-    if let Some(expect) = to_column_data_type(expect_type) {
-        column_type == expect
-    } else {
-        false
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
@@ -1089,189 +1214,204 @@ mod tests {
         let values = values_with_capacity(ColumnDataType::DurationMillisecond, 2);
         let values = values.duration_millisecond_values;
         assert_eq!(2, values.capacity());
+
+        let values = values_with_capacity(ColumnDataType::Decimal128, 2);
+        let values = values.decimal128_values;
+        assert_eq!(2, values.capacity());
     }
 
     #[test]
     fn test_concrete_datatype_from_column_datatype() {
         assert_eq!(
             ConcreteDataType::boolean_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Boolean).into()
+            ColumnDataTypeWrapper::boolean_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::int8_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Int8).into()
+            ColumnDataTypeWrapper::int8_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::int16_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Int16).into()
+            ColumnDataTypeWrapper::int16_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::int32_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Int32).into()
+            ColumnDataTypeWrapper::int32_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::int64_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Int64).into()
+            ColumnDataTypeWrapper::int64_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::uint8_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Uint8).into()
+            ColumnDataTypeWrapper::uint8_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::uint16_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Uint16).into()
+            ColumnDataTypeWrapper::uint16_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::uint32_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Uint32).into()
+            ColumnDataTypeWrapper::uint32_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::uint64_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Uint64).into()
+            ColumnDataTypeWrapper::uint64_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::float32_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Float32).into()
+            ColumnDataTypeWrapper::float32_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::float64_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Float64).into()
+            ColumnDataTypeWrapper::float64_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::binary_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Binary).into()
+            ColumnDataTypeWrapper::binary_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::string_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::String).into()
+            ColumnDataTypeWrapper::string_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::date_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Date).into()
+            ColumnDataTypeWrapper::date_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::datetime_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::Datetime).into()
+            ColumnDataTypeWrapper::datetime_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::timestamp_millisecond_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::TimestampMillisecond).into()
+            ColumnDataTypeWrapper::timestamp_millisecond_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::time_datatype(TimeUnit::Millisecond),
-            ColumnDataTypeWrapper(ColumnDataType::TimeMillisecond).into()
+            ColumnDataTypeWrapper::time_millisecond_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::interval_datatype(IntervalUnit::DayTime),
-            ColumnDataTypeWrapper(ColumnDataType::IntervalDayTime).into()
+            ColumnDataTypeWrapper::interval_day_time_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::interval_datatype(IntervalUnit::YearMonth),
-            ColumnDataTypeWrapper(ColumnDataType::IntervalYearMonth).into()
+            ColumnDataTypeWrapper::interval_year_month_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano),
-            ColumnDataTypeWrapper(ColumnDataType::IntervalMonthDayNano).into()
+            ColumnDataTypeWrapper::interval_month_day_nano_datatype().into()
         );
         assert_eq!(
             ConcreteDataType::duration_millisecond_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::DurationMillisecond).into()
+            ColumnDataTypeWrapper::duration_millisecond_datatype().into()
+        );
+        assert_eq!(
+            ConcreteDataType::decimal128_datatype(10, 2),
+            ColumnDataTypeWrapper::decimal128_datatype(10, 2).into()
         )
     }
 
     #[test]
     fn test_column_datatype_from_concrete_datatype() {
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Boolean),
+            ColumnDataTypeWrapper::boolean_datatype(),
             ConcreteDataType::boolean_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Int8),
+            ColumnDataTypeWrapper::int8_datatype(),
             ConcreteDataType::int8_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Int16),
+            ColumnDataTypeWrapper::int16_datatype(),
             ConcreteDataType::int16_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Int32),
+            ColumnDataTypeWrapper::int32_datatype(),
             ConcreteDataType::int32_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Int64),
+            ColumnDataTypeWrapper::int64_datatype(),
             ConcreteDataType::int64_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Uint8),
+            ColumnDataTypeWrapper::uint8_datatype(),
             ConcreteDataType::uint8_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Uint16),
+            ColumnDataTypeWrapper::uint16_datatype(),
             ConcreteDataType::uint16_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Uint32),
+            ColumnDataTypeWrapper::uint32_datatype(),
             ConcreteDataType::uint32_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Uint64),
+            ColumnDataTypeWrapper::uint64_datatype(),
             ConcreteDataType::uint64_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Float32),
+            ColumnDataTypeWrapper::float32_datatype(),
             ConcreteDataType::float32_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Float64),
+            ColumnDataTypeWrapper::float64_datatype(),
             ConcreteDataType::float64_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Binary),
+            ColumnDataTypeWrapper::binary_datatype(),
             ConcreteDataType::binary_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::String),
+            ColumnDataTypeWrapper::string_datatype(),
             ConcreteDataType::string_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Date),
+            ColumnDataTypeWrapper::date_datatype(),
             ConcreteDataType::date_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::Datetime),
+            ColumnDataTypeWrapper::datetime_datatype(),
             ConcreteDataType::datetime_datatype().try_into().unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::TimestampMillisecond),
+            ColumnDataTypeWrapper::timestamp_millisecond_datatype(),
             ConcreteDataType::timestamp_millisecond_datatype()
                 .try_into()
                 .unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::IntervalYearMonth),
+            ColumnDataTypeWrapper::interval_year_month_datatype(),
             ConcreteDataType::interval_datatype(IntervalUnit::YearMonth)
                 .try_into()
                 .unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::IntervalDayTime),
+            ColumnDataTypeWrapper::interval_day_time_datatype(),
             ConcreteDataType::interval_datatype(IntervalUnit::DayTime)
                 .try_into()
                 .unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::IntervalMonthDayNano),
+            ColumnDataTypeWrapper::interval_month_day_nano_datatype(),
             ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano)
                 .try_into()
                 .unwrap()
         );
         assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::DurationMillisecond),
+            ColumnDataTypeWrapper::duration_millisecond_datatype(),
             ConcreteDataType::duration_millisecond_datatype()
                 .try_into()
                 .unwrap()
         );
 
+        assert_eq!(
+            ColumnDataTypeWrapper::decimal128_datatype(10, 2),
+            ConcreteDataType::decimal128_datatype(10, 2)
+                .try_into()
+                .unwrap()
+        );
+
         let result: Result<ColumnDataTypeWrapper> = ConcreteDataType::null_datatype().try_into();
         assert!(result.is_err());
         assert_eq!(
@@ -1298,6 +1438,7 @@ mod tests {
             }),
             null_mask: vec![],
             datatype: 0,
+            ..Default::default()
         };
 
         let vector = Arc::new(TimestampNanosecondVector::from_vec(vec![1, 2, 3]));
@@ -1339,6 +1480,7 @@ mod tests {
             }),
             null_mask: vec![],
             datatype: 0,
+            ..Default::default()
         };
 
         let vector = Arc::new(TimeNanosecondVector::from_vec(vec![1, 2, 3]));
@@ -1380,6 +1522,7 @@ mod tests {
             }),
             null_mask: vec![],
             datatype: 0,
+            ..Default::default()
         };
 
         let vector = Arc::new(IntervalYearMonthVector::from_vec(vec![1, 2, 3]));
@@ -1424,6 +1567,7 @@ mod tests {
             }),
             null_mask: vec![],
             datatype: 0,
+            ..Default::default()
         };
 
         let vector = Arc::new(DurationNanosecondVector::from_vec(vec![1, 2, 3]));
@@ -1468,6 +1612,7 @@ mod tests {
             }),
             null_mask: vec![2],
             datatype: ColumnDataType::Boolean as i32,
+            ..Default::default()
         };
         let row_count = 4;
 
@@ -1625,17 +1770,17 @@ mod tests {
             &ConcreteDataType::Interval(IntervalType::MonthDayNano(IntervalMonthDayNanoType)),
             Values {
                 interval_month_day_nano_values: vec![
-                    IntervalMonthDayNano {
+                    v1::IntervalMonthDayNano {
                         months: 1,
                         days: 2,
                         nanoseconds: 3,
                     },
-                    IntervalMonthDayNano {
+                    v1::IntervalMonthDayNano {
                         months: 5,
                         days: 6,
                         nanoseconds: 7,
                     },
-                    IntervalMonthDayNano {
+                    v1::IntervalMonthDayNano {
                         months: 9,
                         days: 10,
                         nanoseconds: 11,
@@ -1867,4 +2012,33 @@ mod tests {
         assert_eq!(values[6], ValueData::DateValue(30));
         assert_eq!(values[7], ValueData::StringValue("c".to_string()));
     }
+
+    #[test]
+    fn test_is_column_type_value_eq() {
+        // test column type eq
+        let column1 = Column {
+            column_name: "test".to_string(),
+            semantic_type: 0,
+            values: Some(Values {
+                bool_values: vec![false, true, true],
+                ..Default::default()
+            }),
+            null_mask: vec![2],
+            datatype: ColumnDataType::Boolean as i32,
+            datatype_extension: None,
+        };
+        assert!(is_column_type_value_eq(
+            column1.datatype,
+            column1.datatype_extension,
+            &ConcreteDataType::boolean_datatype(),
+        ));
+    }
+
+    #[test]
+    fn test_convert_to_pb_decimal128() {
+        let decimal = Decimal128::new(123, 3, 1);
+        let pb_decimal = convert_to_pb_decimal128(decimal);
+        assert_eq!(pb_decimal.lo, 123);
+        assert_eq!(pb_decimal.hi, 0);
+    }
 }

src/api/src/v1/column_def.rs

@@ -22,7 +22,10 @@ use crate::helper::ColumnDataTypeWrapper;
 use crate::v1::ColumnDef;
 
 pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
-    let data_type = ColumnDataTypeWrapper::try_new(column_def.data_type)?;
+    let data_type = ColumnDataTypeWrapper::try_new(
+        column_def.data_type,
+        column_def.datatype_extension.clone(),
+    )?;
 
     let constraint = if column_def.default_constraint.is_empty() {
         None

src/catalog/Cargo.toml

@@ -13,6 +13,7 @@ arc-swap = "1.0"
 arrow-schema.workspace = true
 async-stream.workspace = true
 async-trait = "0.1"
+build-data = "0.1"
 common-catalog.workspace = true
 common-error.workspace = true
 common-grpc.workspace = true
@@ -23,7 +24,7 @@ common-recordbatch.workspace = true
 common-runtime.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
-dashmap = "5.4"
+dashmap.workspace = true
 datafusion.workspace = true
 datatypes.workspace = true
 futures = "0.3"
@@ -33,10 +34,11 @@ meta-client.workspace = true
 moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 partition.workspace = true
+paste = "1.0"
 prometheus.workspace = true
 regex.workspace = true
 serde.workspace = true
-serde_json = "1.0"
+serde_json.workspace = true
 session.workspace = true
 snafu.workspace = true
 store-api.workspace = true
@@ -49,5 +51,4 @@ chrono.workspace = true
 common-test-util.workspace = true
 log-store.workspace = true
 object-store.workspace = true
-storage.workspace = true
 tokio.workspace = true

src/catalog/src/error.rs

@@ -180,7 +180,7 @@ pub enum Error {
         source: table::error::Error,
     },
 
-    #[snafu(display(""))]
+    #[snafu(display("Internal error"))]
     Internal {
         location: Location,
         source: BoxedError,
@@ -216,7 +216,7 @@ pub enum Error {
     #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
     QueryAccessDenied { catalog: String, schema: String },
 
-    #[snafu(display(""))]
+    #[snafu(display("DataFusion error"))]
     Datafusion {
         #[snafu(source)]
         error: DataFusionError,

src/catalog/src/information_schema.rs

@@ -13,16 +13,20 @@
 // limitations under the License.
 
 mod columns;
+mod memory_table;
+mod table_names;
 mod tables;
 
 use std::collections::HashMap;
 use std::sync::{Arc, Weak};
 
-use common_catalog::consts::INFORMATION_SCHEMA_NAME;
+use common_catalog::consts::{self, INFORMATION_SCHEMA_NAME};
 use common_error::ext::BoxedError;
-use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
+use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
 use datatypes::schema::SchemaRef;
 use futures_util::StreamExt;
+use lazy_static::lazy_static;
+use paste::paste;
 use snafu::ResultExt;
 use store_api::data_source::DataSource;
 use store_api::storage::{ScanRequest, TableId};
@@ -32,43 +36,102 @@ use table::metadata::{
 };
 use table::thin_table::{ThinTable, ThinTableAdapter};
 use table::TableRef;
+pub use table_names::*;
 
 use self::columns::InformationSchemaColumns;
 use crate::error::Result;
+use crate::information_schema::memory_table::{get_schema_columns, MemoryTable};
 use crate::information_schema::tables::InformationSchemaTables;
 use crate::CatalogManager;
 
-pub const TABLES: &str = "tables";
-pub const COLUMNS: &str = "columns";
+lazy_static! {
+    // Memory tables in `information_schema`.
+    static ref MEMORY_TABLES: &'static [&'static str] = &[
+        ENGINES,
+        COLUMN_PRIVILEGES,
+        COLUMN_STATISTICS,
+        BUILD_INFO,
+    ];
+}
 
+macro_rules! setup_memory_table {
+    ($name: expr) => {
+        paste! {
+            {
+                let (schema, columns) = get_schema_columns($name);
+                Some(Arc::new(MemoryTable::new(
+                    consts::[<INFORMATION_SCHEMA_ $name  _TABLE_ID>],
+                    $name,
+                    schema,
+                    columns
+                )) as _)
+            }
+        }
+    };
+}
+
+/// The `information_schema` tables info provider.
 pub struct InformationSchemaProvider {
     catalog_name: String,
     catalog_manager: Weak<dyn CatalogManager>,
+    tables: HashMap<String, TableRef>,
 }
 
 impl InformationSchemaProvider {
     pub fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
-        Self {
+        let mut provider = Self {
             catalog_name,
             catalog_manager,
-        }
+            tables: HashMap::new(),
+        };
+
+        provider.build_tables();
+
+        provider
     }
 
-    /// Build a map of [TableRef] in information schema.
-    /// Including `tables` and `columns`.
-    pub fn build(
-        catalog_name: String,
-        catalog_manager: Weak<dyn CatalogManager>,
-    ) -> HashMap<String, TableRef> {
-        let provider = Self::new(catalog_name, catalog_manager);
+    /// Returns table names in the order of table id.
+    pub fn table_names(&self) -> Vec<String> {
+        let mut tables = self.tables.values().clone().collect::<Vec<_>>();
 
-        let mut schema = HashMap::new();
-        schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap());
-        schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap());
-        schema
+        tables.sort_by(|t1, t2| {
+            t1.table_info()
+                .table_id()
+                .partial_cmp(&t2.table_info().table_id())
+                .unwrap()
+        });
+        tables
+            .into_iter()
+            .map(|t| t.table_info().name.clone())
+            .collect()
     }
 
+    /// Returns a map of [TableRef] in information schema.
+    pub fn tables(&self) -> &HashMap<String, TableRef> {
+        assert!(!self.tables.is_empty());
+
+        &self.tables
+    }
+
+    /// Returns the [TableRef] by table name.
     pub fn table(&self, name: &str) -> Option<TableRef> {
+        self.tables.get(name).cloned()
+    }
+
+    fn build_tables(&mut self) {
+        let mut tables = HashMap::new();
+        tables.insert(TABLES.to_string(), self.build_table(TABLES).unwrap());
+        tables.insert(COLUMNS.to_string(), self.build_table(COLUMNS).unwrap());
+
+        // Add memory tables
+        for name in MEMORY_TABLES.iter() {
+            tables.insert((*name).to_string(), self.build_table(name).unwrap());
+        }
+
+        self.tables = tables;
+    }
+
+    fn build_table(&self, name: &str) -> Option<TableRef> {
         self.information_table(name).map(|table| {
             let table_info = Self::table_info(self.catalog_name.clone(), &table);
             let filter_pushdown = FilterPushDownType::Unsupported;
@@ -89,6 +152,10 @@ impl InformationSchemaProvider {
                 self.catalog_name.clone(),
                 self.catalog_manager.clone(),
             )) as _),
+            ENGINES => setup_memory_table!(ENGINES),
+            COLUMN_PRIVILEGES => setup_memory_table!(COLUMN_PRIVILEGES),
+            COLUMN_STATISTICS => setup_memory_table!(COLUMN_STATISTICS),
+            BUILD_INFO => setup_memory_table!(BUILD_INFO),
             _ => None,
         }
     }
@@ -102,9 +169,9 @@ impl InformationSchemaProvider {
             .unwrap();
         let table_info = TableInfoBuilder::default()
             .table_id(table.table_id())
-            .name(table.table_name().to_owned())
+            .name(table.table_name().to_string())
             .catalog_name(catalog_name)
-            .schema_name(INFORMATION_SCHEMA_NAME.to_owned())
+            .schema_name(INFORMATION_SCHEMA_NAME.to_string())
             .meta(table_meta)
             .table_type(table.table_type())
             .build()
@@ -171,11 +238,12 @@ impl DataSource for InformationTableDataSource {
                 None => batch,
             });
 
-        let stream = RecordBatchStreamAdaptor {
+        let stream = RecordBatchStreamWrapper {
             schema: projected_schema,
             stream: Box::pin(stream),
             output_ordering: None,
         };
+
         Ok(Box::pin(stream))
     }
 }

src/catalog/src/information_schema/columns.rs

@@ -16,8 +16,8 @@ use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::{
-    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME, SEMANTIC_TYPE_FIELD,
-    SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
+    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_PRIMARY_KEY,
+    SEMANTIC_TYPE_TIME_INDEX,
 };
 use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
@@ -33,8 +33,7 @@ use datatypes::vectors::{StringVectorBuilder, VectorRef};
 use snafu::{OptionExt, ResultExt};
 use store_api::storage::TableId;
 
-use super::tables::InformationSchemaTables;
-use super::{InformationTable, COLUMNS, TABLES};
+use super::{InformationTable, COLUMNS};
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
@@ -102,7 +101,7 @@ impl InformationTable for InformationSchemaColumns {
             schema,
             futures::stream::once(async move {
                 builder
-                    .make_tables()
+                    .make_columns()
                     .await
                     .map(|x| x.into_df_record_batch())
                     .map_err(Into::into)
@@ -148,8 +147,8 @@ impl InformationSchemaColumnsBuilder {
         }
     }
 
-    /// Construct the `information_schema.tables` virtual table
-    async fn make_tables(&mut self) -> Result<RecordBatch> {
+    /// Construct the `information_schema.columns` virtual table
+    async fn make_columns(&mut self) -> Result<RecordBatch> {
         let catalog_name = self.catalog_name.clone();
         let catalog_manager = self
             .catalog_manager
@@ -163,48 +162,38 @@ impl InformationSchemaColumnsBuilder {
             {
                 continue;
             }
+
             for table_name in catalog_manager
                 .table_names(&catalog_name, &schema_name)
                 .await?
             {
-                let (keys, schema) = if let Some(table) = catalog_manager
+                if let Some(table) = catalog_manager
                     .table(&catalog_name, &schema_name, &table_name)
                     .await?
                 {
                     let keys = &table.table_info().meta.primary_key_indices;
                     let schema = table.schema();
-                    (keys.clone(), schema)
-                } else {
-                    // TODO: this specific branch is only a workaround for FrontendCatalogManager.
-                    if schema_name == INFORMATION_SCHEMA_NAME {
-                        if table_name == COLUMNS {
-                            (vec![], InformationSchemaColumns::schema())
-                        } else if table_name == TABLES {
-                            (vec![], InformationSchemaTables::schema())
-                        } else {
-                            continue;
-                        }
-                    } else {
-                        continue;
-                    }
-                };
 
-                for (idx, column) in schema.column_schemas().iter().enumerate() {
-                    let semantic_type = if column.is_time_index() {
-                        SEMANTIC_TYPE_TIME_INDEX
-                    } else if keys.contains(&idx) {
-                        SEMANTIC_TYPE_PRIMARY_KEY
-                    } else {
-                        SEMANTIC_TYPE_FIELD
-                    };
-                    self.add_column(
-                        &catalog_name,
-                        &schema_name,
-                        &table_name,
-                        &column.name,
-                        column.data_type.name(),
-                        semantic_type,
-                    );
+                    for (idx, column) in schema.column_schemas().iter().enumerate() {
+                        let semantic_type = if column.is_time_index() {
+                            SEMANTIC_TYPE_TIME_INDEX
+                        } else if keys.contains(&idx) {
+                            SEMANTIC_TYPE_PRIMARY_KEY
+                        } else {
+                            SEMANTIC_TYPE_FIELD
+                        };
+
+                        self.add_column(
+                            &catalog_name,
+                            &schema_name,
+                            &table_name,
+                            &column.name,
+                            &column.data_type.name(),
+                            semantic_type,
+                        );
+                    }
+                } else {
+                    unreachable!();
                 }
             }
         }
@@ -238,6 +227,7 @@ impl InformationSchemaColumnsBuilder {
             Arc::new(self.data_types.finish()),
             Arc::new(self.semantic_types.finish()),
         ];
+
         RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
     }
 }
@@ -254,7 +244,7 @@ impl DfPartitionStream for InformationSchemaColumns {
             schema,
             futures::stream::once(async move {
                 builder
-                    .make_tables()
+                    .make_columns()
                     .await
                     .map(|x| x.into_df_record_batch())
                     .map_err(Into::into)

src/catalog/src/information_schema/memory_table.rs

@@ -0,0 +1,214 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod tables;
+use std::sync::Arc;
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_error::ext::BoxedError;
+use common_query::physical_plan::TaskContext;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
+use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
+use datatypes::schema::SchemaRef;
+use datatypes::vectors::VectorRef;
+use snafu::ResultExt;
+use store_api::storage::TableId;
+pub use tables::get_schema_columns;
+
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
+use crate::information_schema::InformationTable;
+
+/// A memory table with specified schema and columns.
+pub(super) struct MemoryTable {
+    table_id: TableId,
+    table_name: &'static str,
+    schema: SchemaRef,
+    columns: Vec<VectorRef>,
+}
+
+impl MemoryTable {
+    /// Creates a memory table with table id, name, schema and columns.
+    pub(super) fn new(
+        table_id: TableId,
+        table_name: &'static str,
+        schema: SchemaRef,
+        columns: Vec<VectorRef>,
+    ) -> Self {
+        Self {
+            table_id,
+            table_name,
+            schema,
+            columns,
+        }
+    }
+
+    fn builder(&self) -> MemoryTableBuilder {
+        MemoryTableBuilder::new(self.schema.clone(), self.columns.clone())
+    }
+}
+
+impl InformationTable for MemoryTable {
+    fn table_id(&self) -> TableId {
+        self.table_id
+    }
+
+    fn table_name(&self) -> &'static str {
+        self.table_name
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .memory_records()
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+struct MemoryTableBuilder {
+    schema: SchemaRef,
+    columns: Vec<VectorRef>,
+}
+
+impl MemoryTableBuilder {
+    fn new(schema: SchemaRef, columns: Vec<VectorRef>) -> Self {
+        Self { schema, columns }
+    }
+
+    /// Construct the `information_schema.{table_name}` virtual table
+    async fn memory_records(&mut self) -> Result<RecordBatch> {
+        if self.columns.is_empty() {
+            RecordBatch::new_empty(self.schema.clone()).context(CreateRecordBatchSnafu)
+        } else {
+            RecordBatch::new(self.schema.clone(), std::mem::take(&mut self.columns))
+                .context(CreateRecordBatchSnafu)
+        }
+    }
+}
+
+impl DfPartitionStream for MemoryTable {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .memory_records()
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_recordbatch::RecordBatches;
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::schema::{ColumnSchema, Schema};
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_memory_table() {
+        let schema = Arc::new(Schema::new(vec![
+            ColumnSchema::new("a", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("b", ConcreteDataType::string_datatype(), false),
+        ]));
+
+        let table = MemoryTable::new(
+            42,
+            "test",
+            schema.clone(),
+            vec![
+                Arc::new(StringVector::from(vec!["a1", "a2"])),
+                Arc::new(StringVector::from(vec!["b1", "b2"])),
+            ],
+        );
+
+        assert_eq!(42, table.table_id());
+        assert_eq!("test", table.table_name());
+        assert_eq!(schema, InformationTable::schema(&table));
+
+        let stream = table.to_stream().unwrap();
+
+        let batches = RecordBatches::try_collect(stream).await.unwrap();
+
+        assert_eq!(
+            "\
++----+----+
+| a  | b  |
++----+----+
+| a1 | b1 |
+| a2 | b2 |
++----+----+",
+            batches.pretty_print().unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_empty_memory_table() {
+        let schema = Arc::new(Schema::new(vec![
+            ColumnSchema::new("a", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("b", ConcreteDataType::string_datatype(), false),
+        ]));
+
+        let table = MemoryTable::new(42, "test", schema.clone(), vec![]);
+
+        assert_eq!(42, table.table_id());
+        assert_eq!("test", table.table_name());
+        assert_eq!(schema, InformationTable::schema(&table));
+
+        let stream = table.to_stream().unwrap();
+
+        let batches = RecordBatches::try_collect(stream).await.unwrap();
+
+        assert_eq!(
+            "\
++---+---+
+| a | b |
++---+---+
++---+---+",
+            batches.pretty_print().unwrap()
+        );
+    }
+}

src/catalog/src/information_schema/memory_table/tables.rs

@@ -0,0 +1,135 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use common_catalog::consts::MITO_ENGINE;
+use datatypes::prelude::{ConcreteDataType, VectorRef};
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::vectors::StringVector;
+
+use crate::information_schema::table_names::*;
+
+const UNKNOWN: &str = "unknown";
+
+/// Find the schema and columns by the table_name, only valid for memory tables.
+/// Safety: the user MUST ensure the table schema exists, panic otherwise.
+pub fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec<VectorRef>) {
+    let (column_schemas, columns): (_, Vec<VectorRef>) = match table_name {
+        COLUMN_PRIVILEGES => (
+            string_columns(&[
+                "GRANTEE",
+                "TABLE_CATALOG",
+                "TABLE_SCHEMA",
+                "TABLE_NAME",
+                "COLUMN_NAME",
+                "PRIVILEGE_TYPE",
+                "IS_GRANTABLE",
+            ]),
+            vec![],
+        ),
+
+        COLUMN_STATISTICS => (
+            string_columns(&[
+                "SCHEMA_NAME",
+                "TABLE_NAME",
+                "COLUMN_NAME",
+                // TODO(dennis): It must be a JSON type, but we don't support it yet
+                "HISTOGRAM",
+            ]),
+            vec![],
+        ),
+
+        ENGINES => (
+            string_columns(&[
+                "ENGINE",
+                "SUPPORT",
+                "COMMENT",
+                "TRANSACTIONS",
+                "XA",
+                "SAVEPOINTS",
+            ]),
+            vec![
+                Arc::new(StringVector::from(vec![MITO_ENGINE])),
+                Arc::new(StringVector::from(vec!["DEFAULT"])),
+                Arc::new(StringVector::from(vec![
+                    "Storage engine for time-series data",
+                ])),
+                Arc::new(StringVector::from(vec!["NO"])),
+                Arc::new(StringVector::from(vec!["NO"])),
+                Arc::new(StringVector::from(vec!["NO"])),
+            ],
+        ),
+
+        BUILD_INFO => (
+            string_columns(&[
+                "GIT_BRANCH",
+                "GIT_COMMIT",
+                "GIT_COMMIT_SHORT",
+                "GIT_DIRTY",
+                "PKG_VERSION",
+            ]),
+            vec![
+                Arc::new(StringVector::from(vec![
+                    build_data::get_git_branch().unwrap_or_else(|_| UNKNOWN.to_string())
+                ])),
+                Arc::new(StringVector::from(vec![
+                    build_data::get_git_commit().unwrap_or_else(|_| UNKNOWN.to_string())
+                ])),
+                Arc::new(StringVector::from(vec![
+                    build_data::get_git_commit_short().unwrap_or_else(|_| UNKNOWN.to_string())
+                ])),
+                Arc::new(StringVector::from(vec![
+                    build_data::get_git_dirty().map_or(UNKNOWN.to_string(), |v| v.to_string())
+                ])),
+                Arc::new(StringVector::from(vec![option_env!("CARGO_PKG_VERSION")])),
+            ],
+        ),
+
+        _ => unreachable!("Unknown table in information_schema: {}", table_name),
+    };
+
+    (Arc::new(Schema::new(column_schemas)), columns)
+}
+
+fn string_columns(names: &[&'static str]) -> Vec<ColumnSchema> {
+    names.iter().map(|name| string_column(name)).collect()
+}
+
+fn string_column(name: &str) -> ColumnSchema {
+    ColumnSchema::new(
+        str::to_lowercase(name),
+        ConcreteDataType::string_datatype(),
+        false,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_string_columns() {
+        let columns = ["a", "b", "c"];
+        let column_schemas = string_columns(&columns);
+
+        assert_eq!(3, column_schemas.len());
+        for (i, name) in columns.iter().enumerate() {
+            let cs = column_schemas.get(i).unwrap();
+
+            assert_eq!(*name, cs.name);
+            assert_eq!(ConcreteDataType::string_datatype(), cs.data_type);
+        }
+    }
+}

src/catalog/src/information_schema/table_names.rs

@@ -12,15 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! manifest storage
-pub(crate) mod action;
-pub mod checkpoint;
-pub mod helper;
-mod impl_;
-pub mod region;
-pub(crate) mod storage;
-#[cfg(test)]
-pub mod test_utils;
+/// All table names in `information_schema`.
 
-pub use self::impl_::*;
-pub use self::storage::manifest_compress_type;
+pub const TABLES: &str = "tables";
+pub const COLUMNS: &str = "columns";
+pub const ENGINES: &str = "engines";
+pub const COLUMN_PRIVILEGES: &str = "column_privileges";
+pub const COLUMN_STATISTICS: &str = "column_statistics";
+pub const BUILD_INFO: &str = "build_info";

src/catalog/src/information_schema/tables.rs

@@ -15,10 +15,7 @@
 use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
-use common_catalog::consts::{
-    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME,
-    INFORMATION_SCHEMA_TABLES_TABLE_ID,
-};
+use common_catalog::consts::INFORMATION_SCHEMA_TABLES_TABLE_ID;
 use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
@@ -33,7 +30,7 @@ use snafu::{OptionExt, ResultExt};
 use store_api::storage::TableId;
 use table::metadata::TableType;
 
-use super::{COLUMNS, TABLES};
+use super::TABLES;
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
@@ -178,29 +175,8 @@ impl InformationSchemaTablesBuilder {
                         Some(&table_info.meta.engine),
                     );
                 } else {
-                    // TODO: this specific branch is only a workaround for FrontendCatalogManager.
-                    if schema_name == INFORMATION_SCHEMA_NAME {
-                        if table_name == COLUMNS {
-                            self.add_table(
-                                &catalog_name,
-                                &schema_name,
-                                &table_name,
-                                TableType::Temporary,
-                                Some(INFORMATION_SCHEMA_COLUMNS_TABLE_ID),
-                                None,
-                            );
-                        } else if table_name == TABLES {
-                            self.add_table(
-                                &catalog_name,
-                                &schema_name,
-                                &table_name,
-                                TableType::Temporary,
-                                Some(INFORMATION_SCHEMA_TABLES_TABLE_ID),
-                                None,
-                            );
-                        }
-                    }
-                };
+                    unreachable!();
+                }
             }
         }
 

src/catalog/src/kvbackend/manager.rs

@@ -19,7 +19,6 @@ use std::sync::{Arc, Weak};
 use common_catalog::consts::{DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, NUMBERS_TABLE_ID};
 use common_error::ext::BoxedError;
 use common_meta::cache_invalidator::{CacheInvalidator, CacheInvalidatorRef, Context};
-use common_meta::datanode_manager::DatanodeManagerRef;
 use common_meta::error::Result as MetaResult;
 use common_meta::key::catalog_name::CatalogNameKey;
 use common_meta::key::schema_name::SchemaNameKey;
@@ -39,7 +38,7 @@ use crate::error::{
     self as catalog_err, ListCatalogsSnafu, ListSchemasSnafu, Result as CatalogResult,
     TableMetadataManagerSnafu,
 };
-use crate::information_schema::{InformationSchemaProvider, COLUMNS, TABLES};
+use crate::information_schema::InformationSchemaProvider;
 use crate::CatalogManager;
 
 /// Access all existing catalog, schema and tables.
@@ -55,7 +54,6 @@ pub struct KvBackendCatalogManager {
     cache_invalidator: CacheInvalidatorRef,
     partition_manager: PartitionRuleManagerRef,
     table_metadata_manager: TableMetadataManagerRef,
-    datanode_manager: DatanodeManagerRef,
     /// A sub-CatalogManager that handles system tables
     system_catalog: SystemCatalog,
 }
@@ -76,18 +74,18 @@ impl CacheInvalidator for KvBackendCatalogManager {
 }
 
 impl KvBackendCatalogManager {
-    pub fn new(
-        backend: KvBackendRef,
-        cache_invalidator: CacheInvalidatorRef,
-        datanode_manager: DatanodeManagerRef,
-    ) -> Arc<Self> {
+    pub fn new(backend: KvBackendRef, cache_invalidator: CacheInvalidatorRef) -> Arc<Self> {
         Arc::new_cyclic(|me| Self {
             partition_manager: Arc::new(PartitionRuleManager::new(backend.clone())),
             table_metadata_manager: Arc::new(TableMetadataManager::new(backend)),
             cache_invalidator,
-            datanode_manager,
             system_catalog: SystemCatalog {
                 catalog_manager: me.clone(),
+                information_schema_provider: Arc::new(InformationSchemaProvider::new(
+                    // The catalog name is not used in system_catalog, so let it empty
+                    "".to_string(),
+                    me.clone(),
+                )),
             },
         })
     }
@@ -99,10 +97,6 @@ impl KvBackendCatalogManager {
     pub fn table_metadata_manager_ref(&self) -> &TableMetadataManagerRef {
         &self.table_metadata_manager
     }
-
-    pub fn datanode_manager(&self) -> DatanodeManagerRef {
-        self.datanode_manager.clone()
-    }
 }
 
 #[async_trait::async_trait]
@@ -133,13 +127,11 @@ impl CatalogManager for KvBackendCatalogManager {
             .try_collect::<BTreeSet<_>>()
             .await
             .map_err(BoxedError::new)
-            .context(ListSchemasSnafu { catalog })?
-            .into_iter()
-            .collect::<Vec<_>>();
+            .context(ListSchemasSnafu { catalog })?;
 
-        keys.extend_from_slice(&self.system_catalog.schema_names());
+        keys.extend(self.system_catalog.schema_names());
 
-        Ok(keys)
+        Ok(keys.into_iter().collect())
     }
 
     async fn table_names(&self, catalog: &str, schema: &str) -> CatalogResult<Vec<String>> {
@@ -242,11 +234,11 @@ impl CatalogManager for KvBackendCatalogManager {
 // a new catalog is created.
 /// Existing system tables:
 /// - public.numbers
-/// - information_schema.tables
-/// - information_schema.columns
+/// - information_schema.{tables}
 #[derive(Clone)]
 struct SystemCatalog {
     catalog_manager: Weak<KvBackendCatalogManager>,
+    information_schema_provider: Arc<InformationSchemaProvider>,
 }
 
 impl SystemCatalog {
@@ -256,7 +248,7 @@ impl SystemCatalog {
 
     fn table_names(&self, schema: &str) -> Vec<String> {
         if schema == INFORMATION_SCHEMA_NAME {
-            vec![TABLES.to_string(), COLUMNS.to_string()]
+            self.information_schema_provider.table_names()
         } else if schema == DEFAULT_SCHEMA_NAME {
             vec![NUMBERS_TABLE_NAME.to_string()]
         } else {
@@ -270,7 +262,7 @@ impl SystemCatalog {
 
     fn table_exist(&self, schema: &str, table: &str) -> bool {
         if schema == INFORMATION_SCHEMA_NAME {
-            table == TABLES || table == COLUMNS
+            self.information_schema_provider.table(table).is_some()
         } else if schema == DEFAULT_SCHEMA_NAME {
             table == NUMBERS_TABLE_NAME
         } else {

src/catalog/src/lib.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(trait_upcasting)]
 #![feature(assert_matches)]
 #![feature(try_blocks)]
 

src/catalog/src/memory/manager.rs

@@ -18,7 +18,9 @@ use std::collections::HashMap;
 use std::sync::{Arc, RwLock, Weak};
 
 use common_catalog::build_db_string;
-use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_PRIVATE_SCHEMA_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME,
+};
 use snafu::OptionExt;
 use table::TableRef;
 
@@ -135,6 +137,18 @@ impl MemoryCatalogManager {
                 schema: DEFAULT_SCHEMA_NAME.to_string(),
             })
             .unwrap();
+        manager
+            .register_schema_sync(RegisterSchemaRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_PRIVATE_SCHEMA_NAME.to_string(),
+            })
+            .unwrap();
+        manager
+            .register_schema_sync(RegisterSchemaRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: INFORMATION_SCHEMA_NAME.to_string(),
+            })
+            .unwrap();
 
         manager
     }
@@ -243,10 +257,12 @@ impl MemoryCatalogManager {
     }
 
     fn create_catalog_entry(self: &Arc<Self>, catalog: String) -> SchemaEntries {
-        let information_schema = InformationSchemaProvider::build(
+        let information_schema_provider = InformationSchemaProvider::new(
             catalog,
             Arc::downgrade(self) as Weak<dyn CatalogManager>,
         );
+        let information_schema = information_schema_provider.tables().clone();
+
         let mut catalog = HashMap::new();
         catalog.insert(INFORMATION_SCHEMA_NAME.to_string(), information_schema);
         catalog

src/client/Cargo.toml

@@ -35,7 +35,7 @@ prost.workspace = true
 rand.workspace = true
 session.workspace = true
 snafu.workspace = true
-tokio-stream = { version = "0.1", features = ["net"] }
+tokio-stream = { workspace = true, features = ["net"] }
 tokio.workspace = true
 tonic.workspace = true
 

src/client/examples/logical.rs

@@ -46,6 +46,7 @@ async fn run() {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Timestamp as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "key".to_string(),
@@ -54,6 +55,7 @@ async fn run() {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Tag as i32,
                 comment: String::new(),
+                ..Default::default()
             },
             ColumnDef {
                 name: "value".to_string(),
@@ -62,6 +64,7 @@ async fn run() {
                 default_constraint: vec![],
                 semantic_type: SemanticType::Field as i32,
                 comment: String::new(),
+                ..Default::default()
             },
         ],
         time_index: "timestamp".to_string(),
@@ -78,7 +81,7 @@ async fn run() {
 
     let logical = mock_logical_plan();
     event!(Level::INFO, "plan size: {:#?}", logical.len());
-    let result = db.logical_plan(logical, 0).await.unwrap();
+    let result = db.logical_plan(logical).await.unwrap();
 
     event!(Level::INFO, "result: {:#?}", result);
 }

src/client/src/database.rs

@@ -27,8 +27,9 @@ use common_error::ext::{BoxedError, ErrorExt};
 use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_query::Output;
 use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::RecordBatchStreamAdaptor;
+use common_recordbatch::RecordBatchStreamWrapper;
 use common_telemetry::logging;
+use common_telemetry::tracing_context::W3cTrace;
 use futures_util::StreamExt;
 use prost::Message;
 use snafu::{ensure, ResultExt};
@@ -147,21 +148,21 @@ impl Database {
 
     async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = self.client.make_database_client()?.inner;
-        let request = self.to_rpc_request(request, 0);
+        let request = self.to_rpc_request(request);
         let response = client.handle(request).await?.into_inner();
         from_grpc_response(response)
     }
 
     #[inline]
-    fn to_rpc_request(&self, request: Request, trace_id: u64) -> GreptimeRequest {
+    fn to_rpc_request(&self, request: Request) -> GreptimeRequest {
         GreptimeRequest {
             header: Some(RequestHeader {
                 catalog: self.catalog.clone(),
                 schema: self.schema.clone(),
                 authorization: self.ctx.auth_header.clone(),
                 dbname: self.dbname.clone(),
-                trace_id,
-                span_id: 0,
+                // TODO(Taylor-lagrange): add client grpc tracing
+                tracing_context: W3cTrace::new(),
             }),
             request: Some(request),
         }
@@ -172,23 +173,17 @@ impl Database {
         S: AsRef<str>,
     {
         let _timer = metrics::METRIC_GRPC_SQL.start_timer();
-        self.do_get(
-            Request::Query(QueryRequest {
-                query: Some(Query::Sql(sql.as_ref().to_string())),
-            }),
-            0,
-        )
+        self.do_get(Request::Query(QueryRequest {
+            query: Some(Query::Sql(sql.as_ref().to_string())),
+        }))
         .await
     }
 
-    pub async fn logical_plan(&self, logical_plan: Vec<u8>, trace_id: u64) -> Result<Output> {
+    pub async fn logical_plan(&self, logical_plan: Vec<u8>) -> Result<Output> {
         let _timer = metrics::METRIC_GRPC_LOGICAL_PLAN.start_timer();
-        self.do_get(
-            Request::Query(QueryRequest {
-                query: Some(Query::LogicalPlan(logical_plan)),
-            }),
-            trace_id,
-        )
+        self.do_get(Request::Query(QueryRequest {
+            query: Some(Query::LogicalPlan(logical_plan)),
+        }))
         .await
     }
 
@@ -200,68 +195,53 @@ impl Database {
         step: &str,
     ) -> Result<Output> {
         let _timer = metrics::METRIC_GRPC_PROMQL_RANGE_QUERY.start_timer();
-        self.do_get(
-            Request::Query(QueryRequest {
-                query: Some(Query::PromRangeQuery(PromRangeQuery {
-                    query: promql.to_string(),
-                    start: start.to_string(),
-                    end: end.to_string(),
-                    step: step.to_string(),
-                })),
-            }),
-            0,
-        )
+        self.do_get(Request::Query(QueryRequest {
+            query: Some(Query::PromRangeQuery(PromRangeQuery {
+                query: promql.to_string(),
+                start: start.to_string(),
+                end: end.to_string(),
+                step: step.to_string(),
+            })),
+        }))
         .await
     }
 
     pub async fn create(&self, expr: CreateTableExpr) -> Result<Output> {
         let _timer = metrics::METRIC_GRPC_CREATE_TABLE.start_timer();
-        self.do_get(
-            Request::Ddl(DdlRequest {
-                expr: Some(DdlExpr::CreateTable(expr)),
-            }),
-            0,
-        )
+        self.do_get(Request::Ddl(DdlRequest {
+            expr: Some(DdlExpr::CreateTable(expr)),
+        }))
         .await
     }
 
     pub async fn alter(&self, expr: AlterExpr) -> Result<Output> {
         let _timer = metrics::METRIC_GRPC_ALTER.start_timer();
-        self.do_get(
-            Request::Ddl(DdlRequest {
-                expr: Some(DdlExpr::Alter(expr)),
-            }),
-            0,
-        )
+        self.do_get(Request::Ddl(DdlRequest {
+            expr: Some(DdlExpr::Alter(expr)),
+        }))
         .await
     }
 
     pub async fn drop_table(&self, expr: DropTableExpr) -> Result<Output> {
         let _timer = metrics::METRIC_GRPC_DROP_TABLE.start_timer();
-        self.do_get(
-            Request::Ddl(DdlRequest {
-                expr: Some(DdlExpr::DropTable(expr)),
-            }),
-            0,
-        )
+        self.do_get(Request::Ddl(DdlRequest {
+            expr: Some(DdlExpr::DropTable(expr)),
+        }))
         .await
     }
 
     pub async fn truncate_table(&self, expr: TruncateTableExpr) -> Result<Output> {
         let _timer = metrics::METRIC_GRPC_TRUNCATE_TABLE.start_timer();
-        self.do_get(
-            Request::Ddl(DdlRequest {
-                expr: Some(DdlExpr::TruncateTable(expr)),
-            }),
-            0,
-        )
+        self.do_get(Request::Ddl(DdlRequest {
+            expr: Some(DdlExpr::TruncateTable(expr)),
+        }))
         .await
     }
 
-    async fn do_get(&self, request: Request, trace_id: u64) -> Result<Output> {
+    async fn do_get(&self, request: Request) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
         let _timer = metrics::METRIC_GRPC_DO_GET.start_timer();
-        let request = self.to_rpc_request(request, trace_id);
+        let request = self.to_rpc_request(request);
         let request = Ticket {
             ticket: request.encode_to_vec().into(),
         };
@@ -335,7 +315,7 @@ impl Database {
                         yield Ok(record_batch);
                     }
                 }));
-                let record_batch_stream = RecordBatchStreamAdaptor {
+                let record_batch_stream = RecordBatchStreamWrapper {
                     schema,
                     stream,
                     output_ordering: None,

src/client/src/error.rs

@@ -131,3 +131,15 @@ impl From<Status> for Error {
         Self::Server { code, msg }
     }
 }
+
+impl Error {
+    pub fn should_retry(&self) -> bool {
+        !matches!(
+            self,
+            Self::RegionServer {
+                code: Code::InvalidArgument,
+                ..
+            }
+        )
+    }
+}

src/client/src/region.rs

@@ -23,13 +23,12 @@ use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_meta::datanode_manager::{AffectedRows, Datanode};
 use common_meta::error::{self as meta_error, Result as MetaResult};
 use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
+use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
 use common_telemetry::error;
 use prost::Message;
 use snafu::{location, Location, OptionExt, ResultExt};
 use tokio_stream::StreamExt;
 
-use crate::error::Error::RegionServer;
 use crate::error::{
     self, ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
     MissingFieldSnafu, Result, ServerSnafu,
@@ -45,7 +44,7 @@ pub struct RegionRequester {
 impl Datanode for RegionRequester {
     async fn handle(&self, request: RegionRequest) -> MetaResult<AffectedRows> {
         self.handle_inner(request).await.map_err(|err| {
-            if matches!(err, RegionServer { .. }) {
+            if err.should_retry() {
                 meta_error::Error::RetryLater {
                     source: BoxedError::new(err),
                 }
@@ -137,7 +136,7 @@ impl RegionRequester {
                 yield Ok(record_batch);
             }
         }));
-        let record_batch_stream = RecordBatchStreamAdaptor {
+        let record_batch_stream = RecordBatchStreamWrapper {
             schema,
             stream,
             output_ordering: None,

src/cmd/Cargo.toml

@@ -18,7 +18,7 @@ async-trait.workspace = true
 auth.workspace = true
 catalog.workspace = true
 chrono.workspace = true
-clap = { version = "3.1", features = ["derive"] }
+clap = { version = "4.4", features = ["derive"] }
 client.workspace = true
 common-base.workspace = true
 common-catalog.workspace = true
@@ -32,6 +32,7 @@ common-recordbatch.workspace = true
 common-telemetry = { workspace = true, features = [
     "deadlock_detection",
 ] }
+common-time.workspace = true
 config = "0.13"
 datanode.workspace = true
 datatypes.workspace = true
@@ -40,6 +41,7 @@ etcd-client.workspace = true
 file-engine.workspace = true
 frontend.workspace = true
 futures.workspace = true
+human-panic = "1.2.2"
 lazy_static.workspace = true
 meta-client.workspace = true
 meta-srv.workspace = true
@@ -58,6 +60,7 @@ serde_json.workspace = true
 servers.workspace = true
 session.workspace = true
 snafu.workspace = true
+store-api.workspace = true
 substrait.workspace = true
 table.workspace = true
 tokio.workspace = true

src/cmd/src/bin/greptime.rs

@@ -16,79 +16,12 @@
 
 use std::fmt;
 
-use clap::Parser;
+use clap::{FromArgMatches, Parser, Subcommand};
 use cmd::error::Result;
-use cmd::options::{Options, TopLevelOptions};
-use cmd::{cli, datanode, frontend, metasrv, standalone};
-use common_telemetry::logging::{error, info, TracingOptions};
-
-lazy_static::lazy_static! {
-    static ref APP_VERSION: prometheus::IntGaugeVec =
-        prometheus::register_int_gauge_vec!("app_version", "app version", &["short_version", "version"]).unwrap();
-}
-
-#[derive(Parser)]
-#[clap(name = "greptimedb", version = print_version())]
-struct Command {
-    #[clap(long)]
-    log_dir: Option<String>,
-    #[clap(long)]
-    log_level: Option<String>,
-    #[clap(subcommand)]
-    subcmd: SubCommand,
-
-    #[cfg(feature = "tokio-console")]
-    #[clap(long)]
-    tokio_console_addr: Option<String>,
-}
-
-pub enum Application {
-    Datanode(datanode::Instance),
-    Frontend(frontend::Instance),
-    Metasrv(metasrv::Instance),
-    Standalone(standalone::Instance),
-    Cli(cli::Instance),
-}
-
-impl Application {
-    async fn start(&mut self) -> Result<()> {
-        match self {
-            Application::Datanode(instance) => instance.start().await,
-            Application::Frontend(instance) => instance.start().await,
-            Application::Metasrv(instance) => instance.start().await,
-            Application::Standalone(instance) => instance.start().await,
-            Application::Cli(instance) => instance.start().await,
-        }
-    }
-
-    async fn stop(&self) -> Result<()> {
-        match self {
-            Application::Datanode(instance) => instance.stop().await,
-            Application::Frontend(instance) => instance.stop().await,
-            Application::Metasrv(instance) => instance.stop().await,
-            Application::Standalone(instance) => instance.stop().await,
-            Application::Cli(instance) => instance.stop().await,
-        }
-    }
-}
-
-impl Command {
-    async fn build(self, opts: Options) -> Result<Application> {
-        self.subcmd.build(opts).await
-    }
-
-    fn load_options(&self) -> Result<Options> {
-        let top_level_opts = self.top_level_options();
-        self.subcmd.load_options(top_level_opts)
-    }
-
-    fn top_level_options(&self) -> TopLevelOptions {
-        TopLevelOptions {
-            log_dir: self.log_dir.clone(),
-            log_level: self.log_level.clone(),
-        }
-    }
-}
+use cmd::options::{CliOptions, Options};
+use cmd::{
+    cli, datanode, frontend, greptimedb_cli, log_versions, metasrv, standalone, start_app, App,
+};
 
 #[derive(Parser)]
 enum SubCommand {
@@ -105,40 +38,41 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, opts: Options) -> Result<Application> {
-        match (self, opts) {
+    async fn build(self, opts: Options) -> Result<Box<dyn App>> {
+        let app: Box<dyn App> = match (self, opts) {
             (SubCommand::Datanode(cmd), Options::Datanode(dn_opts)) => {
                 let app = cmd.build(*dn_opts).await?;
-                Ok(Application::Datanode(app))
+                Box::new(app) as _
             }
             (SubCommand::Frontend(cmd), Options::Frontend(fe_opts)) => {
                 let app = cmd.build(*fe_opts).await?;
-                Ok(Application::Frontend(app))
+                Box::new(app) as _
             }
             (SubCommand::Metasrv(cmd), Options::Metasrv(meta_opts)) => {
                 let app = cmd.build(*meta_opts).await?;
-                Ok(Application::Metasrv(app))
+                Box::new(app) as _
             }
             (SubCommand::Standalone(cmd), Options::Standalone(opts)) => {
                 let app = cmd.build(*opts).await?;
-                Ok(Application::Standalone(app))
+                Box::new(app) as _
             }
             (SubCommand::Cli(cmd), Options::Cli(_)) => {
                 let app = cmd.build().await?;
-                Ok(Application::Cli(app))
+                Box::new(app) as _
             }
 
             _ => unreachable!(),
-        }
+        };
+        Ok(app)
     }
 
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Datanode(cmd) => cmd.load_options(top_level_opts),
-            SubCommand::Frontend(cmd) => cmd.load_options(top_level_opts),
-            SubCommand::Metasrv(cmd) => cmd.load_options(top_level_opts),
-            SubCommand::Standalone(cmd) => cmd.load_options(top_level_opts),
-            SubCommand::Cli(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Datanode(cmd) => cmd.load_options(cli_options),
+            SubCommand::Frontend(cmd) => cmd.load_options(cli_options),
+            SubCommand::Metasrv(cmd) => cmd.load_options(cli_options),
+            SubCommand::Standalone(cmd) => cmd.load_options(cli_options),
+            SubCommand::Cli(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -155,89 +89,49 @@ impl fmt::Display for SubCommand {
     }
 }
 
-fn print_version() -> &'static str {
-    concat!(
-        "\nbranch: ",
-        env!("GIT_BRANCH"),
-        "\ncommit: ",
-        env!("GIT_COMMIT"),
-        "\ndirty: ",
-        env!("GIT_DIRTY"),
-        "\nversion: ",
-        env!("CARGO_PKG_VERSION")
-    )
-}
-
-fn short_version() -> &'static str {
-    env!("CARGO_PKG_VERSION")
-}
-
-// {app_name}-{branch_name}-{commit_short}
-// The branch name (tag) of a release build should already contain the short
-// version so the full version doesn't concat the short version explicitly.
-fn full_version() -> &'static str {
-    concat!(
-        "greptimedb-",
-        env!("GIT_BRANCH"),
-        "-",
-        env!("GIT_COMMIT_SHORT")
-    )
-}
-
-fn log_env_flags() {
-    info!("command line arguments");
-    for argument in std::env::args() {
-        info!("argument: {}", argument);
-    }
-}
-
 #[cfg(not(windows))]
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    let cmd = Command::parse();
-    let app_name = &cmd.subcmd.to_string();
-
-    let opts = cmd.load_options()?;
-    let logging_opts = opts.logging_options();
-    let tracing_opts = TracingOptions {
-        #[cfg(feature = "tokio-console")]
-        tokio_console_addr: cmd.tokio_console_addr.clone(),
+    let metadata = human_panic::Metadata {
+        version: env!("CARGO_PKG_VERSION").into(),
+        name: "GreptimeDB".into(),
+        authors: Default::default(),
+        homepage: "https://github.com/GreptimeTeam/greptimedb/discussions".into(),
     };
+    human_panic::setup_panic!(metadata);
 
     common_telemetry::set_panic_hook();
-    let _guard = common_telemetry::init_global_logging(app_name, logging_opts, tracing_opts);
 
-    // Report app version as gauge.
-    APP_VERSION
-        .with_label_values(&[short_version(), full_version()])
-        .inc();
+    let cli = greptimedb_cli();
 
-    // Log version and argument flags.
-    info!(
-        "short_version: {}, full_version: {}",
-        short_version(),
-        full_version()
+    let cli = SubCommand::augment_subcommands(cli);
+
+    let args = cli.get_matches();
+
+    let subcmd = match SubCommand::from_arg_matches(&args) {
+        Ok(subcmd) => subcmd,
+        Err(e) => e.exit(),
+    };
+
+    let app_name = subcmd.to_string();
+
+    let cli_options = CliOptions::new(&args);
+
+    let opts = subcmd.load_options(&cli_options)?;
+
+    let _guard = common_telemetry::init_global_logging(
+        &app_name,
+        opts.logging_options(),
+        cli_options.tracing_options(),
+        opts.node_id(),
     );
-    log_env_flags();
 
-    let mut app = cmd.build(opts).await?;
+    log_versions();
 
-    tokio::select! {
-        result = app.start() => {
-            if let Err(err) = result {
-                error!(err; "Fatal error occurs!");
-            }
-        }
-        _ = tokio::signal::ctrl_c() => {
-            if let Err(err) = app.stop().await {
-                error!(err; "Fatal error occurs!");
-            }
-            info!("Goodbye!");
-        }
-    }
+    let app = subcmd.build(opts).await?;
 
-    Ok(())
+    start_app(app).await
 }

src/cmd/src/cli.rs

@@ -13,9 +13,15 @@
 // limitations under the License.
 
 mod bench;
+
+// Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373
+#[allow(unused)]
 mod cmd;
 mod export;
 mod helper;
+
+// Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373
+#[allow(unused)]
 mod repl;
 // TODO(weny): Removes it
 #[allow(deprecated)]
@@ -30,27 +36,35 @@ use upgrade::UpgradeCommand;
 
 use self::export::ExportCommand;
 use crate::error::Result;
-use crate::options::{Options, TopLevelOptions};
+use crate::options::{CliOptions, Options};
+use crate::App;
 
 #[async_trait]
-pub trait Tool {
+pub trait Tool: Send + Sync {
     async fn do_work(&self) -> Result<()>;
 }
 
-pub enum Instance {
-    Repl(Repl),
-    Tool(Box<dyn Tool>),
+pub struct Instance {
+    tool: Box<dyn Tool>,
 }
 
 impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
-        match self {
-            Instance::Repl(repl) => repl.run().await,
-            Instance::Tool(tool) => tool.do_work().await,
-        }
+    fn new(tool: Box<dyn Tool>) -> Self {
+        Self { tool }
+    }
+}
+
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-cli"
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn start(&mut self) -> Result<()> {
+        self.tool.do_work().await
+    }
+
+    async fn stop(&self) -> Result<()> {
         Ok(())
     }
 }
@@ -66,14 +80,15 @@ impl Command {
         self.cmd.build().await
     }
 
-    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         let mut logging_opts = LoggingOptions::default();
-        if let Some(dir) = top_level_opts.log_dir {
-            logging_opts.dir = dir;
-        }
-        if top_level_opts.log_level.is_some() {
-            logging_opts.level = top_level_opts.log_level;
+
+        if let Some(dir) = &cli_options.log_dir {
+            logging_opts.dir = dir.clone();
         }
+
+        logging_opts.level = cli_options.log_level.clone();
+
         Ok(Options::Cli(Box::new(logging_opts)))
     }
 }
@@ -110,7 +125,6 @@ pub(crate) struct AttachCommand {
 impl AttachCommand {
     #[allow(dead_code)]
     async fn build(self) -> Result<Instance> {
-        let repl = Repl::try_new(&self).await?;
-        Ok(Instance::Repl(repl))
+        unimplemented!("Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373")
     }
 }

src/cmd/src/cli/bench.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 use std::future::Future;
 use std::sync::Arc;
 use std::time::Duration;
@@ -28,6 +28,7 @@ use common_telemetry::info;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use rand::Rng;
+use store_api::storage::RegionNumber;
 use table::metadata::{RawTableInfo, RawTableMeta, TableId, TableIdent, TableType};
 
 use self::metadata::TableMetadataBencher;
@@ -69,7 +70,7 @@ impl BenchTableMetadataCommand {
             table_metadata_manager,
             count: self.count,
         };
-        Ok(Instance::Tool(Box::new(tool)))
+        Ok(Instance::new(Box::new(tool)))
     }
 }
 
@@ -137,12 +138,12 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
     }
 }
 
-fn create_region_routes() -> Vec<RegionRoute> {
-    let mut regions = Vec::with_capacity(100);
+fn create_region_routes(regions: Vec<RegionNumber>) -> Vec<RegionRoute> {
+    let mut region_routes = Vec::with_capacity(100);
     let mut rng = rand::thread_rng();
 
-    for region_id in 0..64u64 {
-        regions.push(RegionRoute {
+    for region_id in regions.into_iter().map(u64::from) {
+        region_routes.push(RegionRoute {
             region: Region {
                 id: region_id.into(),
                 name: String::new(),
@@ -158,5 +159,11 @@ fn create_region_routes() -> Vec<RegionRoute> {
         });
     }
 
-    regions
+    region_routes
+}
+
+fn create_region_wal_options(regions: Vec<RegionNumber>) -> HashMap<RegionNumber, String> {
+    // TODO(niebayes): construct region wal options for benchmark.
+    let _ = regions;
+    HashMap::default()
 }

src/cmd/src/cli/bench/metadata.rs

@@ -14,10 +14,13 @@
 
 use std::time::Instant;
 
+use common_meta::key::table_route::TableRouteValue;
 use common_meta::key::TableMetadataManagerRef;
 use common_meta::table_name::TableName;
 
-use super::{bench_self_recorded, create_region_routes, create_table_info};
+use crate::cli::bench::{
+    bench_self_recorded, create_region_routes, create_region_wal_options, create_table_info,
+};
 
 pub struct TableMetadataBencher {
     table_metadata_manager: TableMetadataManagerRef,
@@ -43,12 +46,19 @@ impl TableMetadataBencher {
                 let table_name = format!("bench_table_name_{}", i);
                 let table_name = TableName::new("bench_catalog", "bench_schema", table_name);
                 let table_info = create_table_info(i, table_name);
-                let region_routes = create_region_routes();
+
+                let regions: Vec<_> = (0..64).collect();
+                let region_routes = create_region_routes(regions.clone());
+                let region_wal_options = create_region_wal_options(regions);
 
                 let start = Instant::now();
 
                 self.table_metadata_manager
-                    .create_table_metadata(table_info, region_routes)
+                    .create_table_metadata(
+                        table_info,
+                        TableRouteValue::physical(region_routes),
+                        region_wal_options,
+                    )
                     .await
                     .unwrap();
 

src/cmd/src/cli/export.rs

@@ -105,7 +105,7 @@ impl ExportCommand {
             }));
         }
 
-        Ok(Instance::Tool(Box::new(Export {
+        Ok(Instance::new(Box::new(Export {
             client: database_client,
             catalog,
             schema,

src/cmd/src/cli/repl.rs

@@ -17,7 +17,6 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use catalog::kvbackend::{CachedMetaKvBackend, KvBackendCatalogManager};
-use client::client_manager::DatanodeClients;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_base::Plugins;
 use common_error::ext::ErrorExt;
@@ -176,7 +175,7 @@ impl Repl {
                 .encode(&plan)
                 .context(SubstraitEncodeLogicalPlanSnafu)?;
 
-            self.database.logical_plan(plan.to_vec(), 0).await
+            self.database.logical_plan(plan.to_vec()).await
         } else {
             self.database.sql(&sql).await
         }
@@ -250,13 +249,8 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
 
     let cached_meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
 
-    let datanode_clients = Arc::new(DatanodeClients::default());
-
-    let catalog_list = KvBackendCatalogManager::new(
-        cached_meta_backend.clone(),
-        cached_meta_backend.clone(),
-        datanode_clients,
-    );
+    let catalog_list =
+        KvBackendCatalogManager::new(cached_meta_backend.clone(), cached_meta_backend);
     let plugins: Plugins = Default::default();
     let state = Arc::new(QueryEngineState::new(
         catalog_list,

src/cmd/src/cli/upgrade.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use async_trait::async_trait;
@@ -26,7 +27,7 @@ use common_meta::key::table_info::{TableInfoKey, TableInfoValue};
 use common_meta::key::table_name::{TableNameKey, TableNameValue};
 use common_meta::key::table_region::{TableRegionKey, TableRegionValue};
 use common_meta::key::table_route::{TableRouteKey, TableRouteValue as NextTableRouteValue};
-use common_meta::key::{RegionDistribution, TableMetaKey};
+use common_meta::key::{RegionDistribution, TableMetaKey, TableMetaValue};
 use common_meta::kv_backend::etcd::EtcdStore;
 use common_meta::kv_backend::KvBackendRef;
 use common_meta::range_stream::PaginationStream;
@@ -76,7 +77,7 @@ impl UpgradeCommand {
             skip_schema_keys: self.skip_schema_keys,
             skip_table_route_keys: self.skip_table_route_keys,
         };
-        Ok(Instance::Tool(Box::new(tool)))
+        Ok(Instance::new(Box::new(tool)))
     }
 }
 
@@ -152,7 +153,7 @@ impl MigrateTableMetadata {
         )
         .unwrap();
 
-        let new_table_value = NextTableRouteValue::new(table_route.region_routes);
+        let new_table_value = NextTableRouteValue::physical(table_route.region_routes);
 
         let table_id = table_route.table.id as u32;
         let new_key = TableRouteKey::new(table_id);
@@ -395,6 +396,9 @@ impl MigrateTableMetadata {
         let region_distribution: RegionDistribution =
             value.regions_id_map.clone().into_iter().collect();
 
+        // TODO(niebayes): properly fetch or construct wal options.
+        let region_wal_options = HashMap::default();
+
         let datanode_table_kvs = region_distribution
             .into_iter()
             .map(|(datanode_id, regions)| {
@@ -409,6 +413,7 @@ impl MigrateTableMetadata {
                             engine: engine.to_string(),
                             region_storage_path: region_storage_path.clone(),
                             region_options: (&value.table_info.meta.options).into(),
+                            region_wal_options: region_wal_options.clone(),
                         },
                     ),
                 )

src/cmd/src/datanode.rs

@@ -12,25 +12,41 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
 use std::time::Duration;
 
+use async_trait::async_trait;
+use catalog::kvbackend::MetaKvBackend;
 use clap::Parser;
-use common_telemetry::logging;
+use common_config::WalConfig;
+use common_telemetry::{info, logging};
 use datanode::config::DatanodeOptions;
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use meta_client::MetaClientOptions;
 use servers::Mode;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 
 use crate::error::{MissingConfigSnafu, Result, ShutdownDatanodeSnafu, StartDatanodeSnafu};
-use crate::options::{Options, TopLevelOptions};
+use crate::options::{CliOptions, Options};
+use crate::App;
 
 pub struct Instance {
     datanode: Datanode,
 }
 
 impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
+    fn new(datanode: Datanode) -> Self {
+        Self { datanode }
+    }
+}
+
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-datanode"
+    }
+
+    async fn start(&mut self) -> Result<()> {
         plugins::start_datanode_plugins(self.datanode.plugins())
             .await
             .context(StartDatanodeSnafu)?;
@@ -38,7 +54,7 @@ impl Instance {
         self.datanode.start().await.context(StartDatanodeSnafu)
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn stop(&self) -> Result<()> {
         self.datanode
             .shutdown()
             .await
@@ -57,8 +73,8 @@ impl Command {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
-        self.subcmd.load_options(top_level_opts)
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        self.subcmd.load_options(cli_options)
     }
 }
 
@@ -74,9 +90,9 @@ impl SubCommand {
         }
     }
 
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Start(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -89,7 +105,7 @@ struct StartCommand {
     rpc_addr: Option<String>,
     #[clap(long)]
     rpc_hostname: Option<String>,
-    #[clap(long, multiple = true, value_delimiter = ',')]
+    #[clap(long, value_delimiter = ',', num_args = 1..)]
     metasrv_addr: Option<Vec<String>>,
     #[clap(short, long)]
     config_file: Option<String>,
@@ -106,19 +122,19 @@ struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         let mut opts: DatanodeOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
             DatanodeOptions::env_list_keys(),
         )?;
 
-        if let Some(dir) = top_level_opts.log_dir {
-            opts.logging.dir = dir;
+        if let Some(dir) = &cli_options.log_dir {
+            opts.logging.dir = dir.clone();
         }
 
-        if top_level_opts.log_level.is_some() {
-            opts.logging.level = top_level_opts.log_level;
+        if cli_options.log_level.is_some() {
+            opts.logging.level = cli_options.log_level.clone();
         }
 
         if let Some(addr) = &self.rpc_addr {
@@ -151,8 +167,18 @@ impl StartCommand {
             opts.storage.data_home = data_home.clone();
         }
 
-        if let Some(wal_dir) = &self.wal_dir {
-            opts.wal.dir = Some(wal_dir.clone());
+        // `wal_dir` only affects raft-engine config.
+        if let Some(wal_dir) = &self.wal_dir
+            && let WalConfig::RaftEngine(raft_engine_config) = &mut opts.wal
+        {
+            if raft_engine_config
+                .dir
+                .as_ref()
+                .is_some_and(|original_dir| original_dir != wal_dir)
+            {
+                info!("The wal dir of raft-engine is altered to {wal_dir}");
+            }
+            raft_engine_config.dir.replace(wal_dir.clone());
         }
 
         if let Some(http_addr) = &self.http_addr {
@@ -177,12 +203,32 @@ impl StartCommand {
         logging::info!("Datanode start command: {:#?}", self);
         logging::info!("Datanode options: {:#?}", opts);
 
-        let datanode = DatanodeBuilder::new(opts, None, plugins)
+        let node_id = opts
+            .node_id
+            .context(MissingConfigSnafu { msg: "'node_id'" })?;
+
+        let meta_config = opts.meta_client.as_ref().context(MissingConfigSnafu {
+            msg: "'meta_client_options'",
+        })?;
+
+        let meta_client = datanode::heartbeat::new_metasrv_client(node_id, meta_config)
+            .await
+            .context(StartDatanodeSnafu)?;
+
+        let meta_backend = Arc::new(MetaKvBackend {
+            client: Arc::new(meta_client.clone()),
+        });
+
+        let datanode = DatanodeBuilder::new(opts, plugins)
+            .with_meta_client(meta_client)
+            .with_kv_backend(meta_backend)
+            .enable_region_server_service()
+            .enable_http_service()
             .build()
             .await
             .context(StartDatanodeSnafu)?;
 
-        Ok(Instance { datanode })
+        Ok(Instance::new(datanode))
     }
 }
 
@@ -192,12 +238,12 @@ mod tests {
     use std::time::Duration;
 
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::config::{CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig};
+    use datanode::config::{FileConfig, GcsConfig, ObjectStoreConfig, S3Config};
     use servers::heartbeat_options::HeartbeatOptions;
     use servers::Mode;
 
     use super::*;
-    use crate::options::ENV_VAR_SEP;
+    use crate::options::{CliOptions, ENV_VAR_SEP};
 
     #[test]
     fn test_read_from_config_file() {
@@ -221,6 +267,7 @@ mod tests {
             tcp_nodelay = true
 
             [wal]
+            provider = "raft_engine"
             dir = "/other/wal"
             file_size = "1GB"
             purge_threshold = "50GB"
@@ -229,18 +276,17 @@ mod tests {
             sync_write = false
 
             [storage]
-            type = "File"
             data_home = "/tmp/greptimedb/"
+            type = "File"
 
-            [storage.compaction]
-            max_inflight_tasks = 3
-            max_files_in_level0 = 7
-            max_purge_tasks = 32
+            [[storage.providers]]
+            type = "Gcs"
+            bucket = "foo"
+            endpoint = "bar"
 
-            [storage.manifest]
-            checkpoint_margin = 9
-            gc_duration = '7s'
-            compress = true
+            [[storage.providers]]
+            type = "S3"
+            bucket = "foo"
 
             [logging]
             level = "debug"
@@ -253,19 +299,24 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Datanode(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Datanode(options) = cmd.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
 
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
-        assert_eq!("/other/wal", options.wal.dir.unwrap());
 
-        assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
-        assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
-        assert_eq!(1024 * 1024 * 1024 * 50, options.wal.purge_threshold.0);
-        assert!(!options.wal.sync_write);
+        let WalConfig::RaftEngine(raft_engine_config) = options.wal else {
+            unreachable!()
+        };
+        assert_eq!("/other/wal", raft_engine_config.dir.unwrap());
+        assert_eq!(Duration::from_secs(600), raft_engine_config.purge_interval);
+        assert_eq!(1024 * 1024 * 1024, raft_engine_config.file_size.0);
+        assert_eq!(
+            1024 * 1024 * 1024 * 50,
+            raft_engine_config.purge_threshold.0
+        );
+        assert!(!raft_engine_config.sync_write);
 
         let HeartbeatOptions {
             interval: heart_beat_interval,
@@ -293,23 +344,15 @@ mod tests {
             &options.storage.store,
             ObjectStoreConfig::File(FileConfig { .. })
         ));
-
-        assert_eq!(
-            CompactionConfig {
-                max_inflight_tasks: 3,
-                max_files_in_level0: 7,
-                max_purge_tasks: 32,
-            },
-            options.storage.compaction,
-        );
-        assert_eq!(
-            RegionManifestConfig {
-                checkpoint_margin: Some(9),
-                gc_duration: Some(Duration::from_secs(7)),
-                compress: true
-            },
-            options.storage.manifest,
-        );
+        assert_eq!(options.storage.providers.len(), 2);
+        assert!(matches!(
+            options.storage.providers[0],
+            ObjectStoreConfig::Gcs(GcsConfig { .. })
+        ));
+        assert!(matches!(
+            options.storage.providers[1],
+            ObjectStoreConfig::S3(S3Config { .. })
+        ));
 
         assert_eq!("debug", options.logging.level.unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), options.logging.dir);
@@ -318,7 +361,7 @@ mod tests {
     #[test]
     fn test_try_from_cmd() {
         if let Options::Datanode(opt) = StartCommand::default()
-            .load_options(TopLevelOptions::default())
+            .load_options(&CliOptions::default())
             .unwrap()
         {
             assert_eq!(Mode::Standalone, opt.mode)
@@ -329,7 +372,7 @@ mod tests {
             metasrv_addr: Some(vec!["127.0.0.1:3002".to_string()]),
             ..Default::default()
         })
-        .load_options(TopLevelOptions::default())
+        .load_options(&CliOptions::default())
         .unwrap()
         {
             assert_eq!(Mode::Distributed, opt.mode)
@@ -339,7 +382,7 @@ mod tests {
             metasrv_addr: Some(vec!["127.0.0.1:3002".to_string()]),
             ..Default::default()
         })
-        .load_options(TopLevelOptions::default())
+        .load_options(&CliOptions::default())
         .is_err());
 
         // Providing node_id but leave metasrv_addr absent is ok since metasrv_addr has default value
@@ -347,18 +390,21 @@ mod tests {
             node_id: Some(42),
             ..Default::default()
         })
-        .load_options(TopLevelOptions::default())
+        .load_options(&CliOptions::default())
         .is_ok());
     }
 
     #[test]
-    fn test_top_level_options() {
+    fn test_load_log_options_from_cli() {
         let cmd = StartCommand::default();
 
         let options = cmd
-            .load_options(TopLevelOptions {
+            .load_options(&CliOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
+
+                #[cfg(feature = "tokio-console")]
+                tokio_console_addr: None,
             })
             .unwrap();
 
@@ -384,21 +430,16 @@ mod tests {
             tcp_nodelay = true
 
             [wal]
+            provider = "raft_engine"
             file_size = "1GB"
             purge_threshold = "50GB"
-            purge_interval = "10m"
-            read_batch_size = 128
+            purge_interval = "5m"
             sync_write = false
 
             [storage]
             type = "File"
             data_home = "/tmp/greptimedb/"
 
-            [storage.compaction]
-            max_inflight_tasks = 3
-            max_files_in_level0 = 7
-            max_purge_tasks = 32
-
             [logging]
             level = "debug"
             dir = "/tmp/greptimedb/test/logs"
@@ -409,26 +450,24 @@ mod tests {
         temp_env::with_vars(
             [
                 (
-                    // storage.manifest.gc_duration = 9s
+                    // wal.purge_interval = 1m
                     [
                         env_prefix.to_string(),
-                        "storage".to_uppercase(),
-                        "manifest".to_uppercase(),
-                        "gc_duration".to_uppercase(),
+                        "wal".to_uppercase(),
+                        "purge_interval".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
-                    Some("9s"),
+                    Some("1m"),
                 ),
                 (
-                    // storage.compaction.max_purge_tasks = 99
+                    // wal.read_batch_size = 100
                     [
                         env_prefix.to_string(),
-                        "storage".to_uppercase(),
-                        "compaction".to_uppercase(),
-                        "max_purge_tasks".to_uppercase(),
+                        "wal".to_uppercase(),
+                        "read_batch_size".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
-                    Some("99"),
+                    Some("100"),
                 ),
                 (
                     // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
@@ -449,17 +488,16 @@ mod tests {
                     ..Default::default()
                 };
 
-                let Options::Datanode(opts) =
-                    command.load_options(TopLevelOptions::default()).unwrap()
+                let Options::Datanode(opts) = command.load_options(&CliOptions::default()).unwrap()
                 else {
                     unreachable!()
                 };
 
                 // Should be read from env, env > default values.
-                assert_eq!(
-                    opts.storage.manifest.gc_duration,
-                    Some(Duration::from_secs(9))
-                );
+                let WalConfig::RaftEngine(raft_engine_config) = opts.wal else {
+                    unreachable!()
+                };
+                assert_eq!(raft_engine_config.read_batch_size, 100);
                 assert_eq!(
                     opts.meta_client.unwrap().metasrv_addrs,
                     vec![
@@ -470,19 +508,16 @@ mod tests {
                 );
 
                 // Should be read from config file, config file > env > default values.
-                assert_eq!(opts.storage.compaction.max_purge_tasks, 32);
+                assert_eq!(
+                    raft_engine_config.purge_interval,
+                    Duration::from_secs(60 * 5)
+                );
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.wal.dir.unwrap(), "/other/wal/dir");
+                assert_eq!(raft_engine_config.dir.unwrap(), "/other/wal/dir");
 
                 // Should be default value.
-                assert_eq!(
-                    opts.storage.manifest.checkpoint_margin,
-                    DatanodeOptions::default()
-                        .storage
-                        .manifest
-                        .checkpoint_margin
-                );
+                assert_eq!(opts.http.addr, DatanodeOptions::default().http.addr);
             },
         );
     }

src/cmd/src/error.rs

@@ -14,7 +14,7 @@
 
 use std::any::Any;
 
-use common_error::ext::ErrorExt;
+use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
 use config::ConfigError;
@@ -37,6 +37,18 @@ pub enum Error {
         source: common_meta::error::Error,
     },
 
+    #[snafu(display("Failed to init DDL manager"))]
+    InitDdlManager {
+        location: Location,
+        source: common_meta::error::Error,
+    },
+
+    #[snafu(display("Failed to init default timezone"))]
+    InitTimezone {
+        location: Location,
+        source: common_time::error::Error,
+    },
+
     #[snafu(display("Failed to start procedure manager"))]
     StartProcedureManager {
         location: Location,
@@ -49,6 +61,12 @@ pub enum Error {
         source: common_procedure::error::Error,
     },
 
+    #[snafu(display("Failed to start wal options allocator"))]
+    StartWalOptionsAllocator {
+        location: Location,
+        source: common_meta::error::Error,
+    },
+
     #[snafu(display("Failed to start datanode"))]
     StartDatanode {
         location: Location,
@@ -225,6 +243,12 @@ pub enum Error {
         #[snafu(source)]
         error: std::io::Error,
     },
+
+    #[snafu(display("Other error"))]
+    Other {
+        source: BoxedError,
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -240,21 +264,26 @@ impl ErrorExt for Error {
             Error::ShutdownMetaServer { source, .. } => source.status_code(),
             Error::BuildMetaServer { source, .. } => source.status_code(),
             Error::UnsupportedSelectorType { source, .. } => source.status_code(),
-            Error::IterStream { source, .. } | Error::InitMetadata { source, .. } => {
-                source.status_code()
-            }
+
+            Error::IterStream { source, .. }
+            | Error::InitMetadata { source, .. }
+            | Error::InitDdlManager { source, .. } => source.status_code(),
+
             Error::ConnectServer { source, .. } => source.status_code(),
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
             | Error::InvalidReplCommand { .. }
+            | Error::InitTimezone { .. }
             | Error::ConnectEtcd { .. }
             | Error::NotDataFromOutput { .. }
             | Error::CreateDir { .. }
             | Error::EmptyResult { .. }
             | Error::InvalidDatabaseName { .. } => StatusCode::InvalidArguments,
+
             Error::StartProcedureManager { source, .. }
             | Error::StopProcedureManager { source, .. } => source.status_code(),
+            Error::StartWalOptionsAllocator { source, .. } => source.status_code(),
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
             Error::RequestDatabase { source, .. } => source.status_code(),
             Error::CollectRecordBatches { source, .. }
@@ -267,6 +296,8 @@ impl ErrorExt for Error {
             Error::StartCatalogManager { source, .. } => source.status_code(),
 
             Error::SerdeJson { .. } | Error::FileIo { .. } => StatusCode::Unexpected,
+
+            Error::Other { source, .. } => source.status_code(),
         }
     }
 

src/cmd/src/frontend.rs

@@ -12,26 +12,48 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
 use std::time::Duration;
 
+use async_trait::async_trait;
+use catalog::kvbackend::CachedMetaKvBackend;
 use clap::Parser;
+use client::client_manager::DatanodeClients;
+use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
+use common_meta::heartbeat::handler::HandlerGroupExecutor;
 use common_telemetry::logging;
+use common_time::timezone::set_default_timezone;
 use frontend::frontend::FrontendOptions;
+use frontend::heartbeat::handler::invalidate_table_cache::InvalidateTableCacheHandler;
+use frontend::heartbeat::HeartbeatTask;
+use frontend::instance::builder::FrontendBuilder;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use meta_client::MetaClientOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 
-use crate::error::{self, Result, StartFrontendSnafu};
-use crate::options::{Options, TopLevelOptions};
+use crate::error::{self, InitTimezoneSnafu, MissingConfigSnafu, Result, StartFrontendSnafu};
+use crate::options::{CliOptions, Options};
+use crate::App;
 
 pub struct Instance {
     frontend: FeInstance,
 }
 
 impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
+    fn new(frontend: FeInstance) -> Self {
+        Self { frontend }
+    }
+}
+
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-frontend"
+    }
+
+    async fn start(&mut self) -> Result<()> {
         plugins::start_frontend_plugins(self.frontend.plugins().clone())
             .await
             .context(StartFrontendSnafu)?;
@@ -39,7 +61,7 @@ impl Instance {
         self.frontend.start().await.context(StartFrontendSnafu)
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn stop(&self) -> Result<()> {
         self.frontend
             .shutdown()
             .await
@@ -58,8 +80,8 @@ impl Command {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
-        self.subcmd.load_options(top_level_opts)
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        self.subcmd.load_options(cli_options)
     }
 }
 
@@ -75,9 +97,9 @@ impl SubCommand {
         }
     }
 
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Start(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -100,7 +122,7 @@ pub struct StartCommand {
     config_file: Option<String>,
     #[clap(short, long)]
     influxdb_enable: Option<bool>,
-    #[clap(long, multiple = true, value_delimiter = ',')]
+    #[clap(long, value_delimiter = ',', num_args = 1..)]
     metasrv_addr: Option<Vec<String>>,
     #[clap(long)]
     tls_mode: Option<TlsMode>,
@@ -117,19 +139,19 @@ pub struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         let mut opts: FrontendOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
             FrontendOptions::env_list_keys(),
         )?;
 
-        if let Some(dir) = top_level_opts.log_dir {
-            opts.logging.dir = dir;
+        if let Some(dir) = &cli_options.log_dir {
+            opts.logging.dir = dir.clone();
         }
 
-        if top_level_opts.log_level.is_some() {
-            opts.logging.level = top_level_opts.log_level;
+        if cli_options.log_level.is_some() {
+            opts.logging.level = cli_options.log_level.clone();
         }
 
         let tls_opts = TlsOption::new(
@@ -196,16 +218,50 @@ impl StartCommand {
         logging::info!("Frontend start command: {:#?}", self);
         logging::info!("Frontend options: {:#?}", opts);
 
-        let mut instance = FeInstance::try_new_distributed(&opts, plugins.clone())
+        set_default_timezone(opts.default_timezone.as_deref()).context(InitTimezoneSnafu)?;
+
+        let meta_client_options = opts.meta_client.as_ref().context(MissingConfigSnafu {
+            msg: "'meta_client'",
+        })?;
+        let meta_client = FeInstance::create_meta_client(meta_client_options)
             .await
             .context(StartFrontendSnafu)?;
 
+        let meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
+
+        let executor = HandlerGroupExecutor::new(vec![
+            Arc::new(ParseMailboxMessageHandler),
+            Arc::new(InvalidateTableCacheHandler::new(meta_backend.clone())),
+        ]);
+
+        let heartbeat_task = HeartbeatTask::new(
+            meta_client.clone(),
+            opts.heartbeat.clone(),
+            Arc::new(executor),
+        );
+
+        let mut instance = FrontendBuilder::new(
+            meta_backend.clone(),
+            Arc::new(DatanodeClients::default()),
+            meta_client,
+        )
+        .with_cache_invalidator(meta_backend)
+        .with_plugin(plugins)
+        .with_heartbeat_task(heartbeat_task)
+        .try_build()
+        .await
+        .context(StartFrontendSnafu)?;
+
+        instance
+            .build_export_metrics_task(&opts.export_metrics)
+            .context(StartFrontendSnafu)?;
+
         instance
             .build_servers(opts)
             .await
             .context(StartFrontendSnafu)?;
 
-        Ok(Instance { frontend: instance })
+        Ok(Instance::new(instance))
     }
 }
 
@@ -221,7 +277,7 @@ mod tests {
     use servers::http::HttpOptions;
 
     use super::*;
-    use crate::options::ENV_VAR_SEP;
+    use crate::options::{CliOptions, ENV_VAR_SEP};
 
     #[test]
     fn test_try_from_start_command() {
@@ -235,8 +291,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Frontend(opts) = command.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Frontend(opts) = command.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
 
@@ -288,7 +343,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Frontend(fe_opts) = command.load_options(TopLevelOptions::default()).unwrap()
+        let Options::Frontend(fe_opts) = command.load_options(&CliOptions::default()).unwrap()
         else {
             unreachable!()
         };
@@ -327,16 +382,19 @@ mod tests {
     }
 
     #[test]
-    fn test_top_level_options() {
+    fn test_load_log_options_from_cli() {
         let cmd = StartCommand {
             disable_dashboard: Some(false),
             ..Default::default()
         };
 
         let options = cmd
-            .load_options(TopLevelOptions {
+            .load_options(&CliOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
+
+                #[cfg(feature = "tokio-console")]
+                tokio_console_addr: None,
             })
             .unwrap();
 
@@ -416,11 +474,8 @@ mod tests {
                     ..Default::default()
                 };
 
-                let top_level_opts = TopLevelOptions {
-                    log_dir: None,
-                    log_level: Some("error".to_string()),
-                };
-                let Options::Frontend(fe_opts) = command.load_options(top_level_opts).unwrap()
+                let Options::Frontend(fe_opts) =
+                    command.load_options(&CliOptions::default()).unwrap()
                 else {
                     unreachable!()
                 };

src/cmd/src/lib.rs

@@ -12,7 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(assert_matches)]
+#![feature(assert_matches, let_chains)]
+
+use async_trait::async_trait;
+use clap::arg;
+use common_telemetry::{error, info};
 
 pub mod cli;
 pub mod datanode;
@@ -21,3 +25,100 @@ pub mod frontend;
 pub mod metasrv;
 pub mod options;
 pub mod standalone;
+
+lazy_static::lazy_static! {
+    static ref APP_VERSION: prometheus::IntGaugeVec =
+        prometheus::register_int_gauge_vec!("app_version", "app version", &["short_version", "version"]).unwrap();
+}
+
+#[async_trait]
+pub trait App {
+    fn name(&self) -> &str;
+
+    async fn start(&mut self) -> error::Result<()>;
+
+    async fn stop(&self) -> error::Result<()>;
+}
+
+pub async fn start_app(mut app: Box<dyn App>) -> error::Result<()> {
+    let name = app.name().to_string();
+
+    tokio::select! {
+        result = app.start() => {
+            if let Err(err) = result {
+                error!(err; "Failed to start app {name}!");
+            }
+        }
+        _ = tokio::signal::ctrl_c() => {
+            if let Err(err) = app.stop().await {
+                error!(err; "Failed to stop app {name}!");
+            }
+            info!("Goodbye!");
+        }
+    }
+
+    Ok(())
+}
+
+pub fn log_versions() {
+    // Report app version as gauge.
+    APP_VERSION
+        .with_label_values(&[short_version(), full_version()])
+        .inc();
+
+    // Log version and argument flags.
+    info!(
+        "short_version: {}, full_version: {}",
+        short_version(),
+        full_version()
+    );
+
+    log_env_flags();
+}
+
+pub fn greptimedb_cli() -> clap::Command {
+    let cmd = clap::Command::new("greptimedb")
+        .version(print_version())
+        .subcommand_required(true);
+
+    #[cfg(feature = "tokio-console")]
+    let cmd = cmd.arg(arg!(--"tokio-console-addr"[TOKIO_CONSOLE_ADDR]));
+
+    cmd.args([arg!(--"log-dir"[LOG_DIR]), arg!(--"log-level"[LOG_LEVEL])])
+}
+
+fn print_version() -> &'static str {
+    concat!(
+        "\nbranch: ",
+        env!("GIT_BRANCH"),
+        "\ncommit: ",
+        env!("GIT_COMMIT"),
+        "\ndirty: ",
+        env!("GIT_DIRTY"),
+        "\nversion: ",
+        env!("CARGO_PKG_VERSION")
+    )
+}
+
+fn short_version() -> &'static str {
+    env!("CARGO_PKG_VERSION")
+}
+
+// {app_name}-{branch_name}-{commit_short}
+// The branch name (tag) of a release build should already contain the short
+// version so the full version doesn't concat the short version explicitly.
+fn full_version() -> &'static str {
+    concat!(
+        "greptimedb-",
+        env!("GIT_BRANCH"),
+        "-",
+        env!("GIT_COMMIT_SHORT")
+    )
+}
+
+fn log_env_flags() {
+    info!("command line arguments");
+    for argument in std::env::args() {
+        info!("argument: {}", argument);
+    }
+}

src/cmd/src/metasrv.rs

@@ -14,6 +14,7 @@
 
 use std::time::Duration;
 
+use async_trait::async_trait;
 use clap::Parser;
 use common_telemetry::logging;
 use meta_srv::bootstrap::MetaSrvInstance;
@@ -21,21 +22,34 @@ use meta_srv::metasrv::MetaSrvOptions;
 use snafu::ResultExt;
 
 use crate::error::{self, Result, StartMetaServerSnafu};
-use crate::options::{Options, TopLevelOptions};
+use crate::options::{CliOptions, Options};
+use crate::App;
 
 pub struct Instance {
     instance: MetaSrvInstance,
 }
 
 impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
+    fn new(instance: MetaSrvInstance) -> Self {
+        Self { instance }
+    }
+}
+
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-metasrv"
+    }
+
+    async fn start(&mut self) -> Result<()> {
         plugins::start_meta_srv_plugins(self.instance.plugins())
             .await
             .context(StartMetaServerSnafu)?;
+
         self.instance.start().await.context(StartMetaServerSnafu)
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn stop(&self) -> Result<()> {
         self.instance
             .shutdown()
             .await
@@ -54,8 +68,8 @@ impl Command {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
-        self.subcmd.load_options(top_level_opts)
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        self.subcmd.load_options(cli_options)
     }
 }
 
@@ -71,9 +85,9 @@ impl SubCommand {
         }
     }
 
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Start(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -100,22 +114,29 @@ struct StartCommand {
     http_timeout: Option<u64>,
     #[clap(long, default_value = "GREPTIMEDB_METASRV")]
     env_prefix: String,
+    /// The working home directory of this metasrv instance.
+    #[clap(long)]
+    data_home: Option<String>,
+
+    /// If it's not empty, the metasrv will store all data with this key prefix.
+    #[clap(long, default_value = "")]
+    store_key_prefix: String,
 }
 
 impl StartCommand {
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         let mut opts: MetaSrvOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
             None,
         )?;
 
-        if let Some(dir) = top_level_opts.log_dir {
-            opts.logging.dir = dir;
+        if let Some(dir) = &cli_options.log_dir {
+            opts.logging.dir = dir.clone();
         }
 
-        if top_level_opts.log_level.is_some() {
-            opts.logging.level = top_level_opts.log_level;
+        if cli_options.log_level.is_some() {
+            opts.logging.level = cli_options.log_level.clone();
         }
 
         if let Some(addr) = &self.bind_addr {
@@ -152,6 +173,14 @@ impl StartCommand {
             opts.http.timeout = Duration::from_secs(http_timeout);
         }
 
+        if let Some(data_home) = &self.data_home {
+            opts.data_home = data_home.clone();
+        }
+
+        if !self.store_key_prefix.is_empty() {
+            opts.store_key_prefix = self.store_key_prefix.clone()
+        }
+
         // Disable dashboard in metasrv.
         opts.http.disable_dashboard = true;
 
@@ -166,11 +195,16 @@ impl StartCommand {
         logging::info!("MetaSrv start command: {:#?}", self);
         logging::info!("MetaSrv options: {:#?}", opts);
 
-        let instance = MetaSrvInstance::new(opts, plugins)
+        let builder = meta_srv::bootstrap::metasrv_builder(&opts, plugins.clone(), None)
+            .await
+            .context(error::BuildMetaServerSnafu)?;
+        let metasrv = builder.build().await.context(error::BuildMetaServerSnafu)?;
+
+        let instance = MetaSrvInstance::new(opts, plugins, metasrv)
             .await
             .context(error::BuildMetaServerSnafu)?;
 
-        Ok(Instance { instance })
+        Ok(Instance::new(instance))
     }
 }
 
@@ -194,8 +228,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Metasrv(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Metasrv(options) = cmd.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
@@ -230,8 +263,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Metasrv(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Metasrv(options) = cmd.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
@@ -262,7 +294,7 @@ mod tests {
     }
 
     #[test]
-    fn test_top_level_options() {
+    fn test_load_log_options_from_cli() {
         let cmd = StartCommand {
             bind_addr: Some("127.0.0.1:3002".to_string()),
             server_addr: Some("127.0.0.1:3002".to_string()),
@@ -272,9 +304,12 @@ mod tests {
         };
 
         let options = cmd
-            .load_options(TopLevelOptions {
+            .load_options(&CliOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
+
+                #[cfg(feature = "tokio-console")]
+                tokio_console_addr: None,
             })
             .unwrap();
 
@@ -333,8 +368,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let Options::Metasrv(opts) =
-                    command.load_options(TopLevelOptions::default()).unwrap()
+                let Options::Metasrv(opts) = command.load_options(&CliOptions::default()).unwrap()
                 else {
                     unreachable!()
                 };

src/cmd/src/options.rs

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use clap::ArgMatches;
 use common_config::KvBackendConfig;
-use common_telemetry::logging::LoggingOptions;
+use common_meta::wal::WalConfig as MetaSrvWalConfig;
+use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use config::{Config, Environment, File, FileFormat};
 use datanode::config::{DatanodeOptions, ProcedureConfig};
 use frontend::error::{Result as FeResult, TomlFormatSnafu};
@@ -28,7 +30,7 @@ pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
 
 /// Options mixed up from datanode, frontend and metasrv.
-#[derive(Serialize)]
+#[derive(Serialize, Debug, Clone)]
 pub struct MixOptions {
     pub data_home: String,
     pub procedure: ProcedureConfig,
@@ -36,6 +38,7 @@ pub struct MixOptions {
     pub frontend: FrontendOptions,
     pub datanode: DatanodeOptions,
     pub logging: LoggingOptions,
+    pub wal_meta: MetaSrvWalConfig,
 }
 
 impl From<MixOptions> for FrontendOptions {
@@ -58,10 +61,32 @@ pub enum Options {
     Cli(Box<LoggingOptions>),
 }
 
-#[derive(Clone, Debug, Default)]
-pub struct TopLevelOptions {
+#[derive(Default)]
+pub struct CliOptions {
     pub log_dir: Option<String>,
     pub log_level: Option<String>,
+
+    #[cfg(feature = "tokio-console")]
+    pub tokio_console_addr: Option<String>,
+}
+
+impl CliOptions {
+    pub fn new(args: &ArgMatches) -> Self {
+        Self {
+            log_dir: args.get_one::<String>("log-dir").cloned(),
+            log_level: args.get_one::<String>("log-level").cloned(),
+
+            #[cfg(feature = "tokio-console")]
+            tokio_console_addr: args.get_one::<String>("tokio-console-addr").cloned(),
+        }
+    }
+
+    pub fn tracing_options(&self) -> TracingOptions {
+        TracingOptions {
+            #[cfg(feature = "tokio-console")]
+            tokio_console_addr: self.tokio_console_addr.clone(),
+        }
+    }
 }
 
 impl Options {
@@ -133,13 +158,22 @@ impl Options {
 
         Ok(opts)
     }
+
+    pub fn node_id(&self) -> Option<String> {
+        match self {
+            Options::Metasrv(_) | Options::Cli(_) => None,
+            Options::Datanode(opt) => opt.node_id.map(|x| x.to_string()),
+            Options::Frontend(opt) => opt.node_id.clone(),
+            Options::Standalone(opt) => opt.frontend.node_id.clone(),
+        }
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use std::io::Write;
-    use std::time::Duration;
 
+    use common_config::WalConfig;
     use common_test_util::temp_dir::create_named_temp_file;
     use datanode::config::{DatanodeOptions, ObjectStoreConfig};
 
@@ -163,6 +197,7 @@ mod tests {
             tcp_nodelay = true
 
             [wal]
+            provider = "raft_engine"
             dir = "/tmp/greptimedb/wal"
             file_size = "1GB"
             purge_threshold = "50GB"
@@ -170,11 +205,6 @@ mod tests {
             read_batch_size = 128
             sync_write = false
 
-            [storage.compaction]
-            max_inflight_tasks = 3
-            max_files_in_level0 = 7
-            max_purge_tasks = 32
-
             [logging]
             level = "debug"
             dir = "/tmp/greptimedb/test/logs"
@@ -185,17 +215,6 @@ mod tests {
         temp_env::with_vars(
             // The following environment variables will be used to override the values in the config file.
             [
-                (
-                    // storage.manifest.checkpoint_margin = 99
-                    [
-                        env_prefix.to_string(),
-                        "storage".to_uppercase(),
-                        "manifest".to_uppercase(),
-                        "checkpoint_margin".to_uppercase(),
-                    ]
-                    .join(ENV_VAR_SEP),
-                    Some("99"),
-                ),
                 (
                     // storage.type = S3
                     [
@@ -216,17 +235,6 @@ mod tests {
                     .join(ENV_VAR_SEP),
                     Some("mybucket"),
                 ),
-                (
-                    // storage.manifest.gc_duration = 42s
-                    [
-                        env_prefix.to_string(),
-                        "storage".to_uppercase(),
-                        "manifest".to_uppercase(),
-                        "gc_duration".to_uppercase(),
-                    ]
-                    .join(ENV_VAR_SEP),
-                    Some("42s"),
-                ),
                 (
                     // wal.dir = /other/wal/dir
                     [
@@ -257,17 +265,12 @@ mod tests {
                 .unwrap();
 
                 // Check the configs from environment variables.
-                assert_eq!(opts.storage.manifest.checkpoint_margin, Some(99));
-                match opts.storage.store {
+                match &opts.storage.store {
                     ObjectStoreConfig::S3(s3_config) => {
                         assert_eq!(s3_config.bucket, "mybucket".to_string());
                     }
                     _ => panic!("unexpected store type"),
                 }
-                assert_eq!(
-                    opts.storage.manifest.gc_duration,
-                    Some(Duration::from_secs(42))
-                );
                 assert_eq!(
                     opts.meta_client.unwrap().metasrv_addrs,
                     vec![
@@ -278,7 +281,10 @@ mod tests {
                 );
 
                 // Should be the values from config file, not environment variables.
-                assert_eq!(opts.wal.dir.unwrap(), "/tmp/greptimedb/wal");
+                let WalConfig::RaftEngine(raft_engine_config) = opts.wal else {
+                    unreachable!()
+                };
+                assert_eq!(raft_engine_config.dir.unwrap(), "/tmp/greptimedb/wal");
 
                 // Should be default values.
                 assert_eq!(opts.node_id, None);

src/cmd/src/standalone.rs

@@ -15,38 +15,49 @@
 use std::sync::Arc;
 use std::{fs, path};
 
-use catalog::kvbackend::KvBackendCatalogManager;
-use catalog::CatalogManagerRef;
+use async_trait::async_trait;
 use clap::Parser;
-use common_base::Plugins;
-use common_config::{metadata_store_dir, KvBackendConfig, WalConfig};
-use common_meta::cache_invalidator::DummyKvCacheInvalidator;
+use common_catalog::consts::MIN_USER_TABLE_ID;
+use common_config::wal::StandaloneWalConfig;
+use common_config::{metadata_store_dir, KvBackendConfig};
+use common_meta::cache_invalidator::DummyCacheInvalidator;
+use common_meta::datanode_manager::DatanodeManagerRef;
+use common_meta::ddl::{DdlTaskExecutorRef, TableMetadataAllocatorRef};
+use common_meta::ddl_manager::DdlManager;
+use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
+use common_meta::region_keeper::MemoryRegionKeeper;
+use common_meta::sequence::SequenceBuilder;
+use common_meta::wal::{WalOptionsAllocator, WalOptionsAllocatorRef};
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
+use common_time::timezone::set_default_timezone;
 use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
-use datanode::region_server::RegionServer;
 use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
+use frontend::instance::builder::FrontendBuilder;
+use frontend::instance::standalone::StandaloneTableMetadataAllocator;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
 use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
+use servers::export_metrics::ExportMetricsOption;
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::ResultExt;
 
 use crate::error::{
-    CreateDirSnafu, IllegalConfigSnafu, InitMetadataSnafu, Result, ShutdownDatanodeSnafu,
-    ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
-    StopProcedureManagerSnafu,
+    CreateDirSnafu, IllegalConfigSnafu, InitDdlManagerSnafu, InitMetadataSnafu, InitTimezoneSnafu,
+    Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu,
+    StartProcedureManagerSnafu, StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
 };
-use crate::options::{MixOptions, Options, TopLevelOptions};
+use crate::options::{CliOptions, MixOptions, Options};
+use crate::App;
 
 #[derive(Parser)]
 pub struct Command {
@@ -59,8 +70,8 @@ impl Command {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
-        self.subcmd.load_options(top_level_options)
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        self.subcmd.load_options(cli_options)
     }
 }
 
@@ -76,9 +87,9 @@ impl SubCommand {
         }
     }
 
-    fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Start(cmd) => cmd.load_options(top_level_options),
+            SubCommand::Start(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -88,6 +99,7 @@ impl SubCommand {
 pub struct StandaloneOptions {
     pub mode: Mode,
     pub enable_telemetry: bool,
+    pub default_timezone: Option<String>,
     pub http: HttpOptions,
     pub grpc: GrpcOptions,
     pub mysql: MysqlOptions,
@@ -95,7 +107,7 @@ pub struct StandaloneOptions {
     pub opentsdb: OpentsdbOptions,
     pub influxdb: InfluxdbOptions,
     pub prom_store: PromStoreOptions,
-    pub wal: WalConfig,
+    pub wal: StandaloneWalConfig,
     pub storage: StorageConfig,
     pub metadata_store: KvBackendConfig,
     pub procedure: ProcedureConfig,
@@ -103,6 +115,7 @@ pub struct StandaloneOptions {
     pub user_provider: Option<String>,
     /// Options for different store engines.
     pub region_engine: Vec<RegionEngineConfig>,
+    pub export_metrics: ExportMetricsOption,
 }
 
 impl Default for StandaloneOptions {
@@ -110,6 +123,7 @@ impl Default for StandaloneOptions {
         Self {
             mode: Mode::Standalone,
             enable_telemetry: true,
+            default_timezone: None,
             http: HttpOptions::default(),
             grpc: GrpcOptions::default(),
             mysql: MysqlOptions::default(),
@@ -117,11 +131,12 @@ impl Default for StandaloneOptions {
             opentsdb: OpentsdbOptions::default(),
             influxdb: InfluxdbOptions::default(),
             prom_store: PromStoreOptions::default(),
-            wal: WalConfig::default(),
+            wal: StandaloneWalConfig::default(),
             storage: StorageConfig::default(),
             metadata_store: KvBackendConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
+            export_metrics: ExportMetricsOption::default(),
             user_provider: None,
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig::default()),
@@ -135,6 +150,7 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
+            default_timezone: self.default_timezone,
             http: self.http,
             grpc: self.grpc,
             mysql: self.mysql,
@@ -145,6 +161,8 @@ impl StandaloneOptions {
             meta_client: None,
             logging: self.logging,
             user_provider: self.user_provider,
+            // Handle the export metrics task run by standalone to frontend for execution
+            export_metrics: self.export_metrics,
             ..Default::default()
         }
     }
@@ -153,9 +171,10 @@ impl StandaloneOptions {
         DatanodeOptions {
             node_id: Some(0),
             enable_telemetry: self.enable_telemetry,
-            wal: self.wal,
+            wal: self.wal.into(),
             storage: self.storage,
             region_engine: self.region_engine,
+            rpc_addr: self.grpc.addr,
             ..Default::default()
         }
     }
@@ -165,24 +184,33 @@ pub struct Instance {
     datanode: Datanode,
     frontend: FeInstance,
     procedure_manager: ProcedureManagerRef,
+    wal_options_allocator: WalOptionsAllocatorRef,
 }
 
-impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
-        // Start datanode instance before starting services, to avoid requests come in before internal components are started.
-        self.datanode.start().await.context(StartDatanodeSnafu)?;
-        info!("Datanode instance started");
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-standalone"
+    }
+
+    async fn start(&mut self) -> Result<()> {
+        self.datanode.start_telemetry();
 
         self.procedure_manager
             .start()
             .await
             .context(StartProcedureManagerSnafu)?;
 
+        self.wal_options_allocator
+            .start()
+            .await
+            .context(StartWalOptionsAllocatorSnafu)?;
+
         self.frontend.start().await.context(StartFrontendSnafu)?;
         Ok(())
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn stop(&self) -> Result<()> {
         self.frontend
             .shutdown()
             .await
@@ -204,7 +232,7 @@ impl Instance {
 }
 
 #[derive(Debug, Default, Parser)]
-struct StartCommand {
+pub struct StartCommand {
     #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
@@ -218,7 +246,7 @@ struct StartCommand {
     #[clap(short, long)]
     influxdb_enable: bool,
     #[clap(short, long)]
-    config_file: Option<String>,
+    pub config_file: Option<String>,
     #[clap(long)]
     tls_mode: Option<TlsMode>,
     #[clap(long)]
@@ -228,25 +256,36 @@ struct StartCommand {
     #[clap(long)]
     user_provider: Option<String>,
     #[clap(long, default_value = "GREPTIMEDB_STANDALONE")]
-    env_prefix: String,
+    pub env_prefix: String,
+    /// The working home directory of this standalone instance.
+    #[clap(long)]
+    data_home: Option<String>,
 }
 
 impl StartCommand {
-    fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
-        let mut opts: StandaloneOptions = Options::load_layered_options(
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        let opts: StandaloneOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
             None,
         )?;
 
+        self.convert_options(cli_options, opts)
+    }
+
+    pub fn convert_options(
+        &self,
+        cli_options: &CliOptions,
+        mut opts: StandaloneOptions,
+    ) -> Result<Options> {
         opts.mode = Mode::Standalone;
 
-        if let Some(dir) = top_level_options.log_dir {
-            opts.logging.dir = dir;
+        if let Some(dir) = &cli_options.log_dir {
+            opts.logging.dir = dir.clone();
         }
 
-        if top_level_options.log_level.is_some() {
-            opts.logging.level = top_level_options.log_level;
+        if cli_options.log_level.is_some() {
+            opts.logging.level = cli_options.log_level.clone();
         }
 
         let tls_opts = TlsOption::new(
@@ -259,6 +298,10 @@ impl StartCommand {
             opts.http.addr = addr.clone()
         }
 
+        if let Some(data_home) = &self.data_home {
+            opts.storage.data_home = data_home.clone();
+        }
+
         if let Some(addr) = &self.rpc_addr {
             // frontend grpc addr conflict with datanode default grpc addr
             let datanode_grpc_addr = DatanodeOptions::default().rpc_addr;
@@ -300,7 +343,8 @@ impl StartCommand {
         let procedure = opts.procedure.clone();
         let frontend = opts.clone().frontend_options();
         let logging = opts.logging.clone();
-        let datanode = opts.datanode_options();
+        let wal_meta = opts.wal.clone().into();
+        let datanode = opts.datanode_options().clone();
 
         Ok(Options::Standalone(Box::new(MixOptions {
             procedure,
@@ -309,6 +353,7 @@ impl StartCommand {
             frontend,
             datanode,
             logging,
+            wal_meta,
         })))
     }
 
@@ -325,10 +370,11 @@ impl StartCommand {
         let dn_opts = opts.datanode.clone();
 
         info!("Standalone start command: {:#?}", self);
-        info!(
-            "Standalone frontend options: {:#?}, datanode options: {:#?}",
-            fe_opts, dn_opts
-        );
+
+        info!("Building standalone instance with {opts:#?}");
+
+        set_default_timezone(opts.frontend.default_timezone.as_deref())
+            .context(InitTimezoneSnafu)?;
 
         // Ensure the data_home directory exists.
         fs::create_dir_all(path::Path::new(&opts.data_home)).context(CreateDirSnafu {
@@ -344,38 +390,45 @@ impl StartCommand {
         .await
         .context(StartFrontendSnafu)?;
 
-        let datanode = DatanodeBuilder::new(
-            dn_opts.clone(),
-            Some(kv_backend.clone()),
-            Default::default(),
-        )
-        .build()
-        .await
-        .context(StartDatanodeSnafu)?;
-        let region_server = datanode.region_server();
+        let builder =
+            DatanodeBuilder::new(dn_opts, fe_plugins.clone()).with_kv_backend(kv_backend.clone());
+        let datanode = builder.build().await.context(StartDatanodeSnafu)?;
 
-        let catalog_manager = KvBackendCatalogManager::new(
-            kv_backend.clone(),
-            Arc::new(DummyKvCacheInvalidator),
-            Arc::new(StandaloneDatanodeManager(region_server.clone())),
+        let datanode_manager = Arc::new(StandaloneDatanodeManager(datanode.region_server()));
+
+        let table_id_sequence = Arc::new(
+            SequenceBuilder::new("table_id", kv_backend.clone())
+                .initial(MIN_USER_TABLE_ID as u64)
+                .step(10)
+                .build(),
         );
+        let wal_options_allocator = Arc::new(WalOptionsAllocator::new(
+            opts.wal_meta.clone(),
+            kv_backend.clone(),
+        ));
+        let table_meta_allocator = Arc::new(StandaloneTableMetadataAllocator::new(
+            table_id_sequence,
+            wal_options_allocator.clone(),
+        ));
 
-        catalog_manager
-            .table_metadata_manager_ref()
-            .init()
-            .await
-            .context(InitMetadataSnafu)?;
-
-        // TODO: build frontend instance like in distributed mode
-        let mut frontend = build_frontend(
-            fe_plugins,
-            kv_backend,
+        let ddl_task_executor = Self::create_ddl_task_executor(
+            kv_backend.clone(),
             procedure_manager.clone(),
-            catalog_manager,
-            region_server,
+            datanode_manager.clone(),
+            table_meta_allocator,
         )
         .await?;
 
+        let mut frontend = FrontendBuilder::new(kv_backend, datanode_manager, ddl_task_executor)
+            .with_plugin(fe_plugins)
+            .try_build()
+            .await
+            .context(StartFrontendSnafu)?;
+
+        frontend
+            .build_export_metrics_task(&opts.frontend.export_metrics)
+            .context(StartFrontendSnafu)?;
+
         frontend
             .build_servers(opts)
             .await
@@ -385,28 +438,46 @@ impl StartCommand {
             datanode,
             frontend,
             procedure_manager,
+            wal_options_allocator,
         })
     }
-}
 
-/// Build frontend instance in standalone mode
-async fn build_frontend(
-    plugins: Plugins,
-    kv_backend: KvBackendRef,
-    procedure_manager: ProcedureManagerRef,
-    catalog_manager: CatalogManagerRef,
-    region_server: RegionServer,
-) -> Result<FeInstance> {
-    let frontend_instance = FeInstance::try_new_standalone(
-        kv_backend,
-        procedure_manager,
-        catalog_manager,
-        plugins,
-        region_server,
-    )
-    .await
-    .context(StartFrontendSnafu)?;
-    Ok(frontend_instance)
+    pub async fn create_ddl_task_executor(
+        kv_backend: KvBackendRef,
+        procedure_manager: ProcedureManagerRef,
+        datanode_manager: DatanodeManagerRef,
+        table_meta_allocator: TableMetadataAllocatorRef,
+    ) -> Result<DdlTaskExecutorRef> {
+        let table_metadata_manager =
+            Self::create_table_metadata_manager(kv_backend.clone()).await?;
+
+        let ddl_task_executor: DdlTaskExecutorRef = Arc::new(
+            DdlManager::try_new(
+                procedure_manager,
+                datanode_manager,
+                Arc::new(DummyCacheInvalidator),
+                table_metadata_manager,
+                table_meta_allocator,
+                Arc::new(MemoryRegionKeeper::default()),
+            )
+            .context(InitDdlManagerSnafu)?,
+        );
+
+        Ok(ddl_task_executor)
+    }
+
+    async fn create_table_metadata_manager(
+        kv_backend: KvBackendRef,
+    ) -> Result<TableMetadataManagerRef> {
+        let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend));
+
+        table_metadata_manager
+            .init()
+            .await
+            .context(InitMetadataSnafu)?;
+
+        Ok(table_metadata_manager)
+    }
 }
 
 #[cfg(test)]
@@ -417,11 +488,13 @@ mod tests {
 
     use auth::{Identity, Password, UserProviderRef};
     use common_base::readable_size::ReadableSize;
+    use common_config::WalConfig;
     use common_test_util::temp_dir::create_named_temp_file;
+    use datanode::config::{FileConfig, GcsConfig};
     use servers::Mode;
 
     use super::*;
-    use crate::options::ENV_VAR_SEP;
+    use crate::options::{CliOptions, ENV_VAR_SEP};
 
     #[tokio::test]
     async fn test_try_from_start_command_to_anymap() {
@@ -459,6 +532,7 @@ mod tests {
             enable_memory_catalog = true
 
             [wal]
+            provider = "raft_engine"
             dir = "/tmp/greptimedb/test/wal"
             file_size = "1GB"
             purge_threshold = "50GB"
@@ -467,6 +541,15 @@ mod tests {
             sync_write = false
 
             [storage]
+            data_home = "/tmp/greptimedb/"
+            type = "File"
+
+            [[storage.providers]]
+            type = "Gcs"
+            bucket = "foo"
+            endpoint = "bar"
+
+            [[storage.providers]]
             type = "S3"
             access_key_id = "access_key_id"
             secret_access_key = "secret_access_key"
@@ -496,8 +579,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Standalone(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Standalone(options) = cmd.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
         let fe_opts = options.frontend;
@@ -514,9 +596,21 @@ mod tests {
         assert_eq!(None, fe_opts.mysql.reject_no_database);
         assert!(fe_opts.influxdb.enable);
 
-        assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap());
+        let WalConfig::RaftEngine(raft_engine_config) = dn_opts.wal else {
+            unreachable!()
+        };
+        assert_eq!("/tmp/greptimedb/test/wal", raft_engine_config.dir.unwrap());
 
-        match &dn_opts.storage.store {
+        assert!(matches!(
+            &dn_opts.storage.store,
+            datanode::config::ObjectStoreConfig::File(FileConfig { .. })
+        ));
+        assert_eq!(dn_opts.storage.providers.len(), 2);
+        assert!(matches!(
+            dn_opts.storage.providers[0],
+            datanode::config::ObjectStoreConfig::Gcs(GcsConfig { .. })
+        ));
+        match &dn_opts.storage.providers[1] {
             datanode::config::ObjectStoreConfig::S3(s3_config) => {
                 assert_eq!(
                     "Secret([REDACTED alloc::string::String])".to_string(),
@@ -533,16 +627,19 @@ mod tests {
     }
 
     #[test]
-    fn test_top_level_options() {
+    fn test_load_log_options_from_cli() {
         let cmd = StartCommand {
             user_provider: Some("static_user_provider:cmd:test=test".to_string()),
             ..Default::default()
         };
 
         let Options::Standalone(opts) = cmd
-            .load_options(TopLevelOptions {
+            .load_options(&CliOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
+
+                #[cfg(feature = "tokio-console")]
+                tokio_console_addr: None,
             })
             .unwrap()
         else {
@@ -609,11 +706,8 @@ mod tests {
                     ..Default::default()
                 };
 
-                let top_level_opts = TopLevelOptions {
-                    log_dir: None,
-                    log_level: None,
-                };
-                let Options::Standalone(opts) = command.load_options(top_level_opts).unwrap()
+                let Options::Standalone(opts) =
+                    command.load_options(&CliOptions::default()).unwrap()
                 else {
                     unreachable!()
                 };

src/common/base/Cargo.toml

@@ -7,7 +7,7 @@ license.workspace = true
 [dependencies]
 anymap = "1.0.0-beta.2"
 bitvec = "1.0"
-bytes = { version = "1.1", features = ["serde"] }
+bytes.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 paste = "1.0"

src/common/catalog/src/consts.rs

@@ -17,6 +17,7 @@ pub const INFORMATION_SCHEMA_NAME: &str = "information_schema";
 pub const SYSTEM_CATALOG_TABLE_NAME: &str = "system_catalog";
 pub const DEFAULT_CATALOG_NAME: &str = "greptime";
 pub const DEFAULT_SCHEMA_NAME: &str = "public";
+pub const DEFAULT_PRIVATE_SCHEMA_NAME: &str = "greptime_private";
 
 /// Reserves [0,MIN_USER_TABLE_ID) for internal usage.
 /// User defined table id starts from this value.
@@ -29,13 +30,25 @@ pub const SYSTEM_CATALOG_TABLE_ID: u32 = 0;
 pub const SCRIPTS_TABLE_ID: u32 = 1;
 /// numbers table id
 pub const NUMBERS_TABLE_ID: u32 = 2;
+
+/// ----- Begin of information_schema tables -----
 /// id for information_schema.tables
 pub const INFORMATION_SCHEMA_TABLES_TABLE_ID: u32 = 3;
 /// id for information_schema.columns
 pub const INFORMATION_SCHEMA_COLUMNS_TABLE_ID: u32 = 4;
+/// id for information_schema.engines
+pub const INFORMATION_SCHEMA_ENGINES_TABLE_ID: u32 = 5;
+/// id for information_schema.column_privileges
+pub const INFORMATION_SCHEMA_COLUMN_PRIVILEGES_TABLE_ID: u32 = 6;
+/// id for information_schema.column_statistics
+pub const INFORMATION_SCHEMA_COLUMN_STATISTICS_TABLE_ID: u32 = 7;
+/// id for information_schema.build_info
+pub const INFORMATION_SCHEMA_BUILD_INFO_TABLE_ID: u32 = 8;
+/// ----- End of information_schema tables -----
 
 pub const MITO_ENGINE: &str = "mito";
 pub const MITO2_ENGINE: &str = "mito2";
+pub const METRIC_ENGINE: &str = "metric";
 
 pub fn default_engine() -> &'static str {
     MITO_ENGINE

src/common/config/Cargo.toml

@@ -7,4 +7,8 @@ license.workspace = true
 [dependencies]
 common-base.workspace = true
 humantime-serde.workspace = true
+rskafka.workspace = true
 serde.workspace = true
+serde_json.workspace = true
+serde_with = "3"
+toml.workspace = true

src/common/config/src/lib.rs

@@ -12,41 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::time::Duration;
+pub mod wal;
 
 use common_base::readable_size::ReadableSize;
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(default)]
-pub struct WalConfig {
-    // wal directory
-    pub dir: Option<String>,
-    // wal file size in bytes
-    pub file_size: ReadableSize,
-    // wal purge threshold in bytes
-    pub purge_threshold: ReadableSize,
-    // purge interval in seconds
-    #[serde(with = "humantime_serde")]
-    pub purge_interval: Duration,
-    // read batch size
-    pub read_batch_size: usize,
-    // whether to sync log file after every write
-    pub sync_write: bool,
-}
-
-impl Default for WalConfig {
-    fn default() -> Self {
-        Self {
-            dir: None,
-            file_size: ReadableSize::mb(256), // log file size 256MB
-            purge_threshold: ReadableSize::gb(4), // purge threshold 4GB
-            purge_interval: Duration::from_secs(600),
-            read_batch_size: 128,
-            sync_write: false,
-        }
-    }
-}
+pub use crate::wal::{KafkaWalOptions, WalConfig, WalOptions, WAL_OPTIONS_KEY};
 
 pub fn metadata_store_dir(store_dir: &str) -> String {
     format!("{store_dir}/metadata")

src/common/config/src/wal.rs

@@ -0,0 +1,142 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod kafka;
+pub mod raft_engine;
+
+use serde::{Deserialize, Serialize};
+use serde_with::with_prefix;
+
+pub use crate::wal::kafka::{
+    KafkaConfig, KafkaOptions as KafkaWalOptions, StandaloneKafkaConfig, Topic as KafkaWalTopic,
+};
+pub use crate::wal::raft_engine::RaftEngineConfig;
+
+/// An encoded wal options will be wrapped into a (WAL_OPTIONS_KEY, encoded wal options) key-value pair
+/// and inserted into the options of a `RegionCreateRequest`.
+pub const WAL_OPTIONS_KEY: &str = "wal_options";
+
+/// Wal config for datanode.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(tag = "provider", rename_all = "snake_case")]
+pub enum WalConfig {
+    RaftEngine(RaftEngineConfig),
+    Kafka(KafkaConfig),
+}
+
+impl From<StandaloneWalConfig> for WalConfig {
+    fn from(value: StandaloneWalConfig) -> Self {
+        match value {
+            StandaloneWalConfig::RaftEngine(config) => WalConfig::RaftEngine(config),
+            StandaloneWalConfig::Kafka(config) => WalConfig::Kafka(config.base),
+        }
+    }
+}
+
+impl Default for WalConfig {
+    fn default() -> Self {
+        WalConfig::RaftEngine(RaftEngineConfig::default())
+    }
+}
+
+/// Wal config for datanode.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(tag = "provider", rename_all = "snake_case")]
+pub enum StandaloneWalConfig {
+    RaftEngine(RaftEngineConfig),
+    Kafka(StandaloneKafkaConfig),
+}
+
+impl Default for StandaloneWalConfig {
+    fn default() -> Self {
+        StandaloneWalConfig::RaftEngine(RaftEngineConfig::default())
+    }
+}
+
+/// Wal options allocated to a region.
+/// A wal options is encoded by metasrv with `serde_json::to_string`, and then decoded
+/// by datanode with `serde_json::from_str`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[serde(tag = "wal.provider", rename_all = "snake_case")]
+pub enum WalOptions {
+    #[default]
+    RaftEngine,
+    #[serde(with = "prefix_wal_kafka")]
+    Kafka(KafkaWalOptions),
+}
+
+with_prefix!(prefix_wal_kafka "wal.kafka.");
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use common_base::readable_size::ReadableSize;
+    use rskafka::client::partition::Compression as RsKafkaCompression;
+
+    use crate::wal::kafka::KafkaBackoffConfig;
+    use crate::wal::{KafkaConfig, KafkaWalOptions, WalOptions};
+
+    #[test]
+    fn test_serde_kafka_config() {
+        let toml_str = r#"
+            broker_endpoints = ["127.0.0.1:9092"]
+            max_batch_size = "4MB"
+            linger = "200ms"
+            produce_record_timeout = "100ms"
+            backoff_init = "500ms"
+            backoff_max = "10s"
+            backoff_base = 2
+            backoff_deadline = "5mins"
+        "#;
+        let decoded: KafkaConfig = toml::from_str(toml_str).unwrap();
+        let expected = KafkaConfig {
+            broker_endpoints: vec!["127.0.0.1:9092".to_string()],
+            compression: RsKafkaCompression::default(),
+            max_batch_size: ReadableSize::mb(4),
+            linger: Duration::from_millis(200),
+            produce_record_timeout: Duration::from_millis(100),
+            backoff: KafkaBackoffConfig {
+                init: Duration::from_millis(500),
+                max: Duration::from_secs(10),
+                base: 2,
+                deadline: Some(Duration::from_secs(60 * 5)),
+            },
+        };
+        assert_eq!(decoded, expected);
+    }
+
+    #[test]
+    fn test_serde_wal_options() {
+        // Test serde raft-engine wal options.
+        let wal_options = WalOptions::RaftEngine;
+        let encoded = serde_json::to_string(&wal_options).unwrap();
+        let expected = r#"{"wal.provider":"raft_engine"}"#;
+        assert_eq!(&encoded, expected);
+
+        let decoded: WalOptions = serde_json::from_str(&encoded).unwrap();
+        assert_eq!(decoded, wal_options);
+
+        // Test serde kafka wal options.
+        let wal_options = WalOptions::Kafka(KafkaWalOptions {
+            topic: "test_topic".to_string(),
+        });
+        let encoded = serde_json::to_string(&wal_options).unwrap();
+        let expected = r#"{"wal.provider":"kafka","wal.kafka.topic":"test_topic"}"#;
+        assert_eq!(&encoded, expected);
+
+        let decoded: WalOptions = serde_json::from_str(&encoded).unwrap();
+        assert_eq!(decoded, wal_options);
+    }
+}

src/common/config/src/wal/kafka.rs

@@ -0,0 +1,144 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use common_base::readable_size::ReadableSize;
+use rskafka::client::partition::Compression as RsKafkaCompression;
+use serde::{Deserialize, Serialize};
+use serde_with::with_prefix;
+
+/// Topic name prefix.
+pub const TOPIC_NAME_PREFIX: &str = "greptimedb_wal_topic";
+/// Kafka wal topic.
+pub type Topic = String;
+
+/// The type of the topic selector, i.e. with which strategy to select a topic.
+#[derive(Default, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum TopicSelectorType {
+    #[default]
+    RoundRobin,
+}
+
+/// Configurations for kafka wal.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct KafkaConfig {
+    /// The broker endpoints of the Kafka cluster.
+    pub broker_endpoints: Vec<String>,
+    /// The compression algorithm used to compress log entries.
+    #[serde(skip)]
+    #[serde(default)]
+    pub compression: RsKafkaCompression,
+    /// The maximum log size a kakfa batch producer could buffer.
+    pub max_batch_size: ReadableSize,
+    /// The linger duration of a kafka batch producer.
+    #[serde(with = "humantime_serde")]
+    pub linger: Duration,
+    /// The maximum amount of time (in milliseconds) to wait for Kafka records to be returned.
+    #[serde(with = "humantime_serde")]
+    pub produce_record_timeout: Duration,
+    /// The backoff config.
+    #[serde(flatten, with = "kafka_backoff")]
+    pub backoff: KafkaBackoffConfig,
+}
+
+impl Default for KafkaConfig {
+    fn default() -> Self {
+        Self {
+            broker_endpoints: vec!["127.0.0.1:9092".to_string()],
+            compression: RsKafkaCompression::NoCompression,
+            max_batch_size: ReadableSize::mb(4),
+            linger: Duration::from_millis(200),
+            produce_record_timeout: Duration::from_millis(100),
+            backoff: KafkaBackoffConfig::default(),
+        }
+    }
+}
+
+with_prefix!(pub kafka_backoff "backoff_");
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct KafkaBackoffConfig {
+    /// The initial backoff for kafka clients.
+    #[serde(with = "humantime_serde")]
+    pub init: Duration,
+    /// The maximum backoff for kafka clients.
+    #[serde(with = "humantime_serde")]
+    pub max: Duration,
+    /// Exponential backoff rate, i.e. next backoff = base * current backoff.
+    // Sets to u32 type since some structs containing the KafkaConfig need to derive the Eq trait.
+    pub base: u32,
+    /// Stop reconnecting if the total wait time reaches the deadline.
+    /// If it's None, the reconnecting won't terminate.
+    #[serde(with = "humantime_serde")]
+    pub deadline: Option<Duration>,
+}
+
+impl Default for KafkaBackoffConfig {
+    fn default() -> Self {
+        Self {
+            init: Duration::from_millis(500),
+            max: Duration::from_secs(10),
+            base: 2,
+            deadline: Some(Duration::from_secs(60 * 5)), // 5 mins
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct StandaloneKafkaConfig {
+    #[serde(flatten)]
+    pub base: KafkaConfig,
+    /// Number of topics to be created upon start.
+    pub num_topics: usize,
+    /// The type of the topic selector with which to select a topic for a region.
+    pub selector_type: TopicSelectorType,
+    /// Topic name prefix.
+    pub topic_name_prefix: String,
+    /// Number of partitions per topic.
+    pub num_partitions: i32,
+    /// The replication factor of each topic.
+    pub replication_factor: i16,
+    /// Above which a topic creation operation will be cancelled.
+    #[serde(with = "humantime_serde")]
+    pub create_topic_timeout: Duration,
+}
+
+impl Default for StandaloneKafkaConfig {
+    fn default() -> Self {
+        let base = KafkaConfig::default();
+        let replication_factor = base.broker_endpoints.len() as i16;
+
+        Self {
+            base,
+            num_topics: 64,
+            selector_type: TopicSelectorType::RoundRobin,
+            topic_name_prefix: "greptimedb_wal_topic".to_string(),
+            num_partitions: 1,
+            replication_factor,
+            create_topic_timeout: Duration::from_secs(30),
+        }
+    }
+}
+
+/// Kafka wal options allocated to a region.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct KafkaOptions {
+    /// Kafka wal topic.
+    pub topic: Topic,
+}

src/common/config/src/wal/raft_engine.rs

@@ -0,0 +1,50 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use common_base::readable_size::ReadableSize;
+use serde::{Deserialize, Serialize};
+
+/// Configurations for raft-engine wal.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct RaftEngineConfig {
+    // wal directory
+    pub dir: Option<String>,
+    // wal file size in bytes
+    pub file_size: ReadableSize,
+    // wal purge threshold in bytes
+    pub purge_threshold: ReadableSize,
+    // purge interval in seconds
+    #[serde(with = "humantime_serde")]
+    pub purge_interval: Duration,
+    // read batch size
+    pub read_batch_size: usize,
+    // whether to sync log file after every write
+    pub sync_write: bool,
+}
+
+impl Default for RaftEngineConfig {
+    fn default() -> Self {
+        Self {
+            dir: None,
+            file_size: ReadableSize::mb(256),
+            purge_threshold: ReadableSize::gb(4),
+            purge_interval: Duration::from_secs(600),
+            read_batch_size: 128,
+            sync_write: false,
+        }
+    }
+}

src/common/datasource/Cargo.toml

@@ -16,16 +16,18 @@ async-compression = { version = "0.3", features = [
     "tokio",
 ] }
 async-trait.workspace = true
-bytes = "1.1"
+bytes.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 common-runtime.workspace = true
 datafusion.workspace = true
+datatypes.workspace = true
 derive_builder.workspace = true
 futures.workspace = true
 lazy_static.workspace = true
 object-store.workspace = true
 orc-rust = "0.2"
+parquet.workspace = true
 paste = "1.0"
 regex = "1.7"
 serde.workspace = true

src/common/datasource/src/compression.rs

@@ -26,7 +26,9 @@ use tokio::io::{AsyncRead, AsyncWriteExt, BufReader};
 use tokio_util::io::{ReaderStream, StreamReader};
 
 use crate::error::{self, Error, Result};
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
 pub enum CompressionType {
     /// Gzip-ed file
     Gzip,

src/common/datasource/src/error.rs

@@ -166,6 +166,14 @@ pub enum Error {
 
     #[snafu(display("Buffered writer closed"))]
     BufferedWriterClosed { location: Location },
+
+    #[snafu(display("Failed to write parquet file, path: {}", path))]
+    WriteParquet {
+        path: String,
+        location: Location,
+        #[snafu(source)]
+        error: parquet::errors::ParquetError,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -178,7 +186,8 @@ impl ErrorExt for Error {
             | ListObjects { .. }
             | ReadObject { .. }
             | WriteObject { .. }
-            | AsyncWrite { .. } => StatusCode::StorageUnavailable,
+            | AsyncWrite { .. }
+            | WriteParquet { .. } => StatusCode::StorageUnavailable,
 
             UnsupportedBackendProtocol { .. }
             | UnsupportedCompressionType { .. }
@@ -231,6 +240,7 @@ impl ErrorExt for Error {
             InvalidConnection { location, .. } => Some(*location),
             UnsupportedCompressionType { location, .. } => Some(*location),
             UnsupportedFormat { location, .. } => Some(*location),
+            WriteParquet { location, .. } => Some(*location),
         }
     }
 }

src/common/datasource/src/file_format/parquet.rs

@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::future::Future;
+use std::pin::Pin;
 use std::result;
 use std::sync::Arc;
 
 use arrow::record_batch::RecordBatch;
-use arrow_schema::Schema;
+use arrow_schema::{Schema, SchemaRef};
 use async_trait::async_trait;
 use datafusion::datasource::physical_plan::{FileMeta, ParquetFileReaderFactory};
 use datafusion::error::Result as DatafusionResult;
@@ -26,11 +28,15 @@ use datafusion::parquet::errors::{ParquetError, Result as ParquetResult};
 use datafusion::parquet::file::metadata::ParquetMetaData;
 use datafusion::parquet::format::FileMetaData;
 use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
+use datafusion::physical_plan::SendableRecordBatchStream;
 use futures::future::BoxFuture;
+use futures::StreamExt;
 use object_store::{ObjectStore, Reader};
+use parquet::basic::{Compression, ZstdLevel};
+use parquet::file::properties::WriterProperties;
 use snafu::ResultExt;
 
-use crate::buffered_writer::{ArrowWriterCloser, DfRecordBatchEncoder};
+use crate::buffered_writer::{ArrowWriterCloser, DfRecordBatchEncoder, LazyBufferedWriter};
 use crate::error::{self, Result};
 use crate::file_format::FileFormat;
 use crate::share_buffer::SharedBuffer;
@@ -156,6 +162,103 @@ impl ArrowWriterCloser for ArrowWriter<SharedBuffer> {
     }
 }
 
+/// Parquet writer that buffers row groups in memory and writes buffered data to an underlying
+/// storage by chunks to reduce memory consumption.
+pub struct BufferedWriter {
+    inner: InnerBufferedWriter,
+}
+
+type InnerBufferedWriter = LazyBufferedWriter<
+    object_store::Writer,
+    ArrowWriter<SharedBuffer>,
+    Box<
+        dyn FnMut(
+                String,
+            )
+                -> Pin<Box<dyn Future<Output = error::Result<object_store::Writer>> + Send>>
+            + Send,
+    >,
+>;
+
+impl BufferedWriter {
+    pub async fn try_new(
+        path: String,
+        store: ObjectStore,
+        arrow_schema: SchemaRef,
+        props: Option<WriterProperties>,
+        buffer_threshold: usize,
+    ) -> error::Result<Self> {
+        let buffer = SharedBuffer::with_capacity(buffer_threshold);
+
+        let arrow_writer = ArrowWriter::try_new(buffer.clone(), arrow_schema.clone(), props)
+            .context(error::WriteParquetSnafu { path: &path })?;
+
+        Ok(Self {
+            inner: LazyBufferedWriter::new(
+                buffer_threshold,
+                buffer,
+                arrow_writer,
+                &path,
+                Box::new(move |path| {
+                    let store = store.clone();
+                    Box::pin(async move {
+                        store
+                            .writer(&path)
+                            .await
+                            .context(error::WriteObjectSnafu { path })
+                    })
+                }),
+            ),
+        })
+    }
+
+    /// Write a record batch to stream writer.
+    pub async fn write(&mut self, arrow_batch: &RecordBatch) -> error::Result<()> {
+        self.inner.write(arrow_batch).await?;
+        self.inner.try_flush(false).await?;
+
+        Ok(())
+    }
+
+    /// Close parquet writer.
+    ///
+    /// Return file metadata and bytes written.
+    pub async fn close(self) -> error::Result<(FileMetaData, u64)> {
+        self.inner.close_with_arrow_writer().await
+    }
+}
+
+/// Output the stream to a parquet file.
+///
+/// Returns number of rows written.
+pub async fn stream_to_parquet(
+    mut stream: SendableRecordBatchStream,
+    store: ObjectStore,
+    path: &str,
+    threshold: usize,
+) -> Result<usize> {
+    let write_props = WriterProperties::builder()
+        .set_compression(Compression::ZSTD(ZstdLevel::default()))
+        .build();
+    let schema = stream.schema();
+    let mut buffered_writer = BufferedWriter::try_new(
+        path.to_string(),
+        store,
+        schema,
+        Some(write_props),
+        threshold,
+    )
+    .await?;
+    let mut rows_written = 0;
+    while let Some(batch) = stream.next().await {
+        let batch = batch.context(error::ReadRecordBatchSnafu)?;
+        buffered_writer.write(&batch).await?;
+        rows_written += batch.num_rows();
+    }
+    buffered_writer.close().await?;
+    Ok(rows_written)
+}
+
 #[cfg(test)]
 mod tests {
     use common_test_util::find_workspace_path;

src/common/datasource/src/object_store/fs.rs

@@ -23,6 +23,15 @@ pub fn build_fs_backend(root: &str) -> Result<ObjectStore> {
     let _ = builder.root(root);
     let object_store = ObjectStore::new(builder)
         .context(BuildBackendSnafu)?
+        .layer(
+            object_store::layers::LoggingLayer::default()
+                // Print the expected error only in DEBUG level.
+                // See https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html#method.with_error_level
+                .with_error_level(Some("debug"))
+                .expect("input error level must be valid"),
+        )
+        .layer(object_store::layers::TracingLayer)
+        .layer(object_store::layers::PrometheusMetricsLayer)
         .finish();
     Ok(object_store)
 }

src/common/datasource/src/object_store/s3.rs

@@ -80,8 +80,18 @@ pub fn build_s3_backend(
         }
     }
 
+    // TODO(weny): Consider finding a better way to eliminate duplicate code.
     Ok(ObjectStore::new(builder)
         .context(error::BuildBackendSnafu)?
+        .layer(
+            object_store::layers::LoggingLayer::default()
+                // Print the expected error only in DEBUG level.
+                // See https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html#method.with_error_level
+                .with_error_level(Some("debug"))
+                .expect("input error level must be valid"),
+        )
+        .layer(object_store::layers::TracingLayer)
+        .layer(object_store::layers::PrometheusMetricsLayer)
         .finish())
 }
 

src/common/decimal/Cargo.toml

@@ -11,5 +11,5 @@ common-error.workspace = true
 common-macro.workspace = true
 rust_decimal.workspace = true
 serde.workspace = true
-serde_json = "1.0"
+serde_json.workspace = true
 snafu.workspace = true

src/common/decimal/src/decimal128.rs

@@ -96,10 +96,31 @@ impl Decimal128 {
         self.scale
     }
 
-    /// Convert to ScalarValue
+    /// Convert to ScalarValue(value,precision,scale)
     pub fn to_scalar_value(&self) -> (Option<i128>, u8, i8) {
         (Some(self.value), self.precision, self.scale)
     }
+
+    /// split the self.value(i128) to (high-64 bit, low-64 bit), and
+    /// the precision, scale information is discarded.
+    ///
+    /// Return: (high-64 bit, low-64 bit)
+    pub fn split_value(&self) -> (i64, i64) {
+        ((self.value >> 64) as i64, self.value as i64)
+    }
+
+    /// Convert from precision, scale, a i128 value which
+    /// represents by i64 + i64 value(high-64 bit, low-64 bit).
+    pub fn from_value_precision_scale(hi: i64, lo: i64, precision: u8, scale: i8) -> Self {
+        // 128                             64                              0
+        // +-------+-------+-------+-------+-------+-------+-------+-------+
+        // |               hi              |               lo              |
+        // +-------+-------+-------+-------+-------+-------+-------+-------+
+        let hi = (hi as u128 & u64::MAX as u128) << 64;
+        let lo = lo as u128 & u64::MAX as u128;
+        let value = (hi | lo) as i128;
+        Self::new(value, precision, scale)
+    }
 }
 
 /// The default value of Decimal128 is 0, and its precision is 1 and scale is 0.
@@ -414,4 +435,30 @@ mod tests {
         let decimal2 = Decimal128::from_str("1234567890.123").unwrap();
         assert_eq!(decimal1.partial_cmp(&decimal2), None);
     }
+
+    #[test]
+    fn test_convert_with_i128() {
+        let test_decimal128_eq = |value| {
+            let decimal1 =
+                Decimal128::new(value, DECIMAL128_MAX_PRECISION, DECIMAL128_DEFAULT_SCALE);
+            let (hi, lo) = decimal1.split_value();
+            let decimal2 = Decimal128::from_value_precision_scale(
+                hi,
+                lo,
+                DECIMAL128_MAX_PRECISION,
+                DECIMAL128_DEFAULT_SCALE,
+            );
+            assert_eq!(decimal1, decimal2);
+        };
+
+        test_decimal128_eq(1 << 63);
+
+        test_decimal128_eq(0);
+        test_decimal128_eq(1234567890);
+        test_decimal128_eq(-1234567890);
+        test_decimal128_eq(32781372819372817382183218i128);
+        test_decimal128_eq(-32781372819372817382183218i128);
+        test_decimal128_eq(i128::MAX);
+        test_decimal128_eq(i128::MIN);
+    }
 }

src/common/error/src/status_code.rs

@@ -59,6 +59,10 @@ pub enum StatusCode {
     RegionNotFound = 4005,
     RegionAlreadyExists = 4006,
     RegionReadonly = 4007,
+    RegionNotReady = 4008,
+    // If mutually exclusive operations are reached at the same time,
+    // only one can be executed, another one will get region busy.
+    RegionBusy = 4009,
     // ====== End of catalog related status code =======
 
     // ====== Begin of storage related status code =====
@@ -103,7 +107,9 @@ impl StatusCode {
         match self {
             StatusCode::StorageUnavailable
             | StatusCode::RuntimeResourcesExhausted
-            | StatusCode::Internal => true,
+            | StatusCode::Internal
+            | StatusCode::RegionNotReady
+            | StatusCode::RegionBusy => true,
 
             StatusCode::Success
             | StatusCode::Unknown
@@ -138,7 +144,6 @@ impl StatusCode {
     pub fn should_log_error(&self) -> bool {
         match self {
             StatusCode::Unknown
-            | StatusCode::Unsupported
             | StatusCode::Unexpected
             | StatusCode::Internal
             | StatusCode::Cancelled
@@ -147,11 +152,14 @@ impl StatusCode {
             | StatusCode::StorageUnavailable
             | StatusCode::RuntimeResourcesExhausted => true,
             StatusCode::Success
+            | StatusCode::Unsupported
             | StatusCode::InvalidArguments
             | StatusCode::InvalidSyntax
             | StatusCode::TableAlreadyExists
             | StatusCode::TableNotFound
             | StatusCode::RegionNotFound
+            | StatusCode::RegionNotReady
+            | StatusCode::RegionBusy
             | StatusCode::RegionAlreadyExists
             | StatusCode::RegionReadonly
             | StatusCode::TableColumnNotFound
@@ -183,6 +191,8 @@ impl StatusCode {
             v if v == StatusCode::TableAlreadyExists as u32 => Some(StatusCode::TableAlreadyExists),
             v if v == StatusCode::TableNotFound as u32 => Some(StatusCode::TableNotFound),
             v if v == StatusCode::RegionNotFound as u32 => Some(StatusCode::RegionNotFound),
+            v if v == StatusCode::RegionNotReady as u32 => Some(StatusCode::RegionNotReady),
+            v if v == StatusCode::RegionBusy as u32 => Some(StatusCode::RegionBusy),
             v if v == StatusCode::RegionAlreadyExists as u32 => {
                 Some(StatusCode::RegionAlreadyExists)
             }

src/common/function/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 arc-swap = "1.0"
+build-data = "0.1"
 chrono-tz = "0.6"
 common-error.workspace = true
 common-macro.workspace = true

src/common/function/src/function_registry.rs

@@ -18,11 +18,13 @@ use std::sync::{Arc, RwLock};
 
 use once_cell::sync::Lazy;
 
+use crate::function::FunctionRef;
 use crate::scalars::aggregate::{AggregateFunctionMetaRef, AggregateFunctions};
-use crate::scalars::function::FunctionRef;
+use crate::scalars::date::DateFunction;
 use crate::scalars::math::MathFunction;
 use crate::scalars::numpy::NumpyFunction;
 use crate::scalars::timestamp::TimestampFunction;
+use crate::system::SystemFunction;
 
 #[derive(Default)]
 pub struct FunctionRegistry {
@@ -75,9 +77,10 @@ pub static FUNCTION_REGISTRY: Lazy<Arc<FunctionRegistry>> = Lazy::new(|| {
     MathFunction::register(&function_registry);
     NumpyFunction::register(&function_registry);
     TimestampFunction::register(&function_registry);
+    DateFunction::register(&function_registry);
 
     AggregateFunctions::register(&function_registry);
-
+    SystemFunction::register(&function_registry);
     Arc::new(function_registry)
 });
 

src/common/function/src/helper.rs

@@ -12,22 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_error::ext::ErrorExt;
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use datatypes::prelude::ConcreteDataType;
 
-pub trait Encoder {
-    /// The type that is decoded.
-    type Item;
-    type Error: ErrorExt;
+/// Create a function signature with oneof signatures of interleaving two arguments.
+pub fn one_of_sigs2(args1: Vec<ConcreteDataType>, args2: Vec<ConcreteDataType>) -> Signature {
+    let mut sigs = Vec::with_capacity(args1.len() * args2.len());
 
-    /// Encodes a message into the bytes buffer.
-    fn encode(&self, item: &Self::Item, dst: &mut Vec<u8>) -> Result<(), Self::Error>;
-}
-
-pub trait Decoder {
-    /// The type that is decoded.
-    type Item;
-    type Error: ErrorExt;
-
-    /// Decodes a message from the bytes buffer.
-    fn decode(&self, src: &[u8]) -> Result<Self::Item, Self::Error>;
+    for arg1 in &args1 {
+        for arg2 in &args2 {
+            sigs.push(TypeSignature::Exact(vec![arg1.clone(), arg2.clone()]));
+        }
+    }
+
+    Signature::one_of(sigs, Volatility::Immutable)
 }

src/common/function/src/lib.rs

@@ -13,3 +13,8 @@
 // limitations under the License.
 
 pub mod scalars;
+pub mod system;
+
+pub mod function;
+pub mod function_registry;
+pub mod helper;

src/common/function/src/scalars.rs

@@ -13,15 +13,11 @@
 // limitations under the License.
 
 pub mod aggregate;
+pub(crate) mod date;
 pub mod expression;
-pub mod function;
-pub mod function_registry;
 pub mod math;
 pub mod numpy;
 #[cfg(test)]
 pub(crate) mod test;
-mod timestamp;
+pub(crate) mod timestamp;
 pub mod udf;
-
-pub use function::{Function, FunctionRef};
-pub use function_registry::{FunctionRegistry, FUNCTION_REGISTRY};

src/common/function/src/scalars/aggregate.rs

@@ -33,7 +33,7 @@ pub use polyval::PolyvalAccumulatorCreator;
 pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;
 
-use crate::scalars::FunctionRegistry;
+use crate::function_registry::FunctionRegistry;
 
 /// A function creates `AggregateFunctionCreator`.
 /// "Aggregator" *is* AggregatorFunction. Since the later one is long, we named an short alias for it.

src/common/function/src/scalars/date.rs

@@ -12,16 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use storage::write_batch::WriteBatch;
+use std::sync::Arc;
+mod date_add;
+mod date_sub;
 
-use crate::memtable::util::schema_util::{self, ColumnDef};
+use date_add::DateAddFunction;
+use date_sub::DateSubFunction;
 
-pub fn new_write_batch(
-    column_defs: &[ColumnDef],
-    timestamp_index: Option<usize>,
-    row_key_end: usize,
-) -> WriteBatch {
-    let schema = schema_util::new_schema_ref(column_defs, timestamp_index);
+use crate::function_registry::FunctionRegistry;
 
-    WriteBatch::new(schema, row_key_end)
+pub(crate) struct DateFunction;
+
+impl DateFunction {
+    pub fn register(registry: &FunctionRegistry) {
+        registry.register(Arc::new(DateAddFunction));
+        registry.register(Arc::new(DateSubFunction));
+    }
 }

src/common/function/src/scalars/date/date_add.rs

@@ -0,0 +1,278 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt;
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datatypes::data_type::DataType;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::value::ValueRef;
+use datatypes::vectors::VectorRef;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::helper;
+
+/// A function adds an interval value to Timestamp, Date or DateTime, and return the result.
+#[derive(Clone, Debug, Default)]
+pub struct DateAddFunction;
+
+const NAME: &str = "date_add";
+
+impl Function for DateAddFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(input_types[0].clone())
+    }
+
+    fn signature(&self) -> Signature {
+        helper::one_of_sigs2(
+            vec![
+                ConcreteDataType::date_datatype(),
+                ConcreteDataType::datetime_datatype(),
+                ConcreteDataType::timestamp_second_datatype(),
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                ConcreteDataType::timestamp_microsecond_datatype(),
+                ConcreteDataType::timestamp_nanosecond_datatype(),
+            ],
+            vec![
+                ConcreteDataType::interval_month_day_nano_datatype(),
+                ConcreteDataType::interval_year_month_datatype(),
+                ConcreteDataType::interval_day_time_datatype(),
+            ],
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 2, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let left = &columns[0];
+        let right = &columns[1];
+
+        let size = left.len();
+        let left_datatype = columns[0].data_type();
+        match left_datatype {
+            ConcreteDataType::Timestamp(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let ts = left.get(i).as_timestamp();
+                    let interval = right.get(i).as_interval();
+
+                    let new_ts = match (ts, interval) {
+                        (Some(ts), Some(interval)) => ts.add_interval(interval),
+                        _ => ts,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_ts));
+                }
+
+                Ok(result.to_vector())
+            }
+            ConcreteDataType::Date(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let date = left.get(i).as_date();
+                    let interval = right.get(i).as_interval();
+                    let new_date = match (date, interval) {
+                        (Some(date), Some(interval)) => date.add_interval(interval),
+                        _ => date,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_date));
+                }
+
+                Ok(result.to_vector())
+            }
+            ConcreteDataType::DateTime(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let datetime = left.get(i).as_datetime();
+                    let interval = right.get(i).as_interval();
+                    let new_datetime = match (datetime, interval) {
+                        (Some(datetime), Some(interval)) => datetime.add_interval(interval),
+                        _ => datetime,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_datetime));
+                }
+
+                Ok(result.to_vector())
+            }
+            _ => UnsupportedInputDataTypeSnafu {
+                function: NAME,
+                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+            }
+            .fail(),
+        }
+    }
+}
+
+impl fmt::Display for DateAddFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "DATE_ADD")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::{TypeSignature, Volatility};
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::value::Value;
+    use datatypes::vectors::{
+        DateTimeVector, DateVector, IntervalDayTimeVector, IntervalYearMonthVector,
+        TimestampSecondVector,
+    };
+
+    use super::{DateAddFunction, *};
+
+    #[test]
+    fn test_date_add_misc() {
+        let f = DateAddFunction;
+        assert_eq!("date_add", f.name());
+        assert_eq!(
+            ConcreteDataType::timestamp_microsecond_datatype(),
+            f.return_type(&[ConcreteDataType::timestamp_microsecond_datatype()])
+                .unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::timestamp_second_datatype(),
+            f.return_type(&[ConcreteDataType::timestamp_second_datatype()])
+                .unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::date_datatype(),
+            f.return_type(&[ConcreteDataType::date_datatype()]).unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::datetime_datatype(),
+            f.return_type(&[ConcreteDataType::datetime_datatype()])
+                .unwrap()
+        );
+        assert!(matches!(f.signature(),
+                         Signature {
+                             type_signature: TypeSignature::OneOf(sigs),
+                             volatility: Volatility::Immutable
+                         } if  sigs.len() == 18));
+    }
+
+    #[test]
+    fn test_timestamp_date_add() {
+        let f = DateAddFunction;
+
+        let times = vec![Some(123), None, Some(42), None];
+        // Intervals in milliseconds
+        let intervals = vec![1000, 2000, 3000, 1000];
+        let results = [Some(124), None, Some(45), None];
+
+        let time_vector = TimestampSecondVector::from(times.clone());
+        let interval_vector = IntervalDayTimeVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(time_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in times.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Timestamp(ts) => {
+                    assert_eq!(ts.value(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_date_date_add() {
+        let f = DateAddFunction;
+
+        let dates = vec![Some(123), None, Some(42), None];
+        // Intervals in months
+        let intervals = vec![1, 2, 3, 1];
+        let results = [Some(154), None, Some(131), None];
+
+        let date_vector = DateVector::from(dates.clone());
+        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in dates.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Date(date) => {
+                    assert_eq!(date.val(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_datetime_date_add() {
+        let f = DateAddFunction;
+
+        let dates = vec![Some(123), None, Some(42), None];
+        // Intervals in months
+        let intervals = vec![1, 2, 3, 1];
+        let results = [Some(2678400123), None, Some(7776000042), None];
+
+        let date_vector = DateTimeVector::from(dates.clone());
+        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in dates.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::DateTime(date) => {
+                    assert_eq!(date.val(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+}

src/common/function/src/scalars/date/date_sub.rs

@@ -0,0 +1,291 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt;
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datatypes::data_type::DataType;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::value::ValueRef;
+use datatypes::vectors::VectorRef;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::helper;
+
+/// A function subtracts an interval value to Timestamp, Date or DateTime, and return the result.
+#[derive(Clone, Debug, Default)]
+pub struct DateSubFunction;
+
+const NAME: &str = "date_sub";
+
+impl Function for DateSubFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(input_types[0].clone())
+    }
+
+    fn signature(&self) -> Signature {
+        helper::one_of_sigs2(
+            vec![
+                ConcreteDataType::date_datatype(),
+                ConcreteDataType::datetime_datatype(),
+                ConcreteDataType::timestamp_second_datatype(),
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                ConcreteDataType::timestamp_microsecond_datatype(),
+                ConcreteDataType::timestamp_nanosecond_datatype(),
+            ],
+            vec![
+                ConcreteDataType::interval_month_day_nano_datatype(),
+                ConcreteDataType::interval_year_month_datatype(),
+                ConcreteDataType::interval_day_time_datatype(),
+            ],
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 2, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let left = &columns[0];
+        let right = &columns[1];
+
+        let size = left.len();
+        let left_datatype = columns[0].data_type();
+
+        match left_datatype {
+            ConcreteDataType::Timestamp(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let ts = left.get(i).as_timestamp();
+                    let interval = right.get(i).as_interval();
+
+                    let new_ts = match (ts, interval) {
+                        (Some(ts), Some(interval)) => ts.sub_interval(interval),
+                        _ => ts,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_ts));
+                }
+
+                Ok(result.to_vector())
+            }
+            ConcreteDataType::Date(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let date = left.get(i).as_date();
+                    let interval = right.get(i).as_interval();
+                    let new_date = match (date, interval) {
+                        (Some(date), Some(interval)) => date.sub_interval(interval),
+                        _ => date,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_date));
+                }
+
+                Ok(result.to_vector())
+            }
+            ConcreteDataType::DateTime(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let datetime = left.get(i).as_datetime();
+                    let interval = right.get(i).as_interval();
+                    let new_datetime = match (datetime, interval) {
+                        (Some(datetime), Some(interval)) => datetime.sub_interval(interval),
+                        _ => datetime,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_datetime));
+                }
+
+                Ok(result.to_vector())
+            }
+            _ => UnsupportedInputDataTypeSnafu {
+                function: NAME,
+                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+            }
+            .fail(),
+        }
+    }
+}
+
+impl fmt::Display for DateSubFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "DATE_SUB")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::{TypeSignature, Volatility};
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::value::Value;
+    use datatypes::vectors::{
+        DateTimeVector, DateVector, IntervalDayTimeVector, IntervalYearMonthVector,
+        TimestampSecondVector,
+    };
+
+    use super::{DateSubFunction, *};
+
+    #[test]
+    fn test_date_sub_misc() {
+        let f = DateSubFunction;
+        assert_eq!("date_sub", f.name());
+        assert_eq!(
+            ConcreteDataType::timestamp_microsecond_datatype(),
+            f.return_type(&[ConcreteDataType::timestamp_microsecond_datatype()])
+                .unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::timestamp_second_datatype(),
+            f.return_type(&[ConcreteDataType::timestamp_second_datatype()])
+                .unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::date_datatype(),
+            f.return_type(&[ConcreteDataType::date_datatype()]).unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::datetime_datatype(),
+            f.return_type(&[ConcreteDataType::datetime_datatype()])
+                .unwrap()
+        );
+        assert!(matches!(f.signature(),
+                         Signature {
+                             type_signature: TypeSignature::OneOf(sigs),
+                             volatility: Volatility::Immutable
+                         } if  sigs.len() == 18));
+    }
+
+    #[test]
+    fn test_timestamp_date_sub() {
+        let f = DateSubFunction;
+
+        let times = vec![Some(123), None, Some(42), None];
+        // Intervals in milliseconds
+        let intervals = vec![1000, 2000, 3000, 1000];
+        let results = [Some(122), None, Some(39), None];
+
+        let time_vector = TimestampSecondVector::from(times.clone());
+        let interval_vector = IntervalDayTimeVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(time_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in times.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Timestamp(ts) => {
+                    assert_eq!(ts.value(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_date_date_sub() {
+        let f = DateSubFunction;
+        let days_per_month = 30;
+
+        let dates = vec![
+            Some(123 * days_per_month),
+            None,
+            Some(42 * days_per_month),
+            None,
+        ];
+        // Intervals in months
+        let intervals = vec![1, 2, 3, 1];
+        let results = [Some(3659), None, Some(1168), None];
+
+        let date_vector = DateVector::from(dates.clone());
+        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in dates.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Date(date) => {
+                    assert_eq!(date.val(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_datetime_date_sub() {
+        let f = DateSubFunction;
+        let millis_per_month = 3600 * 24 * 30 * 1000;
+
+        let dates = vec![
+            Some(123 * millis_per_month),
+            None,
+            Some(42 * millis_per_month),
+            None,
+        ];
+        // Intervals in months
+        let intervals = vec![1, 2, 3, 1];
+        let results = [Some(316137600000), None, Some(100915200000), None];
+
+        let date_vector = DateTimeVector::from(dates.clone());
+        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in dates.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::DateTime(date) => {
+                    assert_eq!(date.val(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+}

src/common/function/src/scalars/math.rs

@@ -28,9 +28,8 @@ pub use pow::PowFunction;
 pub use rate::RateFunction;
 use snafu::ResultExt;
 
-use super::function::FunctionContext;
-use super::Function;
-use crate::scalars::function_registry::FunctionRegistry;
+use crate::function::{Function, FunctionContext};
+use crate::function_registry::FunctionRegistry;
 
 pub(crate) struct MathFunction;
 

src/common/function/src/scalars/math/pow.rs

@@ -25,8 +25,8 @@ use datatypes::with_match_primitive_type_id;
 use num::traits::Pow;
 use num_traits::AsPrimitive;
 
+use crate::function::{Function, FunctionContext};
 use crate::scalars::expression::{scalar_binary_op, EvalContext};
-use crate::scalars::function::{Function, FunctionContext};
 
 #[derive(Clone, Debug, Default)]
 pub struct PowFunction;
@@ -83,6 +83,7 @@ mod tests {
     use datatypes::vectors::{Float32Vector, Int8Vector};
 
     use super::*;
+    use crate::function::FunctionContext;
     #[test]
     fn test_pow_function() {
         let pow = PowFunction;

src/common/function/src/scalars/math/rate.rs

@@ -23,7 +23,7 @@ use datatypes::prelude::*;
 use datatypes::vectors::{Helper, VectorRef};
 use snafu::ResultExt;
 
-use crate::scalars::function::{Function, FunctionContext};
+use crate::function::{Function, FunctionContext};
 
 /// generates rates from a sequence of adjacent data points.
 #[derive(Clone, Debug, Default)]

src/common/function/src/scalars/numpy.rs

@@ -19,7 +19,7 @@ use std::sync::Arc;
 
 use clip::ClipFunction;
 
-use crate::scalars::function_registry::FunctionRegistry;
+use crate::function_registry::FunctionRegistry;
 
 pub(crate) struct NumpyFunction;
 

src/common/function/src/scalars/numpy/clip.rs

@@ -24,8 +24,8 @@ use datatypes::prelude::*;
 use datatypes::vectors::PrimitiveVector;
 use paste::paste;
 
+use crate::function::{Function, FunctionContext};
 use crate::scalars::expression::{scalar_binary_op, EvalContext};
-use crate::scalars::function::{Function, FunctionContext};
 
 /// numpy.clip function, <https://numpy.org/doc/stable/reference/generated/numpy.clip.html>
 #[derive(Clone, Debug, Default)]

src/common/function/src/scalars/test.rs

@@ -20,8 +20,8 @@ use common_query::prelude::{Signature, Volatility};
 use datatypes::data_type::ConcreteDataType;
 use datatypes::prelude::VectorRef;
 
+use crate::function::{Function, FunctionContext};
 use crate::scalars::expression::{scalar_binary_op, EvalContext};
-use crate::scalars::function::{Function, FunctionContext};
 
 #[derive(Clone, Default)]
 pub(crate) struct TestAndFunction;

src/common/function/src/scalars/timestamp.rs

@@ -19,7 +19,7 @@ mod to_unixtime;
 use greatest::GreatestFunction;
 use to_unixtime::ToUnixtimeFunction;
 
-use crate::scalars::function_registry::FunctionRegistry;
+use crate::function_registry::FunctionRegistry;
 
 pub(crate) struct TimestampFunction;
 

src/common/function/src/scalars/timestamp/greatest.rs

@@ -27,7 +27,7 @@ use datatypes::prelude::ConcreteDataType;
 use datatypes::vectors::{Helper, VectorRef};
 use snafu::{ensure, ResultExt};
 
-use crate::scalars::function::{Function, FunctionContext};
+use crate::function::{Function, FunctionContext};
 
 #[derive(Clone, Debug, Default)]
 pub struct GreatestFunction;
@@ -113,10 +113,7 @@ mod tests {
     use datatypes::value::Value;
     use datatypes::vectors::{DateVector, StringVector, Vector};
 
-    use super::GreatestFunction;
-    use crate::scalars::function::FunctionContext;
-    use crate::scalars::Function;
-
+    use super::*;
     #[test]
     fn test_greatest_takes_string_vector() {
         let function = GreatestFunction;

src/common/function/src/scalars/timestamp/to_unixtime.rs

@@ -23,7 +23,7 @@ use datatypes::prelude::ConcreteDataType;
 use datatypes::vectors::{Int64Vector, VectorRef};
 use snafu::ensure;
 
-use crate::scalars::function::{Function, FunctionContext};
+use crate::function::{Function, FunctionContext};
 
 /// A function to convert the column into the unix timestamp in seconds.
 #[derive(Clone, Debug, Default)]
@@ -152,7 +152,6 @@ mod tests {
     };
 
     use super::{ToUnixtimeFunction, *};
-    use crate::scalars::Function;
 
     #[test]
     fn test_string_to_unixtime() {

src/common/function/src/scalars/udf.rs

@@ -23,7 +23,7 @@ use datatypes::prelude::*;
 use datatypes::vectors::Helper;
 use snafu::ResultExt;
 
-use crate::scalars::function::{FunctionContext, FunctionRef};
+use crate::function::{FunctionContext, FunctionRef};
 
 /// Create a ScalarUdf from function.
 pub fn create_udf(func: FunctionRef) -> ScalarUdf {
@@ -72,7 +72,7 @@ mod tests {
     use datatypes::vectors::{BooleanVector, ConstantVector};
 
     use super::*;
-    use crate::scalars::function::Function;
+    use crate::function::Function;
     use crate::scalars::test::TestAndFunction;
 
     #[test]

src/common/function/src/system.rs

@@ -12,14 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[derive(Debug)]
-pub struct WriteResponse {}
+pub mod build;
 
-#[derive(Debug)]
-pub struct ScanResponse<R> {
-    /// Reader to read result chunks.
-    pub reader: R,
+use std::sync::Arc;
+
+use build::BuildFunction;
+
+use crate::function_registry::FunctionRegistry;
+
+pub(crate) struct SystemFunction;
+
+impl SystemFunction {
+    pub fn register(registry: &FunctionRegistry) {
+        registry.register(Arc::new(BuildFunction));
+    }
 }
-
-#[derive(Debug)]
-pub struct GetResponse {}