feat: resend row insert

* feat: implement task tracker

* feat: implement handle upgrade region instruction

* refactor: remove redundant code

* chore: apply suggestions from CR

* chore: apply suggestions from CR

* refactor: refactor wait_for_replay_millis to wait_for_replay_timeout

* chore: apply suggestions from CR

* chore: apply suggestions from CR

.github/actions/build-greptime-binary/action.yml

@@ -40,9 +40,11 @@ runs:
     - name: Upload artifacts
       uses: ./.github/actions/upload-artifacts
       if: ${{ inputs.build-android-artifacts == 'false' }}
+      env:
+        PROFILE_TARGET: ${{ inputs.cargo-profile == 'dev' && 'debug' || inputs.cargo-profile }}
       with:
         artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: ./target/${{ inputs.cargo-profile }}/greptime
+        target-file: ./target/$PROFILE_TARGET/greptime
         version: ${{ inputs.version }}
         working-dir: ${{ inputs.working-dir }}
 

.github/actions/publish-github-release/action.yml

@@ -31,10 +31,12 @@ runs:
           echo "prerelease=false" >> $GITHUB_ENV
           echo "makeLatest=true" >> $GITHUB_ENV
           echo "generateReleaseNotes=false" >> $GITHUB_ENV
+          echo "omitBody=true" >> $GITHUB_ENV
         else
           echo "prerelease=true" >> $GITHUB_ENV
           echo "makeLatest=false" >> $GITHUB_ENV
           echo "generateReleaseNotes=true" >> $GITHUB_ENV
+          echo "omitBody=false" >> $GITHUB_ENV
         fi
 
     - name: Publish release
@@ -45,6 +47,7 @@ runs:
         makeLatest: ${{ env.makeLatest }}
         tag: ${{ inputs.version }}
         generateReleaseNotes: ${{ env.generateReleaseNotes }}
+        omitBody: ${{ env.omitBody }} # omitBody is true when the release is a official release.
         allowUpdates: true
         artifacts: |
           **/greptime-*/*

.github/actions/upload-artifacts/action.yml

@@ -22,7 +22,7 @@ runs:
       shell: bash
       run: |
         mkdir -p ${{ inputs.artifacts-dir }} && \
-        mv ${{ inputs.target-file }} ${{ inputs.artifacts-dir }}
+        cp ${{ inputs.target-file }} ${{ inputs.artifacts-dir }}
 
     # The compressed artifacts will use the following layout:
     # greptime-linux-amd64-pyo3-v0.3.0sha256sum

.github/doc-label-config.yml

@@ -0,0 +1,4 @@
+Doc not needed:
+    - '- \[x\]  This PR does not require documentation updates.'
+Doc update required:
+    - '- \[ \]  This PR does not require documentation updates.'

.github/pull_request_template.md

@@ -15,5 +15,6 @@ Please explain IN DETAIL what the changes are in this PR and why they are needed
 
 - [ ]  I have written the necessary rustdoc comments.
 - [ ]  I have added the necessary unit tests and integration tests.
+- [ ]  This PR does not require documentation updates.
 
 ## Refer to a related PR or issue link (optional)

.github/scripts/deploy-greptimedb.sh

@@ -107,12 +107,9 @@ function deploy_greptimedb_cluster_with_s3_storage() {
     --set storage.s3.bucket="$AWS_CI_TEST_BUCKET" \
     --set storage.s3.region="$AWS_REGION" \
     --set storage.s3.root="$DATA_ROOT" \
-    --set storage.s3.secretName=s3-credentials \
     --set storage.credentials.secretName=s3-credentials \
-    --set storage.credentials.secretCreation.enabled=true \
-    --set storage.credentials.secretCreation.enableEncryption=false \
-    --set storage.credentials.secretCreation.data.access-key-id="$AWS_ACCESS_KEY_ID" \
-    --set storage.credentials.secretCreation.data.secret-access-key="$AWS_SECRET_ACCESS_KEY"
+    --set storage.credentials.accessKeyId="$AWS_ACCESS_KEY_ID" \
+    --set storage.credentials.secretAccessKey="$AWS_SECRET_ACCESS_KEY"
 
   # Wait for greptimedb cluster to be ready.
   while true; do

.github/workflows/apidoc.yml

@@ -13,7 +13,7 @@ on:
 name: Build API docs
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-10-21
+  RUST_TOOLCHAIN: nightly-2023-12-19
 
 jobs:
   apidoc:

.github/workflows/dev-build.yml

@@ -55,10 +55,18 @@ on:
         description: Build and push images to DockerHub and ACR
         required: false
         default: true
+      cargo_profile:
+        type: choice
+        description: The cargo profile to use in building GreptimeDB.
+        default: nightly
+        options:
+          - dev
+          - release
+          - nightly
 
 # Use env variables to control all the release process.
 env:
-  CARGO_PROFILE: nightly
+  CARGO_PROFILE: ${{ inputs.cargo_profile }}
 
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
   DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}

.github/workflows/develop.yml

@@ -29,7 +29,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-10-21
+  RUST_TOOLCHAIN: nightly-2023-12-19
 
 jobs:
   typos:
@@ -175,6 +175,9 @@ jobs:
       - name: Setup etcd server
         working-directory: tests-integration/fixtures/etcd
         run: docker compose -f docker-compose-standalone.yml up -d --wait
+      - name: Setup kafka server
+        working-directory: tests-integration/fixtures/kafka
+        run: docker compose -f docker-compose-standalone.yml up -d --wait
       - name: Run nextest cases
         run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
         env:
@@ -186,6 +189,7 @@ jobs:
           GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
           GT_S3_REGION: ${{ secrets.S3_REGION }}
           GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
+          GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
           UNITTEST_LOG_DIR: "__unittest_logs"
       - name: Codecov upload
         uses: codecov/codecov-action@v2

.github/workflows/doc-label.yml

@@ -0,0 +1,20 @@
+name: "PR Doc Labeler"
+on:
+  pull_request_target:
+    types: [opened, edited, synchronize, ready_for_review, auto_merge_enabled, labeled, unlabeled]
+
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  triage:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: github/issue-labeler@v3.3
+      with:
+        configuration-path: .github/doc-label-config.yml
+        enable-versioned-regex: false
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        sync-labels: 1

.github/workflows/nightly-ci.yml

@@ -12,11 +12,12 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-10-21
+  RUST_TOOLCHAIN: nightly-2023-12-19
 
 jobs:
   sqlness:
     name: Sqlness Test
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -51,6 +52,7 @@ jobs:
           retention-days: 3
 
   test-on-windows:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: windows-latest-8-cores
     timeout-minutes: 60
     steps:

.github/workflows/nightly-funtional-tests.yml

@@ -9,6 +9,7 @@ on:
 jobs:
   sqlness-test:
     name: Run sqlness test
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout

.github/workflows/release.yml

@@ -82,7 +82,7 @@ on:
 # Use env variables to control all the release process.
 env:
   # The arguments of building greptime.
-  RUST_TOOLCHAIN: nightly-2023-10-21
+  RUST_TOOLCHAIN: nightly-2023-12-19
   CARGO_PROFILE: nightly
 
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
@@ -91,7 +91,7 @@ env:
   # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
   # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.5.0
+  NEXT_RELEASE_VERSION: v0.6.0
 
 jobs:
   allocate-runners:

.github/workflows/size-label.yml

@@ -1,11 +1,14 @@
 name: size-labeler
 
-on: [pull_request]
+on: [pull_request_target]
 
 jobs:
   labeler:
     runs-on: ubuntu-latest
     name: Label the PR size
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
       - uses: codelytv/pr-size-labeler@v1
         with:
@@ -18,9 +21,5 @@ jobs:
           l_max_size: '1000'
           xl_label: 'Size: XL'
           fail_if_xl: 'false'
-          message_if_xl: >
-            This PR exceeds the recommended size of 1000 lines.
-            Please make sure you are NOT addressing multiple issues with one PR.
-            Note this PR might be rejected due to its size.
-          github_api_url: 'api.github.com'
+          message_if_xl: ""
           files_to_ignore: 'Cargo.lock'

.github/workflows/user-doc-label-checker.yml

@@ -0,0 +1,19 @@
+name: Check user doc labels
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - labeled
+      - unlabeled
+
+jobs:
+
+  check_labels:
+    name: Check doc labels
+    runs-on: ubuntu-latest
+    steps:
+      - uses: docker://agilepathway/pull-request-label-checker:latest
+        with:
+          one_of: Doc update required,Doc not needed
+          repo_token: ${{ secrets.GITHUB_TOKEN }}

Cargo.toml

@@ -58,7 +58,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.4.4"
+version = "0.5.0"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -74,7 +74,10 @@ async-trait = "0.1"
 base64 = "0.21"
 bigdecimal = "0.4.2"
 bitflags = "2.4.1"
+bytemuck = "1.12"
+bytes = { version = "1.5", features = ["serde"] }
 chrono = { version = "0.4", features = ["serde"] }
+dashmap = "5.4"
 datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
 datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
 datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
@@ -87,11 +90,12 @@ etcd-client = "0.12"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "2b3ae45740a49ec6a0830d71fc09c3093aeb5fe7" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "a31ea166fc015ea7ff111ac94e26c3a5d64364d2" }
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
+mockall = "0.11.4"
 moka = "0.12"
 once_cell = "1.18"
 opentelemetry-proto = { git = "https://github.com/waynexia/opentelemetry-rust.git", rev = "33841b38dda79b15f2024952be5f32533325ca02", features = [
@@ -107,25 +111,28 @@ prost = "0.12"
 raft-engine = { git = "https://github.com/tikv/raft-engine.git", rev = "22dfb426cd994602b57725ef080287d3e53db479" }
 rand = "0.8"
 regex = "1.8"
+regex-automata = { version = "0.1", features = ["transducer"] }
 reqwest = { version = "0.11", default-features = false, features = [
     "json",
     "rustls-tls-native-roots",
     "stream",
 ] }
+rskafka = "0.5"
 rust_decimal = "1.33"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 smallvec = "1"
 snafu = "0.7"
 # on branch v0.38.x
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "0fbae07d0c46dc18e3381c406d8b9b8abef6b1fd", features = [
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "6a93567ae38d42be5c8d08b13c8ff4dde26502ef", features = [
     "visitor",
 ] }
 strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
 tokio = { version = "1.28", features = ["full"] }
+tokio-stream = { version = "0.1" }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
-toml = "0.7"
+toml = "0.8.8"
 tonic = { version = "0.10", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
 
@@ -165,7 +172,7 @@ frontend = { path = "src/frontend" }
 log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
 meta-srv = { path = "src/meta-srv" }
-mito = { path = "src/mito" }
+metric-engine = { path = "src/metric-engine" }
 mito2 = { path = "src/mito2" }
 object-store = { path = "src/object-store" }
 operator = { path = "src/operator" }
@@ -186,7 +193,7 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git"
 rev = "abbd357c1e193cd270ea65ee7652334a150b628f"
 
 [profile.release]
-debug = true
+debug = 1
 
 [profile.nightly]
 inherits = "release"

README.md

@@ -100,7 +100,7 @@ Please see the online document site for more installation options and [operation
 
 ### Get started
 
-Read the [complete getting started guide](https://docs.greptime.com/getting-started/try-out-greptimedb) on our [official document site](https://docs.greptime.com/).
+Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview) on our [official document site](https://docs.greptime.com/).
 
 To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients/overview).
 

config/datanode.example.toml

@@ -12,6 +12,10 @@ rpc_runtime_size = 8
 # It will block the datanode start if it can't receive leases in the heartbeat from metasrv.
 require_lease_before_startup = false
 
+# Initialize all regions in the background during the startup.
+# By default, it provides services after all regions have been initialized.
+initialize_region_in_background = false
+
 [heartbeat]
 # Interval for sending heartbeat messages to the Metasrv, 3 seconds by default.
 interval = "3s"
@@ -29,9 +33,15 @@ connect_timeout = "1s"
 # `TCP_NODELAY` option for accepted connections, true by default.
 tcp_nodelay = true
 
-# WAL options, see `standalone.example.toml`.
+# WAL options.
+# Currently, users are expected to choose the wal through the provider field. 
+# When a wal provider is chose, the user should comment out all other wal config 
+# except those corresponding to the chosen one.
 [wal]
 # WAL data directory
+provider = "raft_engine"
+
+# Raft-engine wal options, see `standalone.example.toml`.
 # dir = "/tmp/greptimedb/wal"
 file_size = "256MB"
 purge_threshold = "4GB"
@@ -39,10 +49,21 @@ purge_interval = "10m"
 read_batch_size = 128
 sync_write = false
 
+# Kafka wal options, see `standalone.example.toml`.
+# broker_endpoints = ["127.0.0.1:9092"]
+# max_batch_size = "4MB"
+# linger = "200ms"
+# produce_record_timeout = "100ms"
+# backoff_init = "500ms"
+# backoff_max = "10s"
+# backoff_base = 2
+# backoff_deadline = "5mins"
+
 # Storage options, see `standalone.example.toml`.
 [storage]
 # The working home directory.
 data_home = "/tmp/greptimedb/"
+# Storage type.
 type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
@@ -53,6 +74,12 @@ type = "File"
 # The local file cache capacity in bytes.
 # cache_capacity = "256MB"
 
+# Custom storage options
+#[[storage.providers]]
+#type = "S3"
+#[[storage.providers]]
+#type = "Gcs"
+
 # Mito engine options
 [[region_engine]]
 [region_engine.mito]
@@ -82,8 +109,31 @@ vector_cache_size = "512MB"
 page_cache_size = "512MB"
 # Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
+# Parallelism to scan a region (default: 1/4 of cpu cores).
+# - 0: using the default value (1/4 of cpu cores).
+# - 1: scan in current thread.
+# - n: scan in parallelism n.
+scan_parallelism = 0
+# Capacity of the channel to send data from parallel scan tasks to the main task (default 32).
+parallel_scan_channel_size = 32
 
 # Log options, see `standalone.example.toml`
 # [logging]
 # dir = "/tmp/greptimedb/logs"
 # level = "info"
+
+# Datanode export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
+# [export_metrics]
+# whether enable export metrics, default is false
+# enable = false
+# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metrics
+# write_interval = "30s"
+# HTTP headers of Prometheus remote-write carry
+# headers = {}

config/frontend.example.toml

@@ -1,5 +1,7 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
+# The default timezone of the server
+# default_timezone = "UTC"
 
 [heartbeat]
 # Interval for sending heartbeat task to the Metasrv, 5 seconds by default.
@@ -77,3 +79,19 @@ tcp_nodelay = true
 timeout = "10s"
 connect_timeout = "10s"
 tcp_nodelay = true
+
+# Frontend export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
+# [export_metrics]
+# whether enable export metrics, default is false
+# enable = false
+# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metrics
+# write_interval = "30s"
+# HTTP headers of Prometheus remote-write carry
+# headers = {}

config/metasrv.example.toml

@@ -7,14 +7,16 @@ server_addr = "127.0.0.1:3002"
 # Etcd server address, "127.0.0.1:2379" by default.
 store_addr = "127.0.0.1:2379"
 # Datanode selector type.
-# - "LeaseBased" (default value).
-# - "LoadBased"
-# For details, please see "https://docs.greptime.com/developer-guide/meta/selector".
-selector = "LeaseBased"
+# - "lease_based" (default value).
+# - "load_based"
+# For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector".
+selector = "lease_based"
 # Store data in memory, false by default.
 use_memory_store = false
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
+# If it's not empty, the metasrv will store all data with this key prefix.
+store_key_prefix = ""
 
 # Log options, see `standalone.example.toml`
 # [logging]
@@ -42,3 +44,53 @@ first_heartbeat_estimate = "1000ms"
 # timeout = "10s"
 # connect_timeout = "10s"
 # tcp_nodelay = true
+
+[wal]
+# Available wal providers:
+# - "raft_engine" (default)
+# - "kafka"
+provider = "raft_engine"
+
+# There're none raft-engine wal config since meta srv only involves in remote wal currently.
+
+# Kafka wal config.
+# The broker endpoints of the Kafka cluster. ["127.0.0.1:9092"] by default.
+# broker_endpoints = ["127.0.0.1:9092"]
+# Number of topics to be created upon start.
+# num_topics = 64
+# Topic selector type.
+# Available selector types: 
+# - "round_robin" (default)
+# selector_type = "round_robin"
+# A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
+# topic_name_prefix = "greptimedb_wal_topic"
+# Number of partitions per topic.
+# num_partitions = 1
+# Expected number of replicas of each partition.
+# replication_factor = 1
+# Above which a topic creation operation will be cancelled.
+# create_topic_timeout = "30s"
+# The initial backoff for kafka clients.
+# backoff_init = "500ms"
+# The maximum backoff for kafka clients.
+# backoff_max = "10s"
+# Exponential backoff rate, i.e. next backoff = base * current backoff.
+# backoff_base = 2
+# Stop reconnecting if the total wait time reaches the deadline. If this config is missing, the reconnecting won't terminate.
+# backoff_deadline = "5mins"
+
+# Metasrv export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
+# [export_metrics]
+# whether enable export metrics, default is false
+# enable = false
+# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metrics
+# write_interval = "30s"
+# HTTP headers of Prometheus remote-write carry
+# headers = {}

config/standalone.example.toml

@@ -2,6 +2,8 @@
 mode = "standalone"
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
+# The default timezone of the server
+# default_timezone = "UTC"
 
 # HTTP server options.
 [http]
@@ -80,8 +82,49 @@ enable = true
 # Whether to enable Prometheus remote write and read in HTTP API, true by default.
 enable = true
 
-# WAL options.
 [wal]
+# Available wal providers:
+# - "raft_engine" (default)
+# - "kafka"
+provider = "raft_engine"
+
+# There're none raft-engine wal config since meta srv only involves in remote wal currently.
+
+# Kafka wal options.
+# The broker endpoints of the Kafka cluster. ["127.0.0.1:9092"] by default.
+# broker_endpoints = ["127.0.0.1:9092"]
+
+# Number of topics to be created upon start.
+# num_topics = 64
+# Topic selector type.
+# Available selector types: 
+# - "round_robin" (default)
+# selector_type = "round_robin"
+# A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
+# topic_name_prefix = "greptimedb_wal_topic"
+# Number of partitions per topic.
+# num_partitions = 1
+# Expected number of replicas of each partition.
+# replication_factor = 1
+
+# The maximum log size a kafka batch producer could buffer.
+# max_batch_size = "4MB"
+# The linger duration of a kafka batch producer.
+# linger = "200ms"
+# The maximum amount of time (in milliseconds) to wait for Kafka records to be returned.
+# produce_record_timeout = "100ms"
+# Above which a topic creation operation will be cancelled.
+# create_topic_timeout = "30s"
+
+# The initial backoff for kafka clients.
+# backoff_init = "500ms"
+# The maximum backoff for kafka clients.
+# backoff_max = "10s"
+# Exponential backoff rate, i.e. next backoff = base * current backoff.
+# backoff_base = 2
+# Stop reconnecting if the total wait time reaches the deadline. If this config is missing, the reconnecting won't terminate.
+# backoff_deadline = "5mins"
+
 # WAL data directory
 # dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
@@ -122,6 +165,12 @@ type = "File"
 # The local file cache capacity in bytes.
 # cache_capacity = "256MB"
 
+# Custom storage options
+#[[storage.providers]]
+#type = "S3"
+#[[storage.providers]]
+#type = "Gcs"
+
 # Mito engine options
 [[region_engine]]
 [region_engine.mito]
@@ -151,6 +200,13 @@ vector_cache_size = "512MB"
 page_cache_size = "512MB"
 # Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
+# Parallelism to scan a region (default: 1/4 of cpu cores).
+# - 0: using the default value (1/4 of cpu cores).
+# - 1: scan in current thread.
+# - n: scan in parallelism n.
+scan_parallelism = 0
+# Capacity of the channel to send data from parallel scan tasks to the main task (default 32).
+parallel_scan_channel_size = 32
 
 # Log options
 # [logging]
@@ -164,3 +220,21 @@ sst_write_buffer_size = "8MB"
 # otlp_endpoint = "localhost:4317"
 # The percentage of tracing will be sampled and exported. Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1. ratio > 1 are treated as 1. Fractions < 0 are treated as 0
 # tracing_sample_ratio = 1.0
+# Whether to append logs to stdout. Defaults to true.
+# append_stdout = true
+
+# Standalone export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
+# [export_metrics]
+# whether enable export metrics, default is false
+# enable = false
+# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metrics
+# write_interval = "30s"
+# HTTP headers of Prometheus remote-write carry
+# headers = {}

docker/dev-builder/centos/Dockerfile

@@ -26,4 +26,5 @@ ARG RUST_TOOLCHAIN
 RUN rustup toolchain install ${RUST_TOOLCHAIN}
 
 # Install nextest.
-RUN cargo install cargo-nextest --locked
+RUN cargo install cargo-binstall --locked
+RUN cargo binstall cargo-nextest --no-confirm

docker/dev-builder/ubuntu/Dockerfile

@@ -43,4 +43,5 @@ ARG RUST_TOOLCHAIN
 RUN rustup toolchain install ${RUST_TOOLCHAIN}
 
 # Install nextest.
-RUN cargo install cargo-nextest --locked
+RUN cargo install cargo-binstall --locked
+RUN cargo binstall cargo-nextest --no-confirm

docker/dev-builder/ubuntu/Dockerfile-18.10

@@ -44,4 +44,5 @@ ARG RUST_TOOLCHAIN
 RUN rustup toolchain install ${RUST_TOOLCHAIN}
 
 # Install nextest.
-RUN cargo install cargo-nextest --locked
+RUN cargo install cargo-binstall --locked
+RUN cargo binstall cargo-nextest --no-confirm

docs/rfcs/2023-12-22-enclose-column-id.md

@@ -0,0 +1,44 @@
+---
+Feature Name: Enclose Column Id
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/2982
+Date: 2023-12-22
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+# Summary
+This RFC proposes to enclose the usage of `ColumnId` into the region engine only.
+
+# Motivation
+`ColumnId` is an identifier for columns. It's assigned by meta server, stored in `TableInfo` and `RegionMetadata` and used in region engine to distinguish columns.
+
+At present, Both Frontend, Datanode and Metasrv are aware of `ColumnId` but it's only used in region engine. Thus this RFC proposes to remove it from Frontend (mainly used in `TableInfo`) and Metasrv.
+
+# Details
+
+`ColumnId` is used widely on both read and write paths. Removing it from Frontend and Metasrv implies several things:
+
+- A column may have different column id in different regions.
+- A column is identified by its name in all components.
+- Column order in the region engine is not restricted, i.e., no need to be in the same order with table info.
+
+The first thing doesn't matter IMO. This concept doesn't exist anymore outside of region server, and each region is autonomous and independent -- the only guarantee it should hold is those columns exist. But if we consider region repartition, where the SST file would be re-assign to different regions, things would become a bit more complicated. A possible solution is store the relation between name and ColumnId in the manifest, but it's out of the scope of this RFC. We can likely give a workaround by introducing a indirection mapping layer of different version of partitions.
+
+And more importantly, we can still assume columns have the same column ids across regions. We have procedure to maintain consistency between regions and the region engine should ensure alterations are idempotent. So it is possible that region repartition doesn't need to consider column ids or other region metadata in the future.
+
+Users write and query column by their names, not by ColumnId or something else. The second point also means to change the column reference in ScanRequest from index to name. This change can hugely alleviate the misuse of the column index, which has given us many surprises.
+
+And for the last one, column order only matters in table info. This order is used in user-faced table structure operation, like add column, describe column or as the default order of INSERT clause. None of them is connected with the order in storage.
+
+# Drawback
+Firstly, this is a breaking change. Delivering this change requires a full upgrade of the cluster. Secondly, this change may introduce some performance regression. For example, we have to pass the full table name in the `ScanRequest` instead of the `ColumnId`. But this influence is very limited, since the column index is only used in the region engine.
+
+# Alternatives
+
+There are two alternatives from the perspective of "what can be used as the column identifier":
+
+- Index of column to the table schema
+- `ColumnId` of that column
+
+The first one is what we are using now. By choosing this way, it's required to keep the column order in the region engine the same as the table info. This is not hard to achieve, but it's a bit annoying. And things become tricky when there is internal column or different schemas like those stored in file format. And this is the initial purpose of this RFC, which is trying to decouple the table schema and region schema.
+
+The second one, in other hand, requires the `ColumnId` should be identical in all regions and `TableInfo`. It has the same drawback with the previous alternative, that the `TableInfo` and `RegionMetadata` are tighted together. Another point is that the `ColumnId` is assigned by the Metasrv, who doesn't need it but have to maintain it. And this also limits the functionality of `ColumnId`, by taking the ability of assigning it from concrete region engine.

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-10-21"
+channel = "nightly-2023-12-19"

src/api/src/helper.rs

@@ -535,11 +535,8 @@ pub fn convert_i128_to_interval(v: i128) -> v1::IntervalMonthDayNano {
 
 /// Convert common decimal128 to grpc decimal128 without precision and scale.
 pub fn convert_to_pb_decimal128(v: Decimal128) -> v1::Decimal128 {
-    let value = v.val();
-    v1::Decimal128 {
-        hi: (value >> 64) as i64,
-        lo: value as i64,
-    }
+    let (hi, lo) = v.split_value();
+    v1::Decimal128 { hi, lo }
 }
 
 pub fn pb_value_to_value_ref<'a>(
@@ -580,9 +577,9 @@ pub fn pb_value_to_value_ref<'a>(
         ValueData::TimeMillisecondValue(t) => ValueRef::Time(Time::new_millisecond(*t)),
         ValueData::TimeMicrosecondValue(t) => ValueRef::Time(Time::new_microsecond(*t)),
         ValueData::TimeNanosecondValue(t) => ValueRef::Time(Time::new_nanosecond(*t)),
-        ValueData::IntervalYearMonthValues(v) => ValueRef::Interval(Interval::from_i32(*v)),
-        ValueData::IntervalDayTimeValues(v) => ValueRef::Interval(Interval::from_i64(*v)),
-        ValueData::IntervalMonthDayNanoValues(v) => {
+        ValueData::IntervalYearMonthValue(v) => ValueRef::Interval(Interval::from_i32(*v)),
+        ValueData::IntervalDayTimeValue(v) => ValueRef::Interval(Interval::from_i64(*v)),
+        ValueData::IntervalMonthDayNanoValue(v) => {
             let interval = Interval::from_month_day_nano(v.months, v.days, v.nanoseconds);
             ValueRef::Interval(interval)
         }
@@ -986,13 +983,13 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
         },
         Value::Interval(v) => match v.unit() {
             IntervalUnit::YearMonth => v1::Value {
-                value_data: Some(ValueData::IntervalYearMonthValues(v.to_i32())),
+                value_data: Some(ValueData::IntervalYearMonthValue(v.to_i32())),
             },
             IntervalUnit::DayTime => v1::Value {
-                value_data: Some(ValueData::IntervalDayTimeValues(v.to_i64())),
+                value_data: Some(ValueData::IntervalDayTimeValue(v.to_i64())),
             },
             IntervalUnit::MonthDayNano => v1::Value {
-                value_data: Some(ValueData::IntervalMonthDayNanoValues(
+                value_data: Some(ValueData::IntervalMonthDayNanoValue(
                     convert_i128_to_interval(v.to_i128()),
                 )),
             },
@@ -1011,12 +1008,9 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
                 value_data: Some(ValueData::DurationNanosecondValue(v.value())),
             },
         },
-        Value::Decimal128(v) => {
-            let (hi, lo) = v.split_value();
-            v1::Value {
-                value_data: Some(ValueData::Decimal128Value(v1::Decimal128 { hi, lo })),
-            }
-        }
+        Value::Decimal128(v) => v1::Value {
+            value_data: Some(ValueData::Decimal128Value(convert_to_pb_decimal128(v))),
+        },
         Value::List(_) => return None,
     };
 
@@ -1051,9 +1045,9 @@ pub fn proto_value_type(value: &v1::Value) -> Option<ColumnDataType> {
         ValueData::TimeMillisecondValue(_) => ColumnDataType::TimeMillisecond,
         ValueData::TimeMicrosecondValue(_) => ColumnDataType::TimeMicrosecond,
         ValueData::TimeNanosecondValue(_) => ColumnDataType::TimeNanosecond,
-        ValueData::IntervalYearMonthValues(_) => ColumnDataType::IntervalYearMonth,
-        ValueData::IntervalDayTimeValues(_) => ColumnDataType::IntervalDayTime,
-        ValueData::IntervalMonthDayNanoValues(_) => ColumnDataType::IntervalMonthDayNano,
+        ValueData::IntervalYearMonthValue(_) => ColumnDataType::IntervalYearMonth,
+        ValueData::IntervalDayTimeValue(_) => ColumnDataType::IntervalDayTime,
+        ValueData::IntervalMonthDayNanoValue(_) => ColumnDataType::IntervalMonthDayNano,
         ValueData::DurationSecondValue(_) => ColumnDataType::DurationSecond,
         ValueData::DurationMillisecondValue(_) => ColumnDataType::DurationMillisecond,
         ValueData::DurationMicrosecondValue(_) => ColumnDataType::DurationMicrosecond,
@@ -1109,10 +1103,10 @@ pub fn value_to_grpc_value(value: Value) -> GrpcValue {
                 TimeUnit::Nanosecond => ValueData::TimeNanosecondValue(v.value()),
             }),
             Value::Interval(v) => Some(match v.unit() {
-                IntervalUnit::YearMonth => ValueData::IntervalYearMonthValues(v.to_i32()),
-                IntervalUnit::DayTime => ValueData::IntervalDayTimeValues(v.to_i64()),
+                IntervalUnit::YearMonth => ValueData::IntervalYearMonthValue(v.to_i32()),
+                IntervalUnit::DayTime => ValueData::IntervalDayTimeValue(v.to_i64()),
                 IntervalUnit::MonthDayNano => {
-                    ValueData::IntervalMonthDayNanoValues(convert_i128_to_interval(v.to_i128()))
+                    ValueData::IntervalMonthDayNanoValue(convert_i128_to_interval(v.to_i128()))
                 }
             }),
             Value::Duration(v) => Some(match v.unit() {
@@ -1121,10 +1115,7 @@ pub fn value_to_grpc_value(value: Value) -> GrpcValue {
                 TimeUnit::Microsecond => ValueData::DurationMicrosecondValue(v.value()),
                 TimeUnit::Nanosecond => ValueData::DurationNanosecondValue(v.value()),
             }),
-            Value::Decimal128(v) => {
-                let (hi, lo) = v.split_value();
-                Some(ValueData::Decimal128Value(v1::Decimal128 { hi, lo }))
-            }
+            Value::Decimal128(v) => Some(ValueData::Decimal128Value(convert_to_pb_decimal128(v))),
             Value::List(_) => unreachable!(),
         },
     }

src/catalog/Cargo.toml

@@ -13,6 +13,7 @@ arc-swap = "1.0"
 arrow-schema.workspace = true
 async-stream.workspace = true
 async-trait = "0.1"
+build-data = "0.1"
 common-catalog.workspace = true
 common-error.workspace = true
 common-grpc.workspace = true
@@ -23,7 +24,7 @@ common-recordbatch.workspace = true
 common-runtime.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
-dashmap = "5.4"
+dashmap.workspace = true
 datafusion.workspace = true
 datatypes.workspace = true
 futures = "0.3"
@@ -33,10 +34,11 @@ meta-client.workspace = true
 moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 partition.workspace = true
+paste = "1.0"
 prometheus.workspace = true
 regex.workspace = true
 serde.workspace = true
-serde_json = "1.0"
+serde_json.workspace = true
 session.workspace = true
 snafu.workspace = true
 store-api.workspace = true

src/catalog/src/information_schema.rs

@@ -13,16 +13,20 @@
 // limitations under the License.
 
 mod columns;
+mod memory_table;
+mod table_names;
 mod tables;
 
 use std::collections::HashMap;
 use std::sync::{Arc, Weak};
 
-use common_catalog::consts::INFORMATION_SCHEMA_NAME;
+use common_catalog::consts::{self, INFORMATION_SCHEMA_NAME};
 use common_error::ext::BoxedError;
-use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
+use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
 use datatypes::schema::SchemaRef;
 use futures_util::StreamExt;
+use lazy_static::lazy_static;
+use paste::paste;
 use snafu::ResultExt;
 use store_api::data_source::DataSource;
 use store_api::storage::{ScanRequest, TableId};
@@ -32,43 +36,102 @@ use table::metadata::{
 };
 use table::thin_table::{ThinTable, ThinTableAdapter};
 use table::TableRef;
+pub use table_names::*;
 
 use self::columns::InformationSchemaColumns;
 use crate::error::Result;
+use crate::information_schema::memory_table::{get_schema_columns, MemoryTable};
 use crate::information_schema::tables::InformationSchemaTables;
 use crate::CatalogManager;
 
-pub const TABLES: &str = "tables";
-pub const COLUMNS: &str = "columns";
+lazy_static! {
+    // Memory tables in `information_schema`.
+    static ref MEMORY_TABLES: &'static [&'static str] = &[
+        ENGINES,
+        COLUMN_PRIVILEGES,
+        COLUMN_STATISTICS,
+        BUILD_INFO,
+    ];
+}
 
+macro_rules! setup_memory_table {
+    ($name: expr) => {
+        paste! {
+            {
+                let (schema, columns) = get_schema_columns($name);
+                Some(Arc::new(MemoryTable::new(
+                    consts::[<INFORMATION_SCHEMA_ $name  _TABLE_ID>],
+                    $name,
+                    schema,
+                    columns
+                )) as _)
+            }
+        }
+    };
+}
+
+/// The `information_schema` tables info provider.
 pub struct InformationSchemaProvider {
     catalog_name: String,
     catalog_manager: Weak<dyn CatalogManager>,
+    tables: HashMap<String, TableRef>,
 }
 
 impl InformationSchemaProvider {
     pub fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
-        Self {
+        let mut provider = Self {
             catalog_name,
             catalog_manager,
-        }
+            tables: HashMap::new(),
+        };
+
+        provider.build_tables();
+
+        provider
     }
 
-    /// Build a map of [TableRef] in information schema.
-    /// Including `tables` and `columns`.
-    pub fn build(
-        catalog_name: String,
-        catalog_manager: Weak<dyn CatalogManager>,
-    ) -> HashMap<String, TableRef> {
-        let provider = Self::new(catalog_name, catalog_manager);
+    /// Returns table names in the order of table id.
+    pub fn table_names(&self) -> Vec<String> {
+        let mut tables = self.tables.values().clone().collect::<Vec<_>>();
 
-        let mut schema = HashMap::new();
-        schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap());
-        schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap());
-        schema
+        tables.sort_by(|t1, t2| {
+            t1.table_info()
+                .table_id()
+                .partial_cmp(&t2.table_info().table_id())
+                .unwrap()
+        });
+        tables
+            .into_iter()
+            .map(|t| t.table_info().name.clone())
+            .collect()
     }
 
+    /// Returns a map of [TableRef] in information schema.
+    pub fn tables(&self) -> &HashMap<String, TableRef> {
+        assert!(!self.tables.is_empty());
+
+        &self.tables
+    }
+
+    /// Returns the [TableRef] by table name.
     pub fn table(&self, name: &str) -> Option<TableRef> {
+        self.tables.get(name).cloned()
+    }
+
+    fn build_tables(&mut self) {
+        let mut tables = HashMap::new();
+        tables.insert(TABLES.to_string(), self.build_table(TABLES).unwrap());
+        tables.insert(COLUMNS.to_string(), self.build_table(COLUMNS).unwrap());
+
+        // Add memory tables
+        for name in MEMORY_TABLES.iter() {
+            tables.insert((*name).to_string(), self.build_table(name).unwrap());
+        }
+
+        self.tables = tables;
+    }
+
+    fn build_table(&self, name: &str) -> Option<TableRef> {
         self.information_table(name).map(|table| {
             let table_info = Self::table_info(self.catalog_name.clone(), &table);
             let filter_pushdown = FilterPushDownType::Unsupported;
@@ -89,6 +152,10 @@ impl InformationSchemaProvider {
                 self.catalog_name.clone(),
                 self.catalog_manager.clone(),
             )) as _),
+            ENGINES => setup_memory_table!(ENGINES),
+            COLUMN_PRIVILEGES => setup_memory_table!(COLUMN_PRIVILEGES),
+            COLUMN_STATISTICS => setup_memory_table!(COLUMN_STATISTICS),
+            BUILD_INFO => setup_memory_table!(BUILD_INFO),
             _ => None,
         }
     }
@@ -102,9 +169,9 @@ impl InformationSchemaProvider {
             .unwrap();
         let table_info = TableInfoBuilder::default()
             .table_id(table.table_id())
-            .name(table.table_name().to_owned())
+            .name(table.table_name().to_string())
             .catalog_name(catalog_name)
-            .schema_name(INFORMATION_SCHEMA_NAME.to_owned())
+            .schema_name(INFORMATION_SCHEMA_NAME.to_string())
             .meta(table_meta)
             .table_type(table.table_type())
             .build()
@@ -171,11 +238,12 @@ impl DataSource for InformationTableDataSource {
                 None => batch,
             });
 
-        let stream = RecordBatchStreamAdaptor {
+        let stream = RecordBatchStreamWrapper {
             schema: projected_schema,
             stream: Box::pin(stream),
             output_ordering: None,
         };
+
         Ok(Box::pin(stream))
     }
 }

src/catalog/src/information_schema/columns.rs

@@ -16,8 +16,8 @@ use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::{
-    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME, SEMANTIC_TYPE_FIELD,
-    SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
+    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_PRIMARY_KEY,
+    SEMANTIC_TYPE_TIME_INDEX,
 };
 use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
@@ -33,8 +33,7 @@ use datatypes::vectors::{StringVectorBuilder, VectorRef};
 use snafu::{OptionExt, ResultExt};
 use store_api::storage::TableId;
 
-use super::tables::InformationSchemaTables;
-use super::{InformationTable, COLUMNS, TABLES};
+use super::{InformationTable, COLUMNS};
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
@@ -102,7 +101,7 @@ impl InformationTable for InformationSchemaColumns {
             schema,
             futures::stream::once(async move {
                 builder
-                    .make_tables()
+                    .make_columns()
                     .await
                     .map(|x| x.into_df_record_batch())
                     .map_err(Into::into)
@@ -148,8 +147,8 @@ impl InformationSchemaColumnsBuilder {
         }
     }
 
-    /// Construct the `information_schema.tables` virtual table
-    async fn make_tables(&mut self) -> Result<RecordBatch> {
+    /// Construct the `information_schema.columns` virtual table
+    async fn make_columns(&mut self) -> Result<RecordBatch> {
         let catalog_name = self.catalog_name.clone();
         let catalog_manager = self
             .catalog_manager
@@ -163,48 +162,38 @@ impl InformationSchemaColumnsBuilder {
             {
                 continue;
             }
+
             for table_name in catalog_manager
                 .table_names(&catalog_name, &schema_name)
                 .await?
             {
-                let (keys, schema) = if let Some(table) = catalog_manager
+                if let Some(table) = catalog_manager
                     .table(&catalog_name, &schema_name, &table_name)
                     .await?
                 {
                     let keys = &table.table_info().meta.primary_key_indices;
                     let schema = table.schema();
-                    (keys.clone(), schema)
-                } else {
-                    // TODO: this specific branch is only a workaround for FrontendCatalogManager.
-                    if schema_name == INFORMATION_SCHEMA_NAME {
-                        if table_name == COLUMNS {
-                            (vec![], InformationSchemaColumns::schema())
-                        } else if table_name == TABLES {
-                            (vec![], InformationSchemaTables::schema())
-                        } else {
-                            continue;
-                        }
-                    } else {
-                        continue;
-                    }
-                };
 
-                for (idx, column) in schema.column_schemas().iter().enumerate() {
-                    let semantic_type = if column.is_time_index() {
-                        SEMANTIC_TYPE_TIME_INDEX
-                    } else if keys.contains(&idx) {
-                        SEMANTIC_TYPE_PRIMARY_KEY
-                    } else {
-                        SEMANTIC_TYPE_FIELD
-                    };
-                    self.add_column(
-                        &catalog_name,
-                        &schema_name,
-                        &table_name,
-                        &column.name,
-                        column.data_type.name(),
-                        semantic_type,
-                    );
+                    for (idx, column) in schema.column_schemas().iter().enumerate() {
+                        let semantic_type = if column.is_time_index() {
+                            SEMANTIC_TYPE_TIME_INDEX
+                        } else if keys.contains(&idx) {
+                            SEMANTIC_TYPE_PRIMARY_KEY
+                        } else {
+                            SEMANTIC_TYPE_FIELD
+                        };
+
+                        self.add_column(
+                            &catalog_name,
+                            &schema_name,
+                            &table_name,
+                            &column.name,
+                            &column.data_type.name(),
+                            semantic_type,
+                        );
+                    }
+                } else {
+                    unreachable!();
                 }
             }
         }
@@ -238,6 +227,7 @@ impl InformationSchemaColumnsBuilder {
             Arc::new(self.data_types.finish()),
             Arc::new(self.semantic_types.finish()),
         ];
+
         RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
     }
 }
@@ -254,7 +244,7 @@ impl DfPartitionStream for InformationSchemaColumns {
             schema,
             futures::stream::once(async move {
                 builder
-                    .make_tables()
+                    .make_columns()
                     .await
                     .map(|x| x.into_df_record_batch())
                     .map_err(Into::into)

src/catalog/src/information_schema/memory_table.rs

@@ -0,0 +1,214 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod tables;
+use std::sync::Arc;
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_error::ext::BoxedError;
+use common_query::physical_plan::TaskContext;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
+use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
+use datatypes::schema::SchemaRef;
+use datatypes::vectors::VectorRef;
+use snafu::ResultExt;
+use store_api::storage::TableId;
+pub use tables::get_schema_columns;
+
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
+use crate::information_schema::InformationTable;
+
+/// A memory table with specified schema and columns.
+pub(super) struct MemoryTable {
+    table_id: TableId,
+    table_name: &'static str,
+    schema: SchemaRef,
+    columns: Vec<VectorRef>,
+}
+
+impl MemoryTable {
+    /// Creates a memory table with table id, name, schema and columns.
+    pub(super) fn new(
+        table_id: TableId,
+        table_name: &'static str,
+        schema: SchemaRef,
+        columns: Vec<VectorRef>,
+    ) -> Self {
+        Self {
+            table_id,
+            table_name,
+            schema,
+            columns,
+        }
+    }
+
+    fn builder(&self) -> MemoryTableBuilder {
+        MemoryTableBuilder::new(self.schema.clone(), self.columns.clone())
+    }
+}
+
+impl InformationTable for MemoryTable {
+    fn table_id(&self) -> TableId {
+        self.table_id
+    }
+
+    fn table_name(&self) -> &'static str {
+        self.table_name
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .memory_records()
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+struct MemoryTableBuilder {
+    schema: SchemaRef,
+    columns: Vec<VectorRef>,
+}
+
+impl MemoryTableBuilder {
+    fn new(schema: SchemaRef, columns: Vec<VectorRef>) -> Self {
+        Self { schema, columns }
+    }
+
+    /// Construct the `information_schema.{table_name}` virtual table
+    async fn memory_records(&mut self) -> Result<RecordBatch> {
+        if self.columns.is_empty() {
+            RecordBatch::new_empty(self.schema.clone()).context(CreateRecordBatchSnafu)
+        } else {
+            RecordBatch::new(self.schema.clone(), std::mem::take(&mut self.columns))
+                .context(CreateRecordBatchSnafu)
+        }
+    }
+}
+
+impl DfPartitionStream for MemoryTable {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .memory_records()
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_recordbatch::RecordBatches;
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::schema::{ColumnSchema, Schema};
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_memory_table() {
+        let schema = Arc::new(Schema::new(vec![
+            ColumnSchema::new("a", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("b", ConcreteDataType::string_datatype(), false),
+        ]));
+
+        let table = MemoryTable::new(
+            42,
+            "test",
+            schema.clone(),
+            vec![
+                Arc::new(StringVector::from(vec!["a1", "a2"])),
+                Arc::new(StringVector::from(vec!["b1", "b2"])),
+            ],
+        );
+
+        assert_eq!(42, table.table_id());
+        assert_eq!("test", table.table_name());
+        assert_eq!(schema, InformationTable::schema(&table));
+
+        let stream = table.to_stream().unwrap();
+
+        let batches = RecordBatches::try_collect(stream).await.unwrap();
+
+        assert_eq!(
+            "\
++----+----+
+| a  | b  |
++----+----+
+| a1 | b1 |
+| a2 | b2 |
++----+----+",
+            batches.pretty_print().unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_empty_memory_table() {
+        let schema = Arc::new(Schema::new(vec![
+            ColumnSchema::new("a", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("b", ConcreteDataType::string_datatype(), false),
+        ]));
+
+        let table = MemoryTable::new(42, "test", schema.clone(), vec![]);
+
+        assert_eq!(42, table.table_id());
+        assert_eq!("test", table.table_name());
+        assert_eq!(schema, InformationTable::schema(&table));
+
+        let stream = table.to_stream().unwrap();
+
+        let batches = RecordBatches::try_collect(stream).await.unwrap();
+
+        assert_eq!(
+            "\
++---+---+
+| a | b |
++---+---+
++---+---+",
+            batches.pretty_print().unwrap()
+        );
+    }
+}

src/catalog/src/information_schema/memory_table/tables.rs

@@ -0,0 +1,135 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use common_catalog::consts::MITO_ENGINE;
+use datatypes::prelude::{ConcreteDataType, VectorRef};
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::vectors::StringVector;
+
+use crate::information_schema::table_names::*;
+
+const UNKNOWN: &str = "unknown";
+
+/// Find the schema and columns by the table_name, only valid for memory tables.
+/// Safety: the user MUST ensure the table schema exists, panic otherwise.
+pub fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec<VectorRef>) {
+    let (column_schemas, columns): (_, Vec<VectorRef>) = match table_name {
+        COLUMN_PRIVILEGES => (
+            string_columns(&[
+                "GRANTEE",
+                "TABLE_CATALOG",
+                "TABLE_SCHEMA",
+                "TABLE_NAME",
+                "COLUMN_NAME",
+                "PRIVILEGE_TYPE",
+                "IS_GRANTABLE",
+            ]),
+            vec![],
+        ),
+
+        COLUMN_STATISTICS => (
+            string_columns(&[
+                "SCHEMA_NAME",
+                "TABLE_NAME",
+                "COLUMN_NAME",
+                // TODO(dennis): It must be a JSON type, but we don't support it yet
+                "HISTOGRAM",
+            ]),
+            vec![],
+        ),
+
+        ENGINES => (
+            string_columns(&[
+                "ENGINE",
+                "SUPPORT",
+                "COMMENT",
+                "TRANSACTIONS",
+                "XA",
+                "SAVEPOINTS",
+            ]),
+            vec![
+                Arc::new(StringVector::from(vec![MITO_ENGINE])),
+                Arc::new(StringVector::from(vec!["DEFAULT"])),
+                Arc::new(StringVector::from(vec![
+                    "Storage engine for time-series data",
+                ])),
+                Arc::new(StringVector::from(vec!["NO"])),
+                Arc::new(StringVector::from(vec!["NO"])),
+                Arc::new(StringVector::from(vec!["NO"])),
+            ],
+        ),
+
+        BUILD_INFO => (
+            string_columns(&[
+                "GIT_BRANCH",
+                "GIT_COMMIT",
+                "GIT_COMMIT_SHORT",
+                "GIT_DIRTY",
+                "PKG_VERSION",
+            ]),
+            vec![
+                Arc::new(StringVector::from(vec![
+                    build_data::get_git_branch().unwrap_or_else(|_| UNKNOWN.to_string())
+                ])),
+                Arc::new(StringVector::from(vec![
+                    build_data::get_git_commit().unwrap_or_else(|_| UNKNOWN.to_string())
+                ])),
+                Arc::new(StringVector::from(vec![
+                    build_data::get_git_commit_short().unwrap_or_else(|_| UNKNOWN.to_string())
+                ])),
+                Arc::new(StringVector::from(vec![
+                    build_data::get_git_dirty().map_or(UNKNOWN.to_string(), |v| v.to_string())
+                ])),
+                Arc::new(StringVector::from(vec![option_env!("CARGO_PKG_VERSION")])),
+            ],
+        ),
+
+        _ => unreachable!("Unknown table in information_schema: {}", table_name),
+    };
+
+    (Arc::new(Schema::new(column_schemas)), columns)
+}
+
+fn string_columns(names: &[&'static str]) -> Vec<ColumnSchema> {
+    names.iter().map(|name| string_column(name)).collect()
+}
+
+fn string_column(name: &str) -> ColumnSchema {
+    ColumnSchema::new(
+        str::to_lowercase(name),
+        ConcreteDataType::string_datatype(),
+        false,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_string_columns() {
+        let columns = ["a", "b", "c"];
+        let column_schemas = string_columns(&columns);
+
+        assert_eq!(3, column_schemas.len());
+        for (i, name) in columns.iter().enumerate() {
+            let cs = column_schemas.get(i).unwrap();
+
+            assert_eq!(*name, cs.name);
+            assert_eq!(ConcreteDataType::string_datatype(), cs.data_type);
+        }
+    }
+}

src/catalog/src/information_schema/table_names.rs

@@ -0,0 +1,22 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// All table names in `information_schema`.
+
+pub const TABLES: &str = "tables";
+pub const COLUMNS: &str = "columns";
+pub const ENGINES: &str = "engines";
+pub const COLUMN_PRIVILEGES: &str = "column_privileges";
+pub const COLUMN_STATISTICS: &str = "column_statistics";
+pub const BUILD_INFO: &str = "build_info";

src/catalog/src/information_schema/tables.rs

@@ -15,10 +15,7 @@
 use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
-use common_catalog::consts::{
-    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME,
-    INFORMATION_SCHEMA_TABLES_TABLE_ID,
-};
+use common_catalog::consts::INFORMATION_SCHEMA_TABLES_TABLE_ID;
 use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
@@ -33,7 +30,7 @@ use snafu::{OptionExt, ResultExt};
 use store_api::storage::TableId;
 use table::metadata::TableType;
 
-use super::{COLUMNS, TABLES};
+use super::TABLES;
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
@@ -178,29 +175,8 @@ impl InformationSchemaTablesBuilder {
                         Some(&table_info.meta.engine),
                     );
                 } else {
-                    // TODO: this specific branch is only a workaround for FrontendCatalogManager.
-                    if schema_name == INFORMATION_SCHEMA_NAME {
-                        if table_name == COLUMNS {
-                            self.add_table(
-                                &catalog_name,
-                                &schema_name,
-                                &table_name,
-                                TableType::Temporary,
-                                Some(INFORMATION_SCHEMA_COLUMNS_TABLE_ID),
-                                None,
-                            );
-                        } else if table_name == TABLES {
-                            self.add_table(
-                                &catalog_name,
-                                &schema_name,
-                                &table_name,
-                                TableType::Temporary,
-                                Some(INFORMATION_SCHEMA_TABLES_TABLE_ID),
-                                None,
-                            );
-                        }
-                    }
-                };
+                    unreachable!();
+                }
             }
         }
 

src/catalog/src/kvbackend/manager.rs

@@ -19,7 +19,6 @@ use std::sync::{Arc, Weak};
 use common_catalog::consts::{DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, NUMBERS_TABLE_ID};
 use common_error::ext::BoxedError;
 use common_meta::cache_invalidator::{CacheInvalidator, CacheInvalidatorRef, Context};
-use common_meta::datanode_manager::DatanodeManagerRef;
 use common_meta::error::Result as MetaResult;
 use common_meta::key::catalog_name::CatalogNameKey;
 use common_meta::key::schema_name::SchemaNameKey;
@@ -39,7 +38,7 @@ use crate::error::{
     self as catalog_err, ListCatalogsSnafu, ListSchemasSnafu, Result as CatalogResult,
     TableMetadataManagerSnafu,
 };
-use crate::information_schema::{InformationSchemaProvider, COLUMNS, TABLES};
+use crate::information_schema::InformationSchemaProvider;
 use crate::CatalogManager;
 
 /// Access all existing catalog, schema and tables.
@@ -55,7 +54,6 @@ pub struct KvBackendCatalogManager {
     cache_invalidator: CacheInvalidatorRef,
     partition_manager: PartitionRuleManagerRef,
     table_metadata_manager: TableMetadataManagerRef,
-    datanode_manager: DatanodeManagerRef,
     /// A sub-CatalogManager that handles system tables
     system_catalog: SystemCatalog,
 }
@@ -76,18 +74,18 @@ impl CacheInvalidator for KvBackendCatalogManager {
 }
 
 impl KvBackendCatalogManager {
-    pub fn new(
-        backend: KvBackendRef,
-        cache_invalidator: CacheInvalidatorRef,
-        datanode_manager: DatanodeManagerRef,
-    ) -> Arc<Self> {
+    pub fn new(backend: KvBackendRef, cache_invalidator: CacheInvalidatorRef) -> Arc<Self> {
         Arc::new_cyclic(|me| Self {
             partition_manager: Arc::new(PartitionRuleManager::new(backend.clone())),
             table_metadata_manager: Arc::new(TableMetadataManager::new(backend)),
             cache_invalidator,
-            datanode_manager,
             system_catalog: SystemCatalog {
                 catalog_manager: me.clone(),
+                information_schema_provider: Arc::new(InformationSchemaProvider::new(
+                    // The catalog name is not used in system_catalog, so let it empty
+                    "".to_string(),
+                    me.clone(),
+                )),
             },
         })
     }
@@ -99,10 +97,6 @@ impl KvBackendCatalogManager {
     pub fn table_metadata_manager_ref(&self) -> &TableMetadataManagerRef {
         &self.table_metadata_manager
     }
-
-    pub fn datanode_manager(&self) -> DatanodeManagerRef {
-        self.datanode_manager.clone()
-    }
 }
 
 #[async_trait::async_trait]
@@ -133,13 +127,11 @@ impl CatalogManager for KvBackendCatalogManager {
             .try_collect::<BTreeSet<_>>()
             .await
             .map_err(BoxedError::new)
-            .context(ListSchemasSnafu { catalog })?
-            .into_iter()
-            .collect::<Vec<_>>();
+            .context(ListSchemasSnafu { catalog })?;
 
-        keys.extend_from_slice(&self.system_catalog.schema_names());
+        keys.extend(self.system_catalog.schema_names());
 
-        Ok(keys)
+        Ok(keys.into_iter().collect())
     }
 
     async fn table_names(&self, catalog: &str, schema: &str) -> CatalogResult<Vec<String>> {
@@ -242,11 +234,11 @@ impl CatalogManager for KvBackendCatalogManager {
 // a new catalog is created.
 /// Existing system tables:
 /// - public.numbers
-/// - information_schema.tables
-/// - information_schema.columns
+/// - information_schema.{tables}
 #[derive(Clone)]
 struct SystemCatalog {
     catalog_manager: Weak<KvBackendCatalogManager>,
+    information_schema_provider: Arc<InformationSchemaProvider>,
 }
 
 impl SystemCatalog {
@@ -256,7 +248,7 @@ impl SystemCatalog {
 
     fn table_names(&self, schema: &str) -> Vec<String> {
         if schema == INFORMATION_SCHEMA_NAME {
-            vec![TABLES.to_string(), COLUMNS.to_string()]
+            self.information_schema_provider.table_names()
         } else if schema == DEFAULT_SCHEMA_NAME {
             vec![NUMBERS_TABLE_NAME.to_string()]
         } else {
@@ -270,7 +262,7 @@ impl SystemCatalog {
 
     fn table_exist(&self, schema: &str, table: &str) -> bool {
         if schema == INFORMATION_SCHEMA_NAME {
-            table == TABLES || table == COLUMNS
+            self.information_schema_provider.table(table).is_some()
         } else if schema == DEFAULT_SCHEMA_NAME {
             table == NUMBERS_TABLE_NAME
         } else {

src/catalog/src/lib.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(trait_upcasting)]
 #![feature(assert_matches)]
 #![feature(try_blocks)]
 

src/catalog/src/memory/manager.rs

@@ -18,7 +18,9 @@ use std::collections::HashMap;
 use std::sync::{Arc, RwLock, Weak};
 
 use common_catalog::build_db_string;
-use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_PRIVATE_SCHEMA_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME,
+};
 use snafu::OptionExt;
 use table::TableRef;
 
@@ -135,6 +137,18 @@ impl MemoryCatalogManager {
                 schema: DEFAULT_SCHEMA_NAME.to_string(),
             })
             .unwrap();
+        manager
+            .register_schema_sync(RegisterSchemaRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_PRIVATE_SCHEMA_NAME.to_string(),
+            })
+            .unwrap();
+        manager
+            .register_schema_sync(RegisterSchemaRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: INFORMATION_SCHEMA_NAME.to_string(),
+            })
+            .unwrap();
 
         manager
     }
@@ -243,10 +257,12 @@ impl MemoryCatalogManager {
     }
 
     fn create_catalog_entry(self: &Arc<Self>, catalog: String) -> SchemaEntries {
-        let information_schema = InformationSchemaProvider::build(
+        let information_schema_provider = InformationSchemaProvider::new(
             catalog,
             Arc::downgrade(self) as Weak<dyn CatalogManager>,
         );
+        let information_schema = information_schema_provider.tables().clone();
+
         let mut catalog = HashMap::new();
         catalog.insert(INFORMATION_SCHEMA_NAME.to_string(), information_schema);
         catalog

src/client/Cargo.toml

@@ -35,7 +35,7 @@ prost.workspace = true
 rand.workspace = true
 session.workspace = true
 snafu.workspace = true
-tokio-stream = { version = "0.1", features = ["net"] }
+tokio-stream = { workspace = true, features = ["net"] }
 tokio.workspace = true
 tonic.workspace = true
 

src/client/src/database.rs

@@ -27,7 +27,7 @@ use common_error::ext::{BoxedError, ErrorExt};
 use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_query::Output;
 use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::RecordBatchStreamAdaptor;
+use common_recordbatch::RecordBatchStreamWrapper;
 use common_telemetry::logging;
 use common_telemetry::tracing_context::W3cTrace;
 use futures_util::StreamExt;
@@ -315,7 +315,7 @@ impl Database {
                         yield Ok(record_batch);
                     }
                 }));
-                let record_batch_stream = RecordBatchStreamAdaptor {
+                let record_batch_stream = RecordBatchStreamWrapper {
                     schema,
                     stream,
                     output_ordering: None,

src/client/src/error.rs

@@ -131,3 +131,15 @@ impl From<Status> for Error {
         Self::Server { code, msg }
     }
 }
+
+impl Error {
+    pub fn should_retry(&self) -> bool {
+        !matches!(
+            self,
+            Self::RegionServer {
+                code: Code::InvalidArgument,
+                ..
+            }
+        )
+    }
+}

src/client/src/region.rs

@@ -23,13 +23,12 @@ use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_meta::datanode_manager::{AffectedRows, Datanode};
 use common_meta::error::{self as meta_error, Result as MetaResult};
 use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
+use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
 use common_telemetry::error;
 use prost::Message;
 use snafu::{location, Location, OptionExt, ResultExt};
 use tokio_stream::StreamExt;
 
-use crate::error::Error::RegionServer;
 use crate::error::{
     self, ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
     MissingFieldSnafu, Result, ServerSnafu,
@@ -45,7 +44,7 @@ pub struct RegionRequester {
 impl Datanode for RegionRequester {
     async fn handle(&self, request: RegionRequest) -> MetaResult<AffectedRows> {
         self.handle_inner(request).await.map_err(|err| {
-            if matches!(err, RegionServer { .. }) {
+            if err.should_retry() {
                 meta_error::Error::RetryLater {
                     source: BoxedError::new(err),
                 }
@@ -137,7 +136,7 @@ impl RegionRequester {
                 yield Ok(record_batch);
             }
         }));
-        let record_batch_stream = RecordBatchStreamAdaptor {
+        let record_batch_stream = RecordBatchStreamWrapper {
             schema,
             stream,
             output_ordering: None,

src/cmd/Cargo.toml

@@ -32,6 +32,7 @@ common-recordbatch.workspace = true
 common-telemetry = { workspace = true, features = [
     "deadlock_detection",
 ] }
+common-time.workspace = true
 config = "0.13"
 datanode.workspace = true
 datatypes.workspace = true
@@ -40,6 +41,7 @@ etcd-client.workspace = true
 file-engine.workspace = true
 frontend.workspace = true
 futures.workspace = true
+human-panic = "1.2.2"
 lazy_static.workspace = true
 meta-client.workspace = true
 meta-srv.workspace = true
@@ -58,6 +60,7 @@ serde_json.workspace = true
 servers.workspace = true
 session.workspace = true
 snafu.workspace = true
+store-api.workspace = true
 substrait.workspace = true
 table.workspace = true
 tokio.workspace = true

src/cmd/src/bin/greptime.rs

@@ -16,79 +16,12 @@
 
 use std::fmt;
 
-use clap::Parser;
+use clap::{FromArgMatches, Parser, Subcommand};
 use cmd::error::Result;
-use cmd::options::{Options, TopLevelOptions};
-use cmd::{cli, datanode, frontend, metasrv, standalone};
-use common_telemetry::logging::{error, info, TracingOptions};
-
-lazy_static::lazy_static! {
-    static ref APP_VERSION: prometheus::IntGaugeVec =
-        prometheus::register_int_gauge_vec!("app_version", "app version", &["short_version", "version"]).unwrap();
-}
-
-#[derive(Parser)]
-#[clap(name = "greptimedb", version = print_version())]
-struct Command {
-    #[clap(long)]
-    log_dir: Option<String>,
-    #[clap(long)]
-    log_level: Option<String>,
-    #[clap(subcommand)]
-    subcmd: SubCommand,
-
-    #[cfg(feature = "tokio-console")]
-    #[clap(long)]
-    tokio_console_addr: Option<String>,
-}
-
-pub enum Application {
-    Datanode(datanode::Instance),
-    Frontend(frontend::Instance),
-    Metasrv(metasrv::Instance),
-    Standalone(standalone::Instance),
-    Cli(cli::Instance),
-}
-
-impl Application {
-    async fn start(&mut self) -> Result<()> {
-        match self {
-            Application::Datanode(instance) => instance.start().await,
-            Application::Frontend(instance) => instance.start().await,
-            Application::Metasrv(instance) => instance.start().await,
-            Application::Standalone(instance) => instance.start().await,
-            Application::Cli(instance) => instance.start().await,
-        }
-    }
-
-    async fn stop(&self) -> Result<()> {
-        match self {
-            Application::Datanode(instance) => instance.stop().await,
-            Application::Frontend(instance) => instance.stop().await,
-            Application::Metasrv(instance) => instance.stop().await,
-            Application::Standalone(instance) => instance.stop().await,
-            Application::Cli(instance) => instance.stop().await,
-        }
-    }
-}
-
-impl Command {
-    async fn build(self, opts: Options) -> Result<Application> {
-        self.subcmd.build(opts).await
-    }
-
-    fn load_options(&self) -> Result<Options> {
-        let top_level_opts = self.top_level_options();
-        self.subcmd.load_options(top_level_opts)
-    }
-
-    fn top_level_options(&self) -> TopLevelOptions {
-        TopLevelOptions {
-            log_dir: self.log_dir.clone(),
-            log_level: self.log_level.clone(),
-        }
-    }
-}
+use cmd::options::{CliOptions, Options};
+use cmd::{
+    cli, datanode, frontend, greptimedb_cli, log_versions, metasrv, standalone, start_app, App,
+};
 
 #[derive(Parser)]
 enum SubCommand {
@@ -105,40 +38,41 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, opts: Options) -> Result<Application> {
-        match (self, opts) {
+    async fn build(self, opts: Options) -> Result<Box<dyn App>> {
+        let app: Box<dyn App> = match (self, opts) {
             (SubCommand::Datanode(cmd), Options::Datanode(dn_opts)) => {
                 let app = cmd.build(*dn_opts).await?;
-                Ok(Application::Datanode(app))
+                Box::new(app) as _
             }
             (SubCommand::Frontend(cmd), Options::Frontend(fe_opts)) => {
                 let app = cmd.build(*fe_opts).await?;
-                Ok(Application::Frontend(app))
+                Box::new(app) as _
             }
             (SubCommand::Metasrv(cmd), Options::Metasrv(meta_opts)) => {
                 let app = cmd.build(*meta_opts).await?;
-                Ok(Application::Metasrv(app))
+                Box::new(app) as _
             }
             (SubCommand::Standalone(cmd), Options::Standalone(opts)) => {
                 let app = cmd.build(*opts).await?;
-                Ok(Application::Standalone(app))
+                Box::new(app) as _
             }
             (SubCommand::Cli(cmd), Options::Cli(_)) => {
                 let app = cmd.build().await?;
-                Ok(Application::Cli(app))
+                Box::new(app) as _
             }
 
             _ => unreachable!(),
-        }
+        };
+        Ok(app)
     }
 
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Datanode(cmd) => cmd.load_options(top_level_opts),
-            SubCommand::Frontend(cmd) => cmd.load_options(top_level_opts),
-            SubCommand::Metasrv(cmd) => cmd.load_options(top_level_opts),
-            SubCommand::Standalone(cmd) => cmd.load_options(top_level_opts),
-            SubCommand::Cli(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Datanode(cmd) => cmd.load_options(cli_options),
+            SubCommand::Frontend(cmd) => cmd.load_options(cli_options),
+            SubCommand::Metasrv(cmd) => cmd.load_options(cli_options),
+            SubCommand::Standalone(cmd) => cmd.load_options(cli_options),
+            SubCommand::Cli(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -155,90 +89,49 @@ impl fmt::Display for SubCommand {
     }
 }
 
-fn print_version() -> &'static str {
-    concat!(
-        "\nbranch: ",
-        env!("GIT_BRANCH"),
-        "\ncommit: ",
-        env!("GIT_COMMIT"),
-        "\ndirty: ",
-        env!("GIT_DIRTY"),
-        "\nversion: ",
-        env!("CARGO_PKG_VERSION")
-    )
-}
-
-fn short_version() -> &'static str {
-    env!("CARGO_PKG_VERSION")
-}
-
-// {app_name}-{branch_name}-{commit_short}
-// The branch name (tag) of a release build should already contain the short
-// version so the full version doesn't concat the short version explicitly.
-fn full_version() -> &'static str {
-    concat!(
-        "greptimedb-",
-        env!("GIT_BRANCH"),
-        "-",
-        env!("GIT_COMMIT_SHORT")
-    )
-}
-
-fn log_env_flags() {
-    info!("command line arguments");
-    for argument in std::env::args() {
-        info!("argument: {}", argument);
-    }
-}
-
 #[cfg(not(windows))]
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    let cmd = Command::parse();
-    let app_name = &cmd.subcmd.to_string();
-
-    let opts = cmd.load_options()?;
-    let logging_opts = opts.logging_options();
-    let tracing_opts = TracingOptions {
-        #[cfg(feature = "tokio-console")]
-        tokio_console_addr: cmd.tokio_console_addr.clone(),
+    let metadata = human_panic::Metadata {
+        version: env!("CARGO_PKG_VERSION").into(),
+        name: "GreptimeDB".into(),
+        authors: Default::default(),
+        homepage: "https://github.com/GreptimeTeam/greptimedb/discussions".into(),
     };
+    human_panic::setup_panic!(metadata);
 
     common_telemetry::set_panic_hook();
-    let _guard =
-        common_telemetry::init_global_logging(app_name, logging_opts, tracing_opts, opts.node_id());
 
-    // Report app version as gauge.
-    APP_VERSION
-        .with_label_values(&[short_version(), full_version()])
-        .inc();
+    let cli = greptimedb_cli();
 
-    // Log version and argument flags.
-    info!(
-        "short_version: {}, full_version: {}",
-        short_version(),
-        full_version()
+    let cli = SubCommand::augment_subcommands(cli);
+
+    let args = cli.get_matches();
+
+    let subcmd = match SubCommand::from_arg_matches(&args) {
+        Ok(subcmd) => subcmd,
+        Err(e) => e.exit(),
+    };
+
+    let app_name = subcmd.to_string();
+
+    let cli_options = CliOptions::new(&args);
+
+    let opts = subcmd.load_options(&cli_options)?;
+
+    let _guard = common_telemetry::init_global_logging(
+        &app_name,
+        opts.logging_options(),
+        cli_options.tracing_options(),
+        opts.node_id(),
     );
-    log_env_flags();
 
-    let mut app = cmd.build(opts).await?;
+    log_versions();
 
-    tokio::select! {
-        result = app.start() => {
-            if let Err(err) = result {
-                error!(err; "Fatal error occurs!");
-            }
-        }
-        _ = tokio::signal::ctrl_c() => {
-            if let Err(err) = app.stop().await {
-                error!(err; "Fatal error occurs!");
-            }
-            info!("Goodbye!");
-        }
-    }
+    let app = subcmd.build(opts).await?;
 
-    Ok(())
+    start_app(app).await
 }

src/cmd/src/cli.rs

@@ -13,9 +13,15 @@
 // limitations under the License.
 
 mod bench;
+
+// Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373
+#[allow(unused)]
 mod cmd;
 mod export;
 mod helper;
+
+// Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373
+#[allow(unused)]
 mod repl;
 // TODO(weny): Removes it
 #[allow(deprecated)]
@@ -30,27 +36,35 @@ use upgrade::UpgradeCommand;
 
 use self::export::ExportCommand;
 use crate::error::Result;
-use crate::options::{Options, TopLevelOptions};
+use crate::options::{CliOptions, Options};
+use crate::App;
 
 #[async_trait]
-pub trait Tool {
+pub trait Tool: Send + Sync {
     async fn do_work(&self) -> Result<()>;
 }
 
-pub enum Instance {
-    Repl(Repl),
-    Tool(Box<dyn Tool>),
+pub struct Instance {
+    tool: Box<dyn Tool>,
 }
 
 impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
-        match self {
-            Instance::Repl(repl) => repl.run().await,
-            Instance::Tool(tool) => tool.do_work().await,
-        }
+    fn new(tool: Box<dyn Tool>) -> Self {
+        Self { tool }
+    }
+}
+
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-cli"
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn start(&mut self) -> Result<()> {
+        self.tool.do_work().await
+    }
+
+    async fn stop(&self) -> Result<()> {
         Ok(())
     }
 }
@@ -66,14 +80,15 @@ impl Command {
         self.cmd.build().await
     }
 
-    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         let mut logging_opts = LoggingOptions::default();
-        if let Some(dir) = top_level_opts.log_dir {
-            logging_opts.dir = dir;
-        }
-        if top_level_opts.log_level.is_some() {
-            logging_opts.level = top_level_opts.log_level;
+
+        if let Some(dir) = &cli_options.log_dir {
+            logging_opts.dir = dir.clone();
         }
+
+        logging_opts.level = cli_options.log_level.clone();
+
         Ok(Options::Cli(Box::new(logging_opts)))
     }
 }
@@ -110,7 +125,6 @@ pub(crate) struct AttachCommand {
 impl AttachCommand {
     #[allow(dead_code)]
     async fn build(self) -> Result<Instance> {
-        let repl = Repl::try_new(&self).await?;
-        Ok(Instance::Repl(repl))
+        unimplemented!("Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373")
     }
 }

src/cmd/src/cli/bench.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 use std::future::Future;
 use std::sync::Arc;
 use std::time::Duration;
@@ -28,6 +28,7 @@ use common_telemetry::info;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use rand::Rng;
+use store_api::storage::RegionNumber;
 use table::metadata::{RawTableInfo, RawTableMeta, TableId, TableIdent, TableType};
 
 use self::metadata::TableMetadataBencher;
@@ -69,7 +70,7 @@ impl BenchTableMetadataCommand {
             table_metadata_manager,
             count: self.count,
         };
-        Ok(Instance::Tool(Box::new(tool)))
+        Ok(Instance::new(Box::new(tool)))
     }
 }
 
@@ -137,12 +138,12 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
     }
 }
 
-fn create_region_routes() -> Vec<RegionRoute> {
-    let mut regions = Vec::with_capacity(100);
+fn create_region_routes(regions: Vec<RegionNumber>) -> Vec<RegionRoute> {
+    let mut region_routes = Vec::with_capacity(100);
     let mut rng = rand::thread_rng();
 
-    for region_id in 0..64u64 {
-        regions.push(RegionRoute {
+    for region_id in regions.into_iter().map(u64::from) {
+        region_routes.push(RegionRoute {
             region: Region {
                 id: region_id.into(),
                 name: String::new(),
@@ -158,5 +159,11 @@ fn create_region_routes() -> Vec<RegionRoute> {
         });
     }
 
-    regions
+    region_routes
+}
+
+fn create_region_wal_options(regions: Vec<RegionNumber>) -> HashMap<RegionNumber, String> {
+    // TODO(niebayes): construct region wal options for benchmark.
+    let _ = regions;
+    HashMap::default()
 }

src/cmd/src/cli/bench/metadata.rs

@@ -14,10 +14,13 @@
 
 use std::time::Instant;
 
+use common_meta::key::table_route::TableRouteValue;
 use common_meta::key::TableMetadataManagerRef;
 use common_meta::table_name::TableName;
 
-use super::{bench_self_recorded, create_region_routes, create_table_info};
+use crate::cli::bench::{
+    bench_self_recorded, create_region_routes, create_region_wal_options, create_table_info,
+};
 
 pub struct TableMetadataBencher {
     table_metadata_manager: TableMetadataManagerRef,
@@ -43,12 +46,19 @@ impl TableMetadataBencher {
                 let table_name = format!("bench_table_name_{}", i);
                 let table_name = TableName::new("bench_catalog", "bench_schema", table_name);
                 let table_info = create_table_info(i, table_name);
-                let region_routes = create_region_routes();
+
+                let regions: Vec<_> = (0..64).collect();
+                let region_routes = create_region_routes(regions.clone());
+                let region_wal_options = create_region_wal_options(regions);
 
                 let start = Instant::now();
 
                 self.table_metadata_manager
-                    .create_table_metadata(table_info, region_routes)
+                    .create_table_metadata(
+                        table_info,
+                        TableRouteValue::physical(region_routes),
+                        region_wal_options,
+                    )
                     .await
                     .unwrap();
 

src/cmd/src/cli/export.rs

@@ -105,7 +105,7 @@ impl ExportCommand {
             }));
         }
 
-        Ok(Instance::Tool(Box::new(Export {
+        Ok(Instance::new(Box::new(Export {
             client: database_client,
             catalog,
             schema,

src/cmd/src/cli/repl.rs

@@ -17,7 +17,6 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use catalog::kvbackend::{CachedMetaKvBackend, KvBackendCatalogManager};
-use client::client_manager::DatanodeClients;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_base::Plugins;
 use common_error::ext::ErrorExt;
@@ -250,13 +249,8 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
 
     let cached_meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
 
-    let datanode_clients = Arc::new(DatanodeClients::default());
-
-    let catalog_list = KvBackendCatalogManager::new(
-        cached_meta_backend.clone(),
-        cached_meta_backend.clone(),
-        datanode_clients,
-    );
+    let catalog_list =
+        KvBackendCatalogManager::new(cached_meta_backend.clone(), cached_meta_backend);
     let plugins: Plugins = Default::default();
     let state = Arc::new(QueryEngineState::new(
         catalog_list,

src/cmd/src/cli/upgrade.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use async_trait::async_trait;
@@ -26,7 +27,7 @@ use common_meta::key::table_info::{TableInfoKey, TableInfoValue};
 use common_meta::key::table_name::{TableNameKey, TableNameValue};
 use common_meta::key::table_region::{TableRegionKey, TableRegionValue};
 use common_meta::key::table_route::{TableRouteKey, TableRouteValue as NextTableRouteValue};
-use common_meta::key::{RegionDistribution, TableMetaKey};
+use common_meta::key::{RegionDistribution, TableMetaKey, TableMetaValue};
 use common_meta::kv_backend::etcd::EtcdStore;
 use common_meta::kv_backend::KvBackendRef;
 use common_meta::range_stream::PaginationStream;
@@ -76,7 +77,7 @@ impl UpgradeCommand {
             skip_schema_keys: self.skip_schema_keys,
             skip_table_route_keys: self.skip_table_route_keys,
         };
-        Ok(Instance::Tool(Box::new(tool)))
+        Ok(Instance::new(Box::new(tool)))
     }
 }
 
@@ -152,7 +153,7 @@ impl MigrateTableMetadata {
         )
         .unwrap();
 
-        let new_table_value = NextTableRouteValue::new(table_route.region_routes);
+        let new_table_value = NextTableRouteValue::physical(table_route.region_routes);
 
         let table_id = table_route.table.id as u32;
         let new_key = TableRouteKey::new(table_id);
@@ -395,6 +396,9 @@ impl MigrateTableMetadata {
         let region_distribution: RegionDistribution =
             value.regions_id_map.clone().into_iter().collect();
 
+        // TODO(niebayes): properly fetch or construct wal options.
+        let region_wal_options = HashMap::default();
+
         let datanode_table_kvs = region_distribution
             .into_iter()
             .map(|(datanode_id, regions)| {
@@ -409,6 +413,7 @@ impl MigrateTableMetadata {
                             engine: engine.to_string(),
                             region_storage_path: region_storage_path.clone(),
                             region_options: (&value.table_info.meta.options).into(),
+                            region_wal_options: region_wal_options.clone(),
                         },
                     ),
                 )

src/cmd/src/datanode.rs

@@ -15,9 +15,11 @@
 use std::sync::Arc;
 use std::time::Duration;
 
+use async_trait::async_trait;
 use catalog::kvbackend::MetaKvBackend;
 use clap::Parser;
-use common_telemetry::logging;
+use common_config::WalConfig;
+use common_telemetry::{info, logging};
 use datanode::config::DatanodeOptions;
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use meta_client::MetaClientOptions;
@@ -25,14 +27,26 @@ use servers::Mode;
 use snafu::{OptionExt, ResultExt};
 
 use crate::error::{MissingConfigSnafu, Result, ShutdownDatanodeSnafu, StartDatanodeSnafu};
-use crate::options::{Options, TopLevelOptions};
+use crate::options::{CliOptions, Options};
+use crate::App;
 
 pub struct Instance {
     datanode: Datanode,
 }
 
 impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
+    fn new(datanode: Datanode) -> Self {
+        Self { datanode }
+    }
+}
+
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-datanode"
+    }
+
+    async fn start(&mut self) -> Result<()> {
         plugins::start_datanode_plugins(self.datanode.plugins())
             .await
             .context(StartDatanodeSnafu)?;
@@ -40,7 +54,7 @@ impl Instance {
         self.datanode.start().await.context(StartDatanodeSnafu)
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn stop(&self) -> Result<()> {
         self.datanode
             .shutdown()
             .await
@@ -59,8 +73,8 @@ impl Command {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
-        self.subcmd.load_options(top_level_opts)
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        self.subcmd.load_options(cli_options)
     }
 }
 
@@ -76,9 +90,9 @@ impl SubCommand {
         }
     }
 
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Start(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -108,19 +122,19 @@ struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         let mut opts: DatanodeOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
             DatanodeOptions::env_list_keys(),
         )?;
 
-        if let Some(dir) = top_level_opts.log_dir {
-            opts.logging.dir = dir;
+        if let Some(dir) = &cli_options.log_dir {
+            opts.logging.dir = dir.clone();
         }
 
-        if top_level_opts.log_level.is_some() {
-            opts.logging.level = top_level_opts.log_level;
+        if cli_options.log_level.is_some() {
+            opts.logging.level = cli_options.log_level.clone();
         }
 
         if let Some(addr) = &self.rpc_addr {
@@ -153,8 +167,18 @@ impl StartCommand {
             opts.storage.data_home = data_home.clone();
         }
 
-        if let Some(wal_dir) = &self.wal_dir {
-            opts.wal.dir = Some(wal_dir.clone());
+        // `wal_dir` only affects raft-engine config.
+        if let Some(wal_dir) = &self.wal_dir
+            && let WalConfig::RaftEngine(raft_engine_config) = &mut opts.wal
+        {
+            if raft_engine_config
+                .dir
+                .as_ref()
+                .is_some_and(|original_dir| original_dir != wal_dir)
+            {
+                info!("The wal dir of raft-engine is altered to {wal_dir}");
+            }
+            raft_engine_config.dir.replace(wal_dir.clone());
         }
 
         if let Some(http_addr) = &self.http_addr {
@@ -204,7 +228,7 @@ impl StartCommand {
             .await
             .context(StartDatanodeSnafu)?;
 
-        Ok(Instance { datanode })
+        Ok(Instance::new(datanode))
     }
 }
 
@@ -214,12 +238,12 @@ mod tests {
     use std::time::Duration;
 
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::config::{FileConfig, ObjectStoreConfig};
+    use datanode::config::{FileConfig, GcsConfig, ObjectStoreConfig, S3Config};
     use servers::heartbeat_options::HeartbeatOptions;
     use servers::Mode;
 
     use super::*;
-    use crate::options::ENV_VAR_SEP;
+    use crate::options::{CliOptions, ENV_VAR_SEP};
 
     #[test]
     fn test_read_from_config_file() {
@@ -243,6 +267,7 @@ mod tests {
             tcp_nodelay = true
 
             [wal]
+            provider = "raft_engine"
             dir = "/other/wal"
             file_size = "1GB"
             purge_threshold = "50GB"
@@ -251,8 +276,17 @@ mod tests {
             sync_write = false
 
             [storage]
-            type = "File"
             data_home = "/tmp/greptimedb/"
+            type = "File"
+
+            [[storage.providers]]
+            type = "Gcs"
+            bucket = "foo"
+            endpoint = "bar"
+
+            [[storage.providers]]
+            type = "S3"
+            bucket = "foo"
 
             [logging]
             level = "debug"
@@ -265,19 +299,24 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Datanode(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Datanode(options) = cmd.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
 
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
-        assert_eq!("/other/wal", options.wal.dir.unwrap());
 
-        assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
-        assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
-        assert_eq!(1024 * 1024 * 1024 * 50, options.wal.purge_threshold.0);
-        assert!(!options.wal.sync_write);
+        let WalConfig::RaftEngine(raft_engine_config) = options.wal else {
+            unreachable!()
+        };
+        assert_eq!("/other/wal", raft_engine_config.dir.unwrap());
+        assert_eq!(Duration::from_secs(600), raft_engine_config.purge_interval);
+        assert_eq!(1024 * 1024 * 1024, raft_engine_config.file_size.0);
+        assert_eq!(
+            1024 * 1024 * 1024 * 50,
+            raft_engine_config.purge_threshold.0
+        );
+        assert!(!raft_engine_config.sync_write);
 
         let HeartbeatOptions {
             interval: heart_beat_interval,
@@ -305,6 +344,15 @@ mod tests {
             &options.storage.store,
             ObjectStoreConfig::File(FileConfig { .. })
         ));
+        assert_eq!(options.storage.providers.len(), 2);
+        assert!(matches!(
+            options.storage.providers[0],
+            ObjectStoreConfig::Gcs(GcsConfig { .. })
+        ));
+        assert!(matches!(
+            options.storage.providers[1],
+            ObjectStoreConfig::S3(S3Config { .. })
+        ));
 
         assert_eq!("debug", options.logging.level.unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), options.logging.dir);
@@ -313,7 +361,7 @@ mod tests {
     #[test]
     fn test_try_from_cmd() {
         if let Options::Datanode(opt) = StartCommand::default()
-            .load_options(TopLevelOptions::default())
+            .load_options(&CliOptions::default())
             .unwrap()
         {
             assert_eq!(Mode::Standalone, opt.mode)
@@ -324,7 +372,7 @@ mod tests {
             metasrv_addr: Some(vec!["127.0.0.1:3002".to_string()]),
             ..Default::default()
         })
-        .load_options(TopLevelOptions::default())
+        .load_options(&CliOptions::default())
         .unwrap()
         {
             assert_eq!(Mode::Distributed, opt.mode)
@@ -334,7 +382,7 @@ mod tests {
             metasrv_addr: Some(vec!["127.0.0.1:3002".to_string()]),
             ..Default::default()
         })
-        .load_options(TopLevelOptions::default())
+        .load_options(&CliOptions::default())
         .is_err());
 
         // Providing node_id but leave metasrv_addr absent is ok since metasrv_addr has default value
@@ -342,18 +390,21 @@ mod tests {
             node_id: Some(42),
             ..Default::default()
         })
-        .load_options(TopLevelOptions::default())
+        .load_options(&CliOptions::default())
         .is_ok());
     }
 
     #[test]
-    fn test_top_level_options() {
+    fn test_load_log_options_from_cli() {
         let cmd = StartCommand::default();
 
         let options = cmd
-            .load_options(TopLevelOptions {
+            .load_options(&CliOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
+
+                #[cfg(feature = "tokio-console")]
+                tokio_console_addr: None,
             })
             .unwrap();
 
@@ -379,9 +430,10 @@ mod tests {
             tcp_nodelay = true
 
             [wal]
+            provider = "raft_engine"
             file_size = "1GB"
             purge_threshold = "50GB"
-            purge_interval = "10m"
+            purge_interval = "5m"
             sync_write = false
 
             [storage]
@@ -436,14 +488,16 @@ mod tests {
                     ..Default::default()
                 };
 
-                let Options::Datanode(opts) =
-                    command.load_options(TopLevelOptions::default()).unwrap()
+                let Options::Datanode(opts) = command.load_options(&CliOptions::default()).unwrap()
                 else {
                     unreachable!()
                 };
 
                 // Should be read from env, env > default values.
-                assert_eq!(opts.wal.read_batch_size, 100,);
+                let WalConfig::RaftEngine(raft_engine_config) = opts.wal else {
+                    unreachable!()
+                };
+                assert_eq!(raft_engine_config.read_batch_size, 100);
                 assert_eq!(
                     opts.meta_client.unwrap().metasrv_addrs,
                     vec![
@@ -454,10 +508,13 @@ mod tests {
                 );
 
                 // Should be read from config file, config file > env > default values.
-                assert_eq!(opts.wal.purge_interval, Duration::from_secs(60 * 10));
+                assert_eq!(
+                    raft_engine_config.purge_interval,
+                    Duration::from_secs(60 * 5)
+                );
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.wal.dir.unwrap(), "/other/wal/dir");
+                assert_eq!(raft_engine_config.dir.unwrap(), "/other/wal/dir");
 
                 // Should be default value.
                 assert_eq!(opts.http.addr, DatanodeOptions::default().http.addr);

src/cmd/src/error.rs

@@ -14,7 +14,7 @@
 
 use std::any::Any;
 
-use common_error::ext::ErrorExt;
+use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
 use config::ConfigError;
@@ -37,6 +37,18 @@ pub enum Error {
         source: common_meta::error::Error,
     },
 
+    #[snafu(display("Failed to init DDL manager"))]
+    InitDdlManager {
+        location: Location,
+        source: common_meta::error::Error,
+    },
+
+    #[snafu(display("Failed to init default timezone"))]
+    InitTimezone {
+        location: Location,
+        source: common_time::error::Error,
+    },
+
     #[snafu(display("Failed to start procedure manager"))]
     StartProcedureManager {
         location: Location,
@@ -49,6 +61,12 @@ pub enum Error {
         source: common_procedure::error::Error,
     },
 
+    #[snafu(display("Failed to start wal options allocator"))]
+    StartWalOptionsAllocator {
+        location: Location,
+        source: common_meta::error::Error,
+    },
+
     #[snafu(display("Failed to start datanode"))]
     StartDatanode {
         location: Location,
@@ -226,11 +244,10 @@ pub enum Error {
         error: std::io::Error,
     },
 
-    #[snafu(display("Failed to parse address {}", addr))]
-    ParseAddr {
-        addr: String,
-        #[snafu(source)]
-        error: std::net::AddrParseError,
+    #[snafu(display("Other error"))]
+    Other {
+        source: BoxedError,
+        location: Location,
     },
 }
 
@@ -247,23 +264,26 @@ impl ErrorExt for Error {
             Error::ShutdownMetaServer { source, .. } => source.status_code(),
             Error::BuildMetaServer { source, .. } => source.status_code(),
             Error::UnsupportedSelectorType { source, .. } => source.status_code(),
-            Error::IterStream { source, .. } | Error::InitMetadata { source, .. } => {
-                source.status_code()
-            }
+
+            Error::IterStream { source, .. }
+            | Error::InitMetadata { source, .. }
+            | Error::InitDdlManager { source, .. } => source.status_code(),
+
             Error::ConnectServer { source, .. } => source.status_code(),
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
             | Error::InvalidReplCommand { .. }
+            | Error::InitTimezone { .. }
             | Error::ConnectEtcd { .. }
             | Error::NotDataFromOutput { .. }
             | Error::CreateDir { .. }
             | Error::EmptyResult { .. }
-            | Error::InvalidDatabaseName { .. }
-            | Error::ParseAddr { .. } => StatusCode::InvalidArguments,
+            | Error::InvalidDatabaseName { .. } => StatusCode::InvalidArguments,
 
             Error::StartProcedureManager { source, .. }
             | Error::StopProcedureManager { source, .. } => source.status_code(),
+            Error::StartWalOptionsAllocator { source, .. } => source.status_code(),
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
             Error::RequestDatabase { source, .. } => source.status_code(),
             Error::CollectRecordBatches { source, .. }
@@ -276,6 +296,8 @@ impl ErrorExt for Error {
             Error::StartCatalogManager { source, .. } => source.status_code(),
 
             Error::SerdeJson { .. } | Error::FileIo { .. } => StatusCode::Unexpected,
+
+            Error::Other { source, .. } => source.status_code(),
         }
     }
 

src/cmd/src/frontend.rs

@@ -12,26 +12,48 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
 use std::time::Duration;
 
+use async_trait::async_trait;
+use catalog::kvbackend::CachedMetaKvBackend;
 use clap::Parser;
+use client::client_manager::DatanodeClients;
+use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
+use common_meta::heartbeat::handler::HandlerGroupExecutor;
 use common_telemetry::logging;
+use common_time::timezone::set_default_timezone;
 use frontend::frontend::FrontendOptions;
+use frontend::heartbeat::handler::invalidate_table_cache::InvalidateTableCacheHandler;
+use frontend::heartbeat::HeartbeatTask;
+use frontend::instance::builder::FrontendBuilder;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use meta_client::MetaClientOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 
-use crate::error::{self, Result, StartFrontendSnafu};
-use crate::options::{Options, TopLevelOptions};
+use crate::error::{self, InitTimezoneSnafu, MissingConfigSnafu, Result, StartFrontendSnafu};
+use crate::options::{CliOptions, Options};
+use crate::App;
 
 pub struct Instance {
     frontend: FeInstance,
 }
 
 impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
+    fn new(frontend: FeInstance) -> Self {
+        Self { frontend }
+    }
+}
+
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-frontend"
+    }
+
+    async fn start(&mut self) -> Result<()> {
         plugins::start_frontend_plugins(self.frontend.plugins().clone())
             .await
             .context(StartFrontendSnafu)?;
@@ -39,7 +61,7 @@ impl Instance {
         self.frontend.start().await.context(StartFrontendSnafu)
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn stop(&self) -> Result<()> {
         self.frontend
             .shutdown()
             .await
@@ -58,8 +80,8 @@ impl Command {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
-        self.subcmd.load_options(top_level_opts)
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        self.subcmd.load_options(cli_options)
     }
 }
 
@@ -75,9 +97,9 @@ impl SubCommand {
         }
     }
 
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Start(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -117,19 +139,19 @@ pub struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         let mut opts: FrontendOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
             FrontendOptions::env_list_keys(),
         )?;
 
-        if let Some(dir) = top_level_opts.log_dir {
-            opts.logging.dir = dir;
+        if let Some(dir) = &cli_options.log_dir {
+            opts.logging.dir = dir.clone();
         }
 
-        if top_level_opts.log_level.is_some() {
-            opts.logging.level = top_level_opts.log_level;
+        if cli_options.log_level.is_some() {
+            opts.logging.level = cli_options.log_level.clone();
         }
 
         let tls_opts = TlsOption::new(
@@ -196,16 +218,50 @@ impl StartCommand {
         logging::info!("Frontend start command: {:#?}", self);
         logging::info!("Frontend options: {:#?}", opts);
 
-        let mut instance = FeInstance::try_new_distributed(&opts, plugins.clone())
+        set_default_timezone(opts.default_timezone.as_deref()).context(InitTimezoneSnafu)?;
+
+        let meta_client_options = opts.meta_client.as_ref().context(MissingConfigSnafu {
+            msg: "'meta_client'",
+        })?;
+        let meta_client = FeInstance::create_meta_client(meta_client_options)
             .await
             .context(StartFrontendSnafu)?;
 
+        let meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
+
+        let executor = HandlerGroupExecutor::new(vec![
+            Arc::new(ParseMailboxMessageHandler),
+            Arc::new(InvalidateTableCacheHandler::new(meta_backend.clone())),
+        ]);
+
+        let heartbeat_task = HeartbeatTask::new(
+            meta_client.clone(),
+            opts.heartbeat.clone(),
+            Arc::new(executor),
+        );
+
+        let mut instance = FrontendBuilder::new(
+            meta_backend.clone(),
+            Arc::new(DatanodeClients::default()),
+            meta_client,
+        )
+        .with_cache_invalidator(meta_backend)
+        .with_plugin(plugins)
+        .with_heartbeat_task(heartbeat_task)
+        .try_build()
+        .await
+        .context(StartFrontendSnafu)?;
+
+        instance
+            .build_export_metrics_task(&opts.export_metrics)
+            .context(StartFrontendSnafu)?;
+
         instance
             .build_servers(opts)
             .await
             .context(StartFrontendSnafu)?;
 
-        Ok(Instance { frontend: instance })
+        Ok(Instance::new(instance))
     }
 }
 
@@ -221,7 +277,7 @@ mod tests {
     use servers::http::HttpOptions;
 
     use super::*;
-    use crate::options::ENV_VAR_SEP;
+    use crate::options::{CliOptions, ENV_VAR_SEP};
 
     #[test]
     fn test_try_from_start_command() {
@@ -235,8 +291,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Frontend(opts) = command.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Frontend(opts) = command.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
 
@@ -288,7 +343,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Frontend(fe_opts) = command.load_options(TopLevelOptions::default()).unwrap()
+        let Options::Frontend(fe_opts) = command.load_options(&CliOptions::default()).unwrap()
         else {
             unreachable!()
         };
@@ -327,16 +382,19 @@ mod tests {
     }
 
     #[test]
-    fn test_top_level_options() {
+    fn test_load_log_options_from_cli() {
         let cmd = StartCommand {
             disable_dashboard: Some(false),
             ..Default::default()
         };
 
         let options = cmd
-            .load_options(TopLevelOptions {
+            .load_options(&CliOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
+
+                #[cfg(feature = "tokio-console")]
+                tokio_console_addr: None,
             })
             .unwrap();
 
@@ -416,11 +474,8 @@ mod tests {
                     ..Default::default()
                 };
 
-                let top_level_opts = TopLevelOptions {
-                    log_dir: None,
-                    log_level: Some("error".to_string()),
-                };
-                let Options::Frontend(fe_opts) = command.load_options(top_level_opts).unwrap()
+                let Options::Frontend(fe_opts) =
+                    command.load_options(&CliOptions::default()).unwrap()
                 else {
                     unreachable!()
                 };

src/cmd/src/lib.rs

@@ -12,7 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(assert_matches)]
+#![feature(assert_matches, let_chains)]
+
+use async_trait::async_trait;
+use clap::arg;
+use common_telemetry::{error, info};
 
 pub mod cli;
 pub mod datanode;
@@ -21,3 +25,100 @@ pub mod frontend;
 pub mod metasrv;
 pub mod options;
 pub mod standalone;
+
+lazy_static::lazy_static! {
+    static ref APP_VERSION: prometheus::IntGaugeVec =
+        prometheus::register_int_gauge_vec!("app_version", "app version", &["short_version", "version"]).unwrap();
+}
+
+#[async_trait]
+pub trait App {
+    fn name(&self) -> &str;
+
+    async fn start(&mut self) -> error::Result<()>;
+
+    async fn stop(&self) -> error::Result<()>;
+}
+
+pub async fn start_app(mut app: Box<dyn App>) -> error::Result<()> {
+    let name = app.name().to_string();
+
+    tokio::select! {
+        result = app.start() => {
+            if let Err(err) = result {
+                error!(err; "Failed to start app {name}!");
+            }
+        }
+        _ = tokio::signal::ctrl_c() => {
+            if let Err(err) = app.stop().await {
+                error!(err; "Failed to stop app {name}!");
+            }
+            info!("Goodbye!");
+        }
+    }
+
+    Ok(())
+}
+
+pub fn log_versions() {
+    // Report app version as gauge.
+    APP_VERSION
+        .with_label_values(&[short_version(), full_version()])
+        .inc();
+
+    // Log version and argument flags.
+    info!(
+        "short_version: {}, full_version: {}",
+        short_version(),
+        full_version()
+    );
+
+    log_env_flags();
+}
+
+pub fn greptimedb_cli() -> clap::Command {
+    let cmd = clap::Command::new("greptimedb")
+        .version(print_version())
+        .subcommand_required(true);
+
+    #[cfg(feature = "tokio-console")]
+    let cmd = cmd.arg(arg!(--"tokio-console-addr"[TOKIO_CONSOLE_ADDR]));
+
+    cmd.args([arg!(--"log-dir"[LOG_DIR]), arg!(--"log-level"[LOG_LEVEL])])
+}
+
+fn print_version() -> &'static str {
+    concat!(
+        "\nbranch: ",
+        env!("GIT_BRANCH"),
+        "\ncommit: ",
+        env!("GIT_COMMIT"),
+        "\ndirty: ",
+        env!("GIT_DIRTY"),
+        "\nversion: ",
+        env!("CARGO_PKG_VERSION")
+    )
+}
+
+fn short_version() -> &'static str {
+    env!("CARGO_PKG_VERSION")
+}
+
+// {app_name}-{branch_name}-{commit_short}
+// The branch name (tag) of a release build should already contain the short
+// version so the full version doesn't concat the short version explicitly.
+fn full_version() -> &'static str {
+    concat!(
+        "greptimedb-",
+        env!("GIT_BRANCH"),
+        "-",
+        env!("GIT_COMMIT_SHORT")
+    )
+}
+
+fn log_env_flags() {
+    info!("command line arguments");
+    for argument in std::env::args() {
+        info!("argument: {}", argument);
+    }
+}

src/cmd/src/metasrv.rs

@@ -14,6 +14,7 @@
 
 use std::time::Duration;
 
+use async_trait::async_trait;
 use clap::Parser;
 use common_telemetry::logging;
 use meta_srv::bootstrap::MetaSrvInstance;
@@ -21,21 +22,34 @@ use meta_srv::metasrv::MetaSrvOptions;
 use snafu::ResultExt;
 
 use crate::error::{self, Result, StartMetaServerSnafu};
-use crate::options::{Options, TopLevelOptions};
+use crate::options::{CliOptions, Options};
+use crate::App;
 
 pub struct Instance {
     instance: MetaSrvInstance,
 }
 
 impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
+    fn new(instance: MetaSrvInstance) -> Self {
+        Self { instance }
+    }
+}
+
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-metasrv"
+    }
+
+    async fn start(&mut self) -> Result<()> {
         plugins::start_meta_srv_plugins(self.instance.plugins())
             .await
             .context(StartMetaServerSnafu)?;
+
         self.instance.start().await.context(StartMetaServerSnafu)
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn stop(&self) -> Result<()> {
         self.instance
             .shutdown()
             .await
@@ -54,8 +68,8 @@ impl Command {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
-        self.subcmd.load_options(top_level_opts)
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        self.subcmd.load_options(cli_options)
     }
 }
 
@@ -71,9 +85,9 @@ impl SubCommand {
         }
     }
 
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Start(cmd) => cmd.load_options(top_level_opts),
+            SubCommand::Start(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -103,22 +117,26 @@ struct StartCommand {
     /// The working home directory of this metasrv instance.
     #[clap(long)]
     data_home: Option<String>,
+
+    /// If it's not empty, the metasrv will store all data with this key prefix.
+    #[clap(long, default_value = "")]
+    store_key_prefix: String,
 }
 
 impl StartCommand {
-    fn load_options(&self, top_level_opts: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         let mut opts: MetaSrvOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
             None,
         )?;
 
-        if let Some(dir) = top_level_opts.log_dir {
-            opts.logging.dir = dir;
+        if let Some(dir) = &cli_options.log_dir {
+            opts.logging.dir = dir.clone();
         }
 
-        if top_level_opts.log_level.is_some() {
-            opts.logging.level = top_level_opts.log_level;
+        if cli_options.log_level.is_some() {
+            opts.logging.level = cli_options.log_level.clone();
         }
 
         if let Some(addr) = &self.bind_addr {
@@ -159,6 +177,10 @@ impl StartCommand {
             opts.data_home = data_home.clone();
         }
 
+        if !self.store_key_prefix.is_empty() {
+            opts.store_key_prefix = self.store_key_prefix.clone()
+        }
+
         // Disable dashboard in metasrv.
         opts.http.disable_dashboard = true;
 
@@ -173,11 +195,16 @@ impl StartCommand {
         logging::info!("MetaSrv start command: {:#?}", self);
         logging::info!("MetaSrv options: {:#?}", opts);
 
-        let instance = MetaSrvInstance::new(opts, plugins)
+        let builder = meta_srv::bootstrap::metasrv_builder(&opts, plugins.clone(), None)
+            .await
+            .context(error::BuildMetaServerSnafu)?;
+        let metasrv = builder.build().await.context(error::BuildMetaServerSnafu)?;
+
+        let instance = MetaSrvInstance::new(opts, plugins, metasrv)
             .await
             .context(error::BuildMetaServerSnafu)?;
 
-        Ok(Instance { instance })
+        Ok(Instance::new(instance))
     }
 }
 
@@ -201,8 +228,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Metasrv(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Metasrv(options) = cmd.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
@@ -237,8 +263,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Metasrv(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Metasrv(options) = cmd.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
@@ -269,7 +294,7 @@ mod tests {
     }
 
     #[test]
-    fn test_top_level_options() {
+    fn test_load_log_options_from_cli() {
         let cmd = StartCommand {
             bind_addr: Some("127.0.0.1:3002".to_string()),
             server_addr: Some("127.0.0.1:3002".to_string()),
@@ -279,9 +304,12 @@ mod tests {
         };
 
         let options = cmd
-            .load_options(TopLevelOptions {
+            .load_options(&CliOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
+
+                #[cfg(feature = "tokio-console")]
+                tokio_console_addr: None,
             })
             .unwrap();
 
@@ -340,8 +368,7 @@ mod tests {
                     ..Default::default()
                 };
 
-                let Options::Metasrv(opts) =
-                    command.load_options(TopLevelOptions::default()).unwrap()
+                let Options::Metasrv(opts) = command.load_options(&CliOptions::default()).unwrap()
                 else {
                     unreachable!()
                 };

src/cmd/src/options.rs

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use clap::ArgMatches;
 use common_config::KvBackendConfig;
-use common_telemetry::logging::LoggingOptions;
+use common_meta::wal::WalConfig as MetaSrvWalConfig;
+use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use config::{Config, Environment, File, FileFormat};
 use datanode::config::{DatanodeOptions, ProcedureConfig};
 use frontend::error::{Result as FeResult, TomlFormatSnafu};
@@ -28,7 +30,7 @@ pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
 
 /// Options mixed up from datanode, frontend and metasrv.
-#[derive(Serialize, Debug)]
+#[derive(Serialize, Debug, Clone)]
 pub struct MixOptions {
     pub data_home: String,
     pub procedure: ProcedureConfig,
@@ -36,6 +38,7 @@ pub struct MixOptions {
     pub frontend: FrontendOptions,
     pub datanode: DatanodeOptions,
     pub logging: LoggingOptions,
+    pub wal_meta: MetaSrvWalConfig,
 }
 
 impl From<MixOptions> for FrontendOptions {
@@ -58,10 +61,32 @@ pub enum Options {
     Cli(Box<LoggingOptions>),
 }
 
-#[derive(Clone, Debug, Default)]
-pub struct TopLevelOptions {
+#[derive(Default)]
+pub struct CliOptions {
     pub log_dir: Option<String>,
     pub log_level: Option<String>,
+
+    #[cfg(feature = "tokio-console")]
+    pub tokio_console_addr: Option<String>,
+}
+
+impl CliOptions {
+    pub fn new(args: &ArgMatches) -> Self {
+        Self {
+            log_dir: args.get_one::<String>("log-dir").cloned(),
+            log_level: args.get_one::<String>("log-level").cloned(),
+
+            #[cfg(feature = "tokio-console")]
+            tokio_console_addr: args.get_one::<String>("tokio-console-addr").cloned(),
+        }
+    }
+
+    pub fn tracing_options(&self) -> TracingOptions {
+        TracingOptions {
+            #[cfg(feature = "tokio-console")]
+            tokio_console_addr: self.tokio_console_addr.clone(),
+        }
+    }
 }
 
 impl Options {
@@ -148,6 +173,7 @@ impl Options {
 mod tests {
     use std::io::Write;
 
+    use common_config::WalConfig;
     use common_test_util::temp_dir::create_named_temp_file;
     use datanode::config::{DatanodeOptions, ObjectStoreConfig};
 
@@ -171,6 +197,7 @@ mod tests {
             tcp_nodelay = true
 
             [wal]
+            provider = "raft_engine"
             dir = "/tmp/greptimedb/wal"
             file_size = "1GB"
             purge_threshold = "50GB"
@@ -238,7 +265,7 @@ mod tests {
                 .unwrap();
 
                 // Check the configs from environment variables.
-                match opts.storage.store {
+                match &opts.storage.store {
                     ObjectStoreConfig::S3(s3_config) => {
                         assert_eq!(s3_config.bucket, "mybucket".to_string());
                     }
@@ -254,7 +281,10 @@ mod tests {
                 );
 
                 // Should be the values from config file, not environment variables.
-                assert_eq!(opts.wal.dir.unwrap(), "/tmp/greptimedb/wal");
+                let WalConfig::RaftEngine(raft_engine_config) = opts.wal else {
+                    unreachable!()
+                };
+                assert_eq!(raft_engine_config.dir.unwrap(), "/tmp/greptimedb/wal");
 
                 // Should be default values.
                 assert_eq!(opts.node_id, None);

src/cmd/src/standalone.rs

@@ -15,38 +15,49 @@
 use std::sync::Arc;
 use std::{fs, path};
 
-use catalog::kvbackend::KvBackendCatalogManager;
-use catalog::CatalogManagerRef;
+use async_trait::async_trait;
 use clap::Parser;
-use common_base::Plugins;
-use common_config::{metadata_store_dir, KvBackendConfig, WalConfig};
-use common_meta::cache_invalidator::DummyKvCacheInvalidator;
+use common_catalog::consts::MIN_USER_TABLE_ID;
+use common_config::wal::StandaloneWalConfig;
+use common_config::{metadata_store_dir, KvBackendConfig};
+use common_meta::cache_invalidator::DummyCacheInvalidator;
+use common_meta::datanode_manager::DatanodeManagerRef;
+use common_meta::ddl::{DdlTaskExecutorRef, TableMetadataAllocatorRef};
+use common_meta::ddl_manager::DdlManager;
+use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
+use common_meta::region_keeper::MemoryRegionKeeper;
+use common_meta::sequence::SequenceBuilder;
+use common_meta::wal::{WalOptionsAllocator, WalOptionsAllocatorRef};
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
+use common_time::timezone::set_default_timezone;
 use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
-use datanode::region_server::RegionServer;
 use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
+use frontend::instance::builder::FrontendBuilder;
+use frontend::instance::standalone::StandaloneTableMetadataAllocator;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
 use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
+use servers::export_metrics::ExportMetricsOption;
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::ResultExt;
 
 use crate::error::{
-    CreateDirSnafu, IllegalConfigSnafu, InitMetadataSnafu, Result, ShutdownDatanodeSnafu,
-    ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
-    StopProcedureManagerSnafu,
+    CreateDirSnafu, IllegalConfigSnafu, InitDdlManagerSnafu, InitMetadataSnafu, InitTimezoneSnafu,
+    Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu,
+    StartProcedureManagerSnafu, StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
 };
-use crate::options::{MixOptions, Options, TopLevelOptions};
+use crate::options::{CliOptions, MixOptions, Options};
+use crate::App;
 
 #[derive(Parser)]
 pub struct Command {
@@ -59,8 +70,8 @@ impl Command {
         self.subcmd.build(opts).await
     }
 
-    pub fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
-        self.subcmd.load_options(top_level_options)
+    pub fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        self.subcmd.load_options(cli_options)
     }
 }
 
@@ -76,9 +87,9 @@ impl SubCommand {
         }
     }
 
-    fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
         match self {
-            SubCommand::Start(cmd) => cmd.load_options(top_level_options),
+            SubCommand::Start(cmd) => cmd.load_options(cli_options),
         }
     }
 }
@@ -88,6 +99,7 @@ impl SubCommand {
 pub struct StandaloneOptions {
     pub mode: Mode,
     pub enable_telemetry: bool,
+    pub default_timezone: Option<String>,
     pub http: HttpOptions,
     pub grpc: GrpcOptions,
     pub mysql: MysqlOptions,
@@ -95,7 +107,7 @@ pub struct StandaloneOptions {
     pub opentsdb: OpentsdbOptions,
     pub influxdb: InfluxdbOptions,
     pub prom_store: PromStoreOptions,
-    pub wal: WalConfig,
+    pub wal: StandaloneWalConfig,
     pub storage: StorageConfig,
     pub metadata_store: KvBackendConfig,
     pub procedure: ProcedureConfig,
@@ -103,6 +115,7 @@ pub struct StandaloneOptions {
     pub user_provider: Option<String>,
     /// Options for different store engines.
     pub region_engine: Vec<RegionEngineConfig>,
+    pub export_metrics: ExportMetricsOption,
 }
 
 impl Default for StandaloneOptions {
@@ -110,6 +123,7 @@ impl Default for StandaloneOptions {
         Self {
             mode: Mode::Standalone,
             enable_telemetry: true,
+            default_timezone: None,
             http: HttpOptions::default(),
             grpc: GrpcOptions::default(),
             mysql: MysqlOptions::default(),
@@ -117,11 +131,12 @@ impl Default for StandaloneOptions {
             opentsdb: OpentsdbOptions::default(),
             influxdb: InfluxdbOptions::default(),
             prom_store: PromStoreOptions::default(),
-            wal: WalConfig::default(),
+            wal: StandaloneWalConfig::default(),
             storage: StorageConfig::default(),
             metadata_store: KvBackendConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
+            export_metrics: ExportMetricsOption::default(),
             user_provider: None,
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig::default()),
@@ -135,6 +150,7 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
+            default_timezone: self.default_timezone,
             http: self.http,
             grpc: self.grpc,
             mysql: self.mysql,
@@ -145,6 +161,8 @@ impl StandaloneOptions {
             meta_client: None,
             logging: self.logging,
             user_provider: self.user_provider,
+            // Handle the export metrics task run by standalone to frontend for execution
+            export_metrics: self.export_metrics,
             ..Default::default()
         }
     }
@@ -153,9 +171,10 @@ impl StandaloneOptions {
         DatanodeOptions {
             node_id: Some(0),
             enable_telemetry: self.enable_telemetry,
-            wal: self.wal,
+            wal: self.wal.into(),
             storage: self.storage,
             region_engine: self.region_engine,
+            rpc_addr: self.grpc.addr,
             ..Default::default()
         }
     }
@@ -165,10 +184,16 @@ pub struct Instance {
     datanode: Datanode,
     frontend: FeInstance,
     procedure_manager: ProcedureManagerRef,
+    wal_options_allocator: WalOptionsAllocatorRef,
 }
 
-impl Instance {
-    pub async fn start(&mut self) -> Result<()> {
+#[async_trait]
+impl App for Instance {
+    fn name(&self) -> &str {
+        "greptime-standalone"
+    }
+
+    async fn start(&mut self) -> Result<()> {
         self.datanode.start_telemetry();
 
         self.procedure_manager
@@ -176,11 +201,16 @@ impl Instance {
             .await
             .context(StartProcedureManagerSnafu)?;
 
+        self.wal_options_allocator
+            .start()
+            .await
+            .context(StartWalOptionsAllocatorSnafu)?;
+
         self.frontend.start().await.context(StartFrontendSnafu)?;
         Ok(())
     }
 
-    pub async fn stop(&self) -> Result<()> {
+    async fn stop(&self) -> Result<()> {
         self.frontend
             .shutdown()
             .await
@@ -202,7 +232,7 @@ impl Instance {
 }
 
 #[derive(Debug, Default, Parser)]
-struct StartCommand {
+pub struct StartCommand {
     #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
@@ -216,7 +246,7 @@ struct StartCommand {
     #[clap(short, long)]
     influxdb_enable: bool,
     #[clap(short, long)]
-    config_file: Option<String>,
+    pub config_file: Option<String>,
     #[clap(long)]
     tls_mode: Option<TlsMode>,
     #[clap(long)]
@@ -226,28 +256,36 @@ struct StartCommand {
     #[clap(long)]
     user_provider: Option<String>,
     #[clap(long, default_value = "GREPTIMEDB_STANDALONE")]
-    env_prefix: String,
+    pub env_prefix: String,
     /// The working home directory of this standalone instance.
     #[clap(long)]
     data_home: Option<String>,
 }
 
 impl StartCommand {
-    fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
-        let mut opts: StandaloneOptions = Options::load_layered_options(
+    fn load_options(&self, cli_options: &CliOptions) -> Result<Options> {
+        let opts: StandaloneOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
             None,
         )?;
 
+        self.convert_options(cli_options, opts)
+    }
+
+    pub fn convert_options(
+        &self,
+        cli_options: &CliOptions,
+        mut opts: StandaloneOptions,
+    ) -> Result<Options> {
         opts.mode = Mode::Standalone;
 
-        if let Some(dir) = top_level_options.log_dir {
-            opts.logging.dir = dir;
+        if let Some(dir) = &cli_options.log_dir {
+            opts.logging.dir = dir.clone();
         }
 
-        if top_level_options.log_level.is_some() {
-            opts.logging.level = top_level_options.log_level;
+        if cli_options.log_level.is_some() {
+            opts.logging.level = cli_options.log_level.clone();
         }
 
         let tls_opts = TlsOption::new(
@@ -305,7 +343,8 @@ impl StartCommand {
         let procedure = opts.procedure.clone();
         let frontend = opts.clone().frontend_options();
         let logging = opts.logging.clone();
-        let datanode = opts.datanode_options();
+        let wal_meta = opts.wal.clone().into();
+        let datanode = opts.datanode_options().clone();
 
         Ok(Options::Standalone(Box::new(MixOptions {
             procedure,
@@ -314,6 +353,7 @@ impl StartCommand {
             frontend,
             datanode,
             logging,
+            wal_meta,
         })))
     }
 
@@ -333,6 +373,9 @@ impl StartCommand {
 
         info!("Building standalone instance with {opts:#?}");
 
+        set_default_timezone(opts.frontend.default_timezone.as_deref())
+            .context(InitTimezoneSnafu)?;
+
         // Ensure the data_home directory exists.
         fs::create_dir_all(path::Path::new(&opts.data_home)).context(CreateDirSnafu {
             dir: &opts.data_home,
@@ -347,36 +390,45 @@ impl StartCommand {
         .await
         .context(StartFrontendSnafu)?;
 
-        let datanode = DatanodeBuilder::new(dn_opts, fe_plugins.clone())
-            .with_kv_backend(kv_backend.clone())
-            .build()
-            .await
-            .context(StartDatanodeSnafu)?;
+        let builder =
+            DatanodeBuilder::new(dn_opts, fe_plugins.clone()).with_kv_backend(kv_backend.clone());
+        let datanode = builder.build().await.context(StartDatanodeSnafu)?;
 
-        let region_server = datanode.region_server();
+        let datanode_manager = Arc::new(StandaloneDatanodeManager(datanode.region_server()));
 
-        let catalog_manager = KvBackendCatalogManager::new(
-            kv_backend.clone(),
-            Arc::new(DummyKvCacheInvalidator),
-            Arc::new(StandaloneDatanodeManager(region_server.clone())),
+        let table_id_sequence = Arc::new(
+            SequenceBuilder::new("table_id", kv_backend.clone())
+                .initial(MIN_USER_TABLE_ID as u64)
+                .step(10)
+                .build(),
         );
+        let wal_options_allocator = Arc::new(WalOptionsAllocator::new(
+            opts.wal_meta.clone(),
+            kv_backend.clone(),
+        ));
+        let table_meta_allocator = Arc::new(StandaloneTableMetadataAllocator::new(
+            table_id_sequence,
+            wal_options_allocator.clone(),
+        ));
 
-        catalog_manager
-            .table_metadata_manager_ref()
-            .init()
-            .await
-            .context(InitMetadataSnafu)?;
-
-        // TODO: build frontend instance like in distributed mode
-        let mut frontend = build_frontend(
-            fe_plugins,
-            kv_backend,
+        let ddl_task_executor = Self::create_ddl_task_executor(
+            kv_backend.clone(),
             procedure_manager.clone(),
-            catalog_manager,
-            region_server,
+            datanode_manager.clone(),
+            table_meta_allocator,
         )
         .await?;
 
+        let mut frontend = FrontendBuilder::new(kv_backend, datanode_manager, ddl_task_executor)
+            .with_plugin(fe_plugins)
+            .try_build()
+            .await
+            .context(StartFrontendSnafu)?;
+
+        frontend
+            .build_export_metrics_task(&opts.frontend.export_metrics)
+            .context(StartFrontendSnafu)?;
+
         frontend
             .build_servers(opts)
             .await
@@ -386,28 +438,46 @@ impl StartCommand {
             datanode,
             frontend,
             procedure_manager,
+            wal_options_allocator,
         })
     }
-}
 
-/// Build frontend instance in standalone mode
-async fn build_frontend(
-    plugins: Plugins,
-    kv_backend: KvBackendRef,
-    procedure_manager: ProcedureManagerRef,
-    catalog_manager: CatalogManagerRef,
-    region_server: RegionServer,
-) -> Result<FeInstance> {
-    let frontend_instance = FeInstance::try_new_standalone(
-        kv_backend,
-        procedure_manager,
-        catalog_manager,
-        plugins,
-        region_server,
-    )
-    .await
-    .context(StartFrontendSnafu)?;
-    Ok(frontend_instance)
+    pub async fn create_ddl_task_executor(
+        kv_backend: KvBackendRef,
+        procedure_manager: ProcedureManagerRef,
+        datanode_manager: DatanodeManagerRef,
+        table_meta_allocator: TableMetadataAllocatorRef,
+    ) -> Result<DdlTaskExecutorRef> {
+        let table_metadata_manager =
+            Self::create_table_metadata_manager(kv_backend.clone()).await?;
+
+        let ddl_task_executor: DdlTaskExecutorRef = Arc::new(
+            DdlManager::try_new(
+                procedure_manager,
+                datanode_manager,
+                Arc::new(DummyCacheInvalidator),
+                table_metadata_manager,
+                table_meta_allocator,
+                Arc::new(MemoryRegionKeeper::default()),
+            )
+            .context(InitDdlManagerSnafu)?,
+        );
+
+        Ok(ddl_task_executor)
+    }
+
+    async fn create_table_metadata_manager(
+        kv_backend: KvBackendRef,
+    ) -> Result<TableMetadataManagerRef> {
+        let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend));
+
+        table_metadata_manager
+            .init()
+            .await
+            .context(InitMetadataSnafu)?;
+
+        Ok(table_metadata_manager)
+    }
 }
 
 #[cfg(test)]
@@ -418,11 +488,13 @@ mod tests {
 
     use auth::{Identity, Password, UserProviderRef};
     use common_base::readable_size::ReadableSize;
+    use common_config::WalConfig;
     use common_test_util::temp_dir::create_named_temp_file;
+    use datanode::config::{FileConfig, GcsConfig};
     use servers::Mode;
 
     use super::*;
-    use crate::options::ENV_VAR_SEP;
+    use crate::options::{CliOptions, ENV_VAR_SEP};
 
     #[tokio::test]
     async fn test_try_from_start_command_to_anymap() {
@@ -460,6 +532,7 @@ mod tests {
             enable_memory_catalog = true
 
             [wal]
+            provider = "raft_engine"
             dir = "/tmp/greptimedb/test/wal"
             file_size = "1GB"
             purge_threshold = "50GB"
@@ -468,6 +541,15 @@ mod tests {
             sync_write = false
 
             [storage]
+            data_home = "/tmp/greptimedb/"
+            type = "File"
+
+            [[storage.providers]]
+            type = "Gcs"
+            bucket = "foo"
+            endpoint = "bar"
+
+            [[storage.providers]]
             type = "S3"
             access_key_id = "access_key_id"
             secret_access_key = "secret_access_key"
@@ -497,8 +579,7 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Standalone(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
-        else {
+        let Options::Standalone(options) = cmd.load_options(&CliOptions::default()).unwrap() else {
             unreachable!()
         };
         let fe_opts = options.frontend;
@@ -515,9 +596,21 @@ mod tests {
         assert_eq!(None, fe_opts.mysql.reject_no_database);
         assert!(fe_opts.influxdb.enable);
 
-        assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap());
+        let WalConfig::RaftEngine(raft_engine_config) = dn_opts.wal else {
+            unreachable!()
+        };
+        assert_eq!("/tmp/greptimedb/test/wal", raft_engine_config.dir.unwrap());
 
-        match &dn_opts.storage.store {
+        assert!(matches!(
+            &dn_opts.storage.store,
+            datanode::config::ObjectStoreConfig::File(FileConfig { .. })
+        ));
+        assert_eq!(dn_opts.storage.providers.len(), 2);
+        assert!(matches!(
+            dn_opts.storage.providers[0],
+            datanode::config::ObjectStoreConfig::Gcs(GcsConfig { .. })
+        ));
+        match &dn_opts.storage.providers[1] {
             datanode::config::ObjectStoreConfig::S3(s3_config) => {
                 assert_eq!(
                     "Secret([REDACTED alloc::string::String])".to_string(),
@@ -534,16 +627,19 @@ mod tests {
     }
 
     #[test]
-    fn test_top_level_options() {
+    fn test_load_log_options_from_cli() {
         let cmd = StartCommand {
             user_provider: Some("static_user_provider:cmd:test=test".to_string()),
             ..Default::default()
         };
 
         let Options::Standalone(opts) = cmd
-            .load_options(TopLevelOptions {
+            .load_options(&CliOptions {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
+
+                #[cfg(feature = "tokio-console")]
+                tokio_console_addr: None,
             })
             .unwrap()
         else {
@@ -610,11 +706,8 @@ mod tests {
                     ..Default::default()
                 };
 
-                let top_level_opts = TopLevelOptions {
-                    log_dir: None,
-                    log_level: None,
-                };
-                let Options::Standalone(opts) = command.load_options(top_level_opts).unwrap()
+                let Options::Standalone(opts) =
+                    command.load_options(&CliOptions::default()).unwrap()
                 else {
                     unreachable!()
                 };

src/common/base/Cargo.toml

@@ -7,7 +7,7 @@ license.workspace = true
 [dependencies]
 anymap = "1.0.0-beta.2"
 bitvec = "1.0"
-bytes = { version = "1.1", features = ["serde"] }
+bytes.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 paste = "1.0"

src/common/catalog/src/consts.rs

@@ -17,6 +17,7 @@ pub const INFORMATION_SCHEMA_NAME: &str = "information_schema";
 pub const SYSTEM_CATALOG_TABLE_NAME: &str = "system_catalog";
 pub const DEFAULT_CATALOG_NAME: &str = "greptime";
 pub const DEFAULT_SCHEMA_NAME: &str = "public";
+pub const DEFAULT_PRIVATE_SCHEMA_NAME: &str = "greptime_private";
 
 /// Reserves [0,MIN_USER_TABLE_ID) for internal usage.
 /// User defined table id starts from this value.
@@ -29,13 +30,25 @@ pub const SYSTEM_CATALOG_TABLE_ID: u32 = 0;
 pub const SCRIPTS_TABLE_ID: u32 = 1;
 /// numbers table id
 pub const NUMBERS_TABLE_ID: u32 = 2;
+
+/// ----- Begin of information_schema tables -----
 /// id for information_schema.tables
 pub const INFORMATION_SCHEMA_TABLES_TABLE_ID: u32 = 3;
 /// id for information_schema.columns
 pub const INFORMATION_SCHEMA_COLUMNS_TABLE_ID: u32 = 4;
+/// id for information_schema.engines
+pub const INFORMATION_SCHEMA_ENGINES_TABLE_ID: u32 = 5;
+/// id for information_schema.column_privileges
+pub const INFORMATION_SCHEMA_COLUMN_PRIVILEGES_TABLE_ID: u32 = 6;
+/// id for information_schema.column_statistics
+pub const INFORMATION_SCHEMA_COLUMN_STATISTICS_TABLE_ID: u32 = 7;
+/// id for information_schema.build_info
+pub const INFORMATION_SCHEMA_BUILD_INFO_TABLE_ID: u32 = 8;
+/// ----- End of information_schema tables -----
 
 pub const MITO_ENGINE: &str = "mito";
 pub const MITO2_ENGINE: &str = "mito2";
+pub const METRIC_ENGINE: &str = "metric";
 
 pub fn default_engine() -> &'static str {
     MITO_ENGINE

src/common/config/Cargo.toml

@@ -7,4 +7,8 @@ license.workspace = true
 [dependencies]
 common-base.workspace = true
 humantime-serde.workspace = true
+rskafka.workspace = true
 serde.workspace = true
+serde_json.workspace = true
+serde_with = "3"
+toml.workspace = true

src/common/config/src/lib.rs

@@ -12,41 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::time::Duration;
+pub mod wal;
 
 use common_base::readable_size::ReadableSize;
 use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(default)]
-pub struct WalConfig {
-    // wal directory
-    pub dir: Option<String>,
-    // wal file size in bytes
-    pub file_size: ReadableSize,
-    // wal purge threshold in bytes
-    pub purge_threshold: ReadableSize,
-    // purge interval in seconds
-    #[serde(with = "humantime_serde")]
-    pub purge_interval: Duration,
-    // read batch size
-    pub read_batch_size: usize,
-    // whether to sync log file after every write
-    pub sync_write: bool,
-}
-
-impl Default for WalConfig {
-    fn default() -> Self {
-        Self {
-            dir: None,
-            file_size: ReadableSize::mb(256), // log file size 256MB
-            purge_threshold: ReadableSize::gb(4), // purge threshold 4GB
-            purge_interval: Duration::from_secs(600),
-            read_batch_size: 128,
-            sync_write: false,
-        }
-    }
-}
+pub use crate::wal::{KafkaWalOptions, WalConfig, WalOptions, WAL_OPTIONS_KEY};
 
 pub fn metadata_store_dir(store_dir: &str) -> String {
     format!("{store_dir}/metadata")

src/common/config/src/wal.rs

@@ -0,0 +1,142 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod kafka;
+pub mod raft_engine;
+
+use serde::{Deserialize, Serialize};
+use serde_with::with_prefix;
+
+pub use crate::wal::kafka::{
+    KafkaConfig, KafkaOptions as KafkaWalOptions, StandaloneKafkaConfig, Topic as KafkaWalTopic,
+};
+pub use crate::wal::raft_engine::RaftEngineConfig;
+
+/// An encoded wal options will be wrapped into a (WAL_OPTIONS_KEY, encoded wal options) key-value pair
+/// and inserted into the options of a `RegionCreateRequest`.
+pub const WAL_OPTIONS_KEY: &str = "wal_options";
+
+/// Wal config for datanode.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(tag = "provider", rename_all = "snake_case")]
+pub enum WalConfig {
+    RaftEngine(RaftEngineConfig),
+    Kafka(KafkaConfig),
+}
+
+impl From<StandaloneWalConfig> for WalConfig {
+    fn from(value: StandaloneWalConfig) -> Self {
+        match value {
+            StandaloneWalConfig::RaftEngine(config) => WalConfig::RaftEngine(config),
+            StandaloneWalConfig::Kafka(config) => WalConfig::Kafka(config.base),
+        }
+    }
+}
+
+impl Default for WalConfig {
+    fn default() -> Self {
+        WalConfig::RaftEngine(RaftEngineConfig::default())
+    }
+}
+
+/// Wal config for datanode.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(tag = "provider", rename_all = "snake_case")]
+pub enum StandaloneWalConfig {
+    RaftEngine(RaftEngineConfig),
+    Kafka(StandaloneKafkaConfig),
+}
+
+impl Default for StandaloneWalConfig {
+    fn default() -> Self {
+        StandaloneWalConfig::RaftEngine(RaftEngineConfig::default())
+    }
+}
+
+/// Wal options allocated to a region.
+/// A wal options is encoded by metasrv with `serde_json::to_string`, and then decoded
+/// by datanode with `serde_json::from_str`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[serde(tag = "wal.provider", rename_all = "snake_case")]
+pub enum WalOptions {
+    #[default]
+    RaftEngine,
+    #[serde(with = "prefix_wal_kafka")]
+    Kafka(KafkaWalOptions),
+}
+
+with_prefix!(prefix_wal_kafka "wal.kafka.");
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use common_base::readable_size::ReadableSize;
+    use rskafka::client::partition::Compression as RsKafkaCompression;
+
+    use crate::wal::kafka::KafkaBackoffConfig;
+    use crate::wal::{KafkaConfig, KafkaWalOptions, WalOptions};
+
+    #[test]
+    fn test_serde_kafka_config() {
+        let toml_str = r#"
+            broker_endpoints = ["127.0.0.1:9092"]
+            max_batch_size = "4MB"
+            linger = "200ms"
+            produce_record_timeout = "100ms"
+            backoff_init = "500ms"
+            backoff_max = "10s"
+            backoff_base = 2
+            backoff_deadline = "5mins"
+        "#;
+        let decoded: KafkaConfig = toml::from_str(toml_str).unwrap();
+        let expected = KafkaConfig {
+            broker_endpoints: vec!["127.0.0.1:9092".to_string()],
+            compression: RsKafkaCompression::default(),
+            max_batch_size: ReadableSize::mb(4),
+            linger: Duration::from_millis(200),
+            produce_record_timeout: Duration::from_millis(100),
+            backoff: KafkaBackoffConfig {
+                init: Duration::from_millis(500),
+                max: Duration::from_secs(10),
+                base: 2,
+                deadline: Some(Duration::from_secs(60 * 5)),
+            },
+        };
+        assert_eq!(decoded, expected);
+    }
+
+    #[test]
+    fn test_serde_wal_options() {
+        // Test serde raft-engine wal options.
+        let wal_options = WalOptions::RaftEngine;
+        let encoded = serde_json::to_string(&wal_options).unwrap();
+        let expected = r#"{"wal.provider":"raft_engine"}"#;
+        assert_eq!(&encoded, expected);
+
+        let decoded: WalOptions = serde_json::from_str(&encoded).unwrap();
+        assert_eq!(decoded, wal_options);
+
+        // Test serde kafka wal options.
+        let wal_options = WalOptions::Kafka(KafkaWalOptions {
+            topic: "test_topic".to_string(),
+        });
+        let encoded = serde_json::to_string(&wal_options).unwrap();
+        let expected = r#"{"wal.provider":"kafka","wal.kafka.topic":"test_topic"}"#;
+        assert_eq!(&encoded, expected);
+
+        let decoded: WalOptions = serde_json::from_str(&encoded).unwrap();
+        assert_eq!(decoded, wal_options);
+    }
+}

src/common/config/src/wal/kafka.rs

@@ -0,0 +1,144 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use common_base::readable_size::ReadableSize;
+use rskafka::client::partition::Compression as RsKafkaCompression;
+use serde::{Deserialize, Serialize};
+use serde_with::with_prefix;
+
+/// Topic name prefix.
+pub const TOPIC_NAME_PREFIX: &str = "greptimedb_wal_topic";
+/// Kafka wal topic.
+pub type Topic = String;
+
+/// The type of the topic selector, i.e. with which strategy to select a topic.
+#[derive(Default, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum TopicSelectorType {
+    #[default]
+    RoundRobin,
+}
+
+/// Configurations for kafka wal.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct KafkaConfig {
+    /// The broker endpoints of the Kafka cluster.
+    pub broker_endpoints: Vec<String>,
+    /// The compression algorithm used to compress log entries.
+    #[serde(skip)]
+    #[serde(default)]
+    pub compression: RsKafkaCompression,
+    /// The maximum log size a kakfa batch producer could buffer.
+    pub max_batch_size: ReadableSize,
+    /// The linger duration of a kafka batch producer.
+    #[serde(with = "humantime_serde")]
+    pub linger: Duration,
+    /// The maximum amount of time (in milliseconds) to wait for Kafka records to be returned.
+    #[serde(with = "humantime_serde")]
+    pub produce_record_timeout: Duration,
+    /// The backoff config.
+    #[serde(flatten, with = "kafka_backoff")]
+    pub backoff: KafkaBackoffConfig,
+}
+
+impl Default for KafkaConfig {
+    fn default() -> Self {
+        Self {
+            broker_endpoints: vec!["127.0.0.1:9092".to_string()],
+            compression: RsKafkaCompression::NoCompression,
+            max_batch_size: ReadableSize::mb(4),
+            linger: Duration::from_millis(200),
+            produce_record_timeout: Duration::from_millis(100),
+            backoff: KafkaBackoffConfig::default(),
+        }
+    }
+}
+
+with_prefix!(pub kafka_backoff "backoff_");
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct KafkaBackoffConfig {
+    /// The initial backoff for kafka clients.
+    #[serde(with = "humantime_serde")]
+    pub init: Duration,
+    /// The maximum backoff for kafka clients.
+    #[serde(with = "humantime_serde")]
+    pub max: Duration,
+    /// Exponential backoff rate, i.e. next backoff = base * current backoff.
+    // Sets to u32 type since some structs containing the KafkaConfig need to derive the Eq trait.
+    pub base: u32,
+    /// Stop reconnecting if the total wait time reaches the deadline.
+    /// If it's None, the reconnecting won't terminate.
+    #[serde(with = "humantime_serde")]
+    pub deadline: Option<Duration>,
+}
+
+impl Default for KafkaBackoffConfig {
+    fn default() -> Self {
+        Self {
+            init: Duration::from_millis(500),
+            max: Duration::from_secs(10),
+            base: 2,
+            deadline: Some(Duration::from_secs(60 * 5)), // 5 mins
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct StandaloneKafkaConfig {
+    #[serde(flatten)]
+    pub base: KafkaConfig,
+    /// Number of topics to be created upon start.
+    pub num_topics: usize,
+    /// The type of the topic selector with which to select a topic for a region.
+    pub selector_type: TopicSelectorType,
+    /// Topic name prefix.
+    pub topic_name_prefix: String,
+    /// Number of partitions per topic.
+    pub num_partitions: i32,
+    /// The replication factor of each topic.
+    pub replication_factor: i16,
+    /// Above which a topic creation operation will be cancelled.
+    #[serde(with = "humantime_serde")]
+    pub create_topic_timeout: Duration,
+}
+
+impl Default for StandaloneKafkaConfig {
+    fn default() -> Self {
+        let base = KafkaConfig::default();
+        let replication_factor = base.broker_endpoints.len() as i16;
+
+        Self {
+            base,
+            num_topics: 64,
+            selector_type: TopicSelectorType::RoundRobin,
+            topic_name_prefix: "greptimedb_wal_topic".to_string(),
+            num_partitions: 1,
+            replication_factor,
+            create_topic_timeout: Duration::from_secs(30),
+        }
+    }
+}
+
+/// Kafka wal options allocated to a region.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct KafkaOptions {
+    /// Kafka wal topic.
+    pub topic: Topic,
+}

src/common/config/src/wal/raft_engine.rs

@@ -0,0 +1,50 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use common_base::readable_size::ReadableSize;
+use serde::{Deserialize, Serialize};
+
+/// Configurations for raft-engine wal.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct RaftEngineConfig {
+    // wal directory
+    pub dir: Option<String>,
+    // wal file size in bytes
+    pub file_size: ReadableSize,
+    // wal purge threshold in bytes
+    pub purge_threshold: ReadableSize,
+    // purge interval in seconds
+    #[serde(with = "humantime_serde")]
+    pub purge_interval: Duration,
+    // read batch size
+    pub read_batch_size: usize,
+    // whether to sync log file after every write
+    pub sync_write: bool,
+}
+
+impl Default for RaftEngineConfig {
+    fn default() -> Self {
+        Self {
+            dir: None,
+            file_size: ReadableSize::mb(256),
+            purge_threshold: ReadableSize::gb(4),
+            purge_interval: Duration::from_secs(600),
+            read_batch_size: 128,
+            sync_write: false,
+        }
+    }
+}

src/common/datasource/Cargo.toml

@@ -16,7 +16,7 @@ async-compression = { version = "0.3", features = [
     "tokio",
 ] }
 async-trait.workspace = true
-bytes = "1.1"
+bytes.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 common-runtime.workspace = true

src/common/datasource/src/object_store/fs.rs

@@ -23,6 +23,15 @@ pub fn build_fs_backend(root: &str) -> Result<ObjectStore> {
     let _ = builder.root(root);
     let object_store = ObjectStore::new(builder)
         .context(BuildBackendSnafu)?
+        .layer(
+            object_store::layers::LoggingLayer::default()
+                // Print the expected error only in DEBUG level.
+                // See https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html#method.with_error_level
+                .with_error_level(Some("debug"))
+                .expect("input error level must be valid"),
+        )
+        .layer(object_store::layers::TracingLayer)
+        .layer(object_store::layers::PrometheusMetricsLayer)
         .finish();
     Ok(object_store)
 }

src/common/datasource/src/object_store/s3.rs

@@ -80,8 +80,18 @@ pub fn build_s3_backend(
         }
     }
 
+    // TODO(weny): Consider finding a better way to eliminate duplicate code.
     Ok(ObjectStore::new(builder)
         .context(error::BuildBackendSnafu)?
+        .layer(
+            object_store::layers::LoggingLayer::default()
+                // Print the expected error only in DEBUG level.
+                // See https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html#method.with_error_level
+                .with_error_level(Some("debug"))
+                .expect("input error level must be valid"),
+        )
+        .layer(object_store::layers::TracingLayer)
+        .layer(object_store::layers::PrometheusMetricsLayer)
         .finish())
 }
 

src/common/decimal/Cargo.toml

@@ -11,5 +11,5 @@ common-error.workspace = true
 common-macro.workspace = true
 rust_decimal.workspace = true
 serde.workspace = true
-serde_json = "1.0"
+serde_json.workspace = true
 snafu.workspace = true

src/common/decimal/src/decimal128.rs

@@ -110,9 +110,15 @@ impl Decimal128 {
     }
 
     /// Convert from precision, scale, a i128 value which
-    /// represents by two i64 value(high-64 bit, low-64 bit).
+    /// represents by i64 + i64 value(high-64 bit, low-64 bit).
     pub fn from_value_precision_scale(hi: i64, lo: i64, precision: u8, scale: i8) -> Self {
-        let value = (hi as i128) << 64 | lo as i128;
+        // 128                             64                              0
+        // +-------+-------+-------+-------+-------+-------+-------+-------+
+        // |               hi              |               lo              |
+        // +-------+-------+-------+-------+-------+-------+-------+-------+
+        let hi = (hi as u128 & u64::MAX as u128) << 64;
+        let lo = lo as u128 & u64::MAX as u128;
+        let value = (hi | lo) as i128;
         Self::new(value, precision, scale)
     }
 }
@@ -429,4 +435,30 @@ mod tests {
         let decimal2 = Decimal128::from_str("1234567890.123").unwrap();
         assert_eq!(decimal1.partial_cmp(&decimal2), None);
     }
+
+    #[test]
+    fn test_convert_with_i128() {
+        let test_decimal128_eq = |value| {
+            let decimal1 =
+                Decimal128::new(value, DECIMAL128_MAX_PRECISION, DECIMAL128_DEFAULT_SCALE);
+            let (hi, lo) = decimal1.split_value();
+            let decimal2 = Decimal128::from_value_precision_scale(
+                hi,
+                lo,
+                DECIMAL128_MAX_PRECISION,
+                DECIMAL128_DEFAULT_SCALE,
+            );
+            assert_eq!(decimal1, decimal2);
+        };
+
+        test_decimal128_eq(1 << 63);
+
+        test_decimal128_eq(0);
+        test_decimal128_eq(1234567890);
+        test_decimal128_eq(-1234567890);
+        test_decimal128_eq(32781372819372817382183218i128);
+        test_decimal128_eq(-32781372819372817382183218i128);
+        test_decimal128_eq(i128::MAX);
+        test_decimal128_eq(i128::MIN);
+    }
 }

src/common/error/src/status_code.rs

@@ -59,6 +59,10 @@ pub enum StatusCode {
     RegionNotFound = 4005,
     RegionAlreadyExists = 4006,
     RegionReadonly = 4007,
+    RegionNotReady = 4008,
+    // If mutually exclusive operations are reached at the same time,
+    // only one can be executed, another one will get region busy.
+    RegionBusy = 4009,
     // ====== End of catalog related status code =======
 
     // ====== Begin of storage related status code =====
@@ -103,7 +107,9 @@ impl StatusCode {
         match self {
             StatusCode::StorageUnavailable
             | StatusCode::RuntimeResourcesExhausted
-            | StatusCode::Internal => true,
+            | StatusCode::Internal
+            | StatusCode::RegionNotReady
+            | StatusCode::RegionBusy => true,
 
             StatusCode::Success
             | StatusCode::Unknown
@@ -138,7 +144,6 @@ impl StatusCode {
     pub fn should_log_error(&self) -> bool {
         match self {
             StatusCode::Unknown
-            | StatusCode::Unsupported
             | StatusCode::Unexpected
             | StatusCode::Internal
             | StatusCode::Cancelled
@@ -147,11 +152,14 @@ impl StatusCode {
             | StatusCode::StorageUnavailable
             | StatusCode::RuntimeResourcesExhausted => true,
             StatusCode::Success
+            | StatusCode::Unsupported
             | StatusCode::InvalidArguments
             | StatusCode::InvalidSyntax
             | StatusCode::TableAlreadyExists
             | StatusCode::TableNotFound
             | StatusCode::RegionNotFound
+            | StatusCode::RegionNotReady
+            | StatusCode::RegionBusy
             | StatusCode::RegionAlreadyExists
             | StatusCode::RegionReadonly
             | StatusCode::TableColumnNotFound
@@ -183,6 +191,8 @@ impl StatusCode {
             v if v == StatusCode::TableAlreadyExists as u32 => Some(StatusCode::TableAlreadyExists),
             v if v == StatusCode::TableNotFound as u32 => Some(StatusCode::TableNotFound),
             v if v == StatusCode::RegionNotFound as u32 => Some(StatusCode::RegionNotFound),
+            v if v == StatusCode::RegionNotReady as u32 => Some(StatusCode::RegionNotReady),
+            v if v == StatusCode::RegionBusy as u32 => Some(StatusCode::RegionBusy),
             v if v == StatusCode::RegionAlreadyExists as u32 => {
                 Some(StatusCode::RegionAlreadyExists)
             }

src/common/function/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 
 [dependencies]
 arc-swap = "1.0"
+build-data = "0.1"
 chrono-tz = "0.6"
 common-error.workspace = true
 common-macro.workspace = true

src/common/function/src/function_registry.rs

@@ -18,11 +18,13 @@ use std::sync::{Arc, RwLock};
 
 use once_cell::sync::Lazy;
 
+use crate::function::FunctionRef;
 use crate::scalars::aggregate::{AggregateFunctionMetaRef, AggregateFunctions};
-use crate::scalars::function::FunctionRef;
+use crate::scalars::date::DateFunction;
 use crate::scalars::math::MathFunction;
 use crate::scalars::numpy::NumpyFunction;
 use crate::scalars::timestamp::TimestampFunction;
+use crate::system::SystemFunction;
 
 #[derive(Default)]
 pub struct FunctionRegistry {
@@ -75,9 +77,10 @@ pub static FUNCTION_REGISTRY: Lazy<Arc<FunctionRegistry>> = Lazy::new(|| {
     MathFunction::register(&function_registry);
     NumpyFunction::register(&function_registry);
     TimestampFunction::register(&function_registry);
+    DateFunction::register(&function_registry);
 
     AggregateFunctions::register(&function_registry);
-
+    SystemFunction::register(&function_registry);
     Arc::new(function_registry)
 });
 

src/common/function/src/helper.rs

@@ -0,0 +1,29 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use datatypes::prelude::ConcreteDataType;
+
+/// Create a function signature with oneof signatures of interleaving two arguments.
+pub fn one_of_sigs2(args1: Vec<ConcreteDataType>, args2: Vec<ConcreteDataType>) -> Signature {
+    let mut sigs = Vec::with_capacity(args1.len() * args2.len());
+
+    for arg1 in &args1 {
+        for arg2 in &args2 {
+            sigs.push(TypeSignature::Exact(vec![arg1.clone(), arg2.clone()]));
+        }
+    }
+
+    Signature::one_of(sigs, Volatility::Immutable)
+}

src/common/function/src/lib.rs

@@ -13,3 +13,8 @@
 // limitations under the License.
 
 pub mod scalars;
+pub mod system;
+
+pub mod function;
+pub mod function_registry;
+pub mod helper;

src/common/function/src/scalars.rs

@@ -13,15 +13,11 @@
 // limitations under the License.
 
 pub mod aggregate;
+pub(crate) mod date;
 pub mod expression;
-pub mod function;
-pub mod function_registry;
 pub mod math;
 pub mod numpy;
 #[cfg(test)]
 pub(crate) mod test;
-mod timestamp;
+pub(crate) mod timestamp;
 pub mod udf;
-
-pub use function::{Function, FunctionRef};
-pub use function_registry::{FunctionRegistry, FUNCTION_REGISTRY};

src/common/function/src/scalars/aggregate.rs

@@ -33,7 +33,7 @@ pub use polyval::PolyvalAccumulatorCreator;
 pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;
 
-use crate::scalars::FunctionRegistry;
+use crate::function_registry::FunctionRegistry;
 
 /// A function creates `AggregateFunctionCreator`.
 /// "Aggregator" *is* AggregatorFunction. Since the later one is long, we named an short alias for it.

src/common/function/src/scalars/date.rs

@@ -0,0 +1,31 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+mod date_add;
+mod date_sub;
+
+use date_add::DateAddFunction;
+use date_sub::DateSubFunction;
+
+use crate::function_registry::FunctionRegistry;
+
+pub(crate) struct DateFunction;
+
+impl DateFunction {
+    pub fn register(registry: &FunctionRegistry) {
+        registry.register(Arc::new(DateAddFunction));
+        registry.register(Arc::new(DateSubFunction));
+    }
+}

src/common/function/src/scalars/date/date_add.rs

@@ -0,0 +1,278 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt;
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datatypes::data_type::DataType;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::value::ValueRef;
+use datatypes::vectors::VectorRef;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::helper;
+
+/// A function adds an interval value to Timestamp, Date or DateTime, and return the result.
+#[derive(Clone, Debug, Default)]
+pub struct DateAddFunction;
+
+const NAME: &str = "date_add";
+
+impl Function for DateAddFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(input_types[0].clone())
+    }
+
+    fn signature(&self) -> Signature {
+        helper::one_of_sigs2(
+            vec![
+                ConcreteDataType::date_datatype(),
+                ConcreteDataType::datetime_datatype(),
+                ConcreteDataType::timestamp_second_datatype(),
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                ConcreteDataType::timestamp_microsecond_datatype(),
+                ConcreteDataType::timestamp_nanosecond_datatype(),
+            ],
+            vec![
+                ConcreteDataType::interval_month_day_nano_datatype(),
+                ConcreteDataType::interval_year_month_datatype(),
+                ConcreteDataType::interval_day_time_datatype(),
+            ],
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 2, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let left = &columns[0];
+        let right = &columns[1];
+
+        let size = left.len();
+        let left_datatype = columns[0].data_type();
+        match left_datatype {
+            ConcreteDataType::Timestamp(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let ts = left.get(i).as_timestamp();
+                    let interval = right.get(i).as_interval();
+
+                    let new_ts = match (ts, interval) {
+                        (Some(ts), Some(interval)) => ts.add_interval(interval),
+                        _ => ts,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_ts));
+                }
+
+                Ok(result.to_vector())
+            }
+            ConcreteDataType::Date(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let date = left.get(i).as_date();
+                    let interval = right.get(i).as_interval();
+                    let new_date = match (date, interval) {
+                        (Some(date), Some(interval)) => date.add_interval(interval),
+                        _ => date,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_date));
+                }
+
+                Ok(result.to_vector())
+            }
+            ConcreteDataType::DateTime(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let datetime = left.get(i).as_datetime();
+                    let interval = right.get(i).as_interval();
+                    let new_datetime = match (datetime, interval) {
+                        (Some(datetime), Some(interval)) => datetime.add_interval(interval),
+                        _ => datetime,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_datetime));
+                }
+
+                Ok(result.to_vector())
+            }
+            _ => UnsupportedInputDataTypeSnafu {
+                function: NAME,
+                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+            }
+            .fail(),
+        }
+    }
+}
+
+impl fmt::Display for DateAddFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "DATE_ADD")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::{TypeSignature, Volatility};
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::value::Value;
+    use datatypes::vectors::{
+        DateTimeVector, DateVector, IntervalDayTimeVector, IntervalYearMonthVector,
+        TimestampSecondVector,
+    };
+
+    use super::{DateAddFunction, *};
+
+    #[test]
+    fn test_date_add_misc() {
+        let f = DateAddFunction;
+        assert_eq!("date_add", f.name());
+        assert_eq!(
+            ConcreteDataType::timestamp_microsecond_datatype(),
+            f.return_type(&[ConcreteDataType::timestamp_microsecond_datatype()])
+                .unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::timestamp_second_datatype(),
+            f.return_type(&[ConcreteDataType::timestamp_second_datatype()])
+                .unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::date_datatype(),
+            f.return_type(&[ConcreteDataType::date_datatype()]).unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::datetime_datatype(),
+            f.return_type(&[ConcreteDataType::datetime_datatype()])
+                .unwrap()
+        );
+        assert!(matches!(f.signature(),
+                         Signature {
+                             type_signature: TypeSignature::OneOf(sigs),
+                             volatility: Volatility::Immutable
+                         } if  sigs.len() == 18));
+    }
+
+    #[test]
+    fn test_timestamp_date_add() {
+        let f = DateAddFunction;
+
+        let times = vec![Some(123), None, Some(42), None];
+        // Intervals in milliseconds
+        let intervals = vec![1000, 2000, 3000, 1000];
+        let results = [Some(124), None, Some(45), None];
+
+        let time_vector = TimestampSecondVector::from(times.clone());
+        let interval_vector = IntervalDayTimeVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(time_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in times.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Timestamp(ts) => {
+                    assert_eq!(ts.value(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_date_date_add() {
+        let f = DateAddFunction;
+
+        let dates = vec![Some(123), None, Some(42), None];
+        // Intervals in months
+        let intervals = vec![1, 2, 3, 1];
+        let results = [Some(154), None, Some(131), None];
+
+        let date_vector = DateVector::from(dates.clone());
+        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in dates.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Date(date) => {
+                    assert_eq!(date.val(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_datetime_date_add() {
+        let f = DateAddFunction;
+
+        let dates = vec![Some(123), None, Some(42), None];
+        // Intervals in months
+        let intervals = vec![1, 2, 3, 1];
+        let results = [Some(2678400123), None, Some(7776000042), None];
+
+        let date_vector = DateTimeVector::from(dates.clone());
+        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in dates.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::DateTime(date) => {
+                    assert_eq!(date.val(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+}

src/common/function/src/scalars/date/date_sub.rs

@@ -0,0 +1,291 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt;
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datatypes::data_type::DataType;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::value::ValueRef;
+use datatypes::vectors::VectorRef;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::helper;
+
+/// A function subtracts an interval value to Timestamp, Date or DateTime, and return the result.
+#[derive(Clone, Debug, Default)]
+pub struct DateSubFunction;
+
+const NAME: &str = "date_sub";
+
+impl Function for DateSubFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(input_types[0].clone())
+    }
+
+    fn signature(&self) -> Signature {
+        helper::one_of_sigs2(
+            vec![
+                ConcreteDataType::date_datatype(),
+                ConcreteDataType::datetime_datatype(),
+                ConcreteDataType::timestamp_second_datatype(),
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                ConcreteDataType::timestamp_microsecond_datatype(),
+                ConcreteDataType::timestamp_nanosecond_datatype(),
+            ],
+            vec![
+                ConcreteDataType::interval_month_day_nano_datatype(),
+                ConcreteDataType::interval_year_month_datatype(),
+                ConcreteDataType::interval_day_time_datatype(),
+            ],
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 2, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let left = &columns[0];
+        let right = &columns[1];
+
+        let size = left.len();
+        let left_datatype = columns[0].data_type();
+
+        match left_datatype {
+            ConcreteDataType::Timestamp(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let ts = left.get(i).as_timestamp();
+                    let interval = right.get(i).as_interval();
+
+                    let new_ts = match (ts, interval) {
+                        (Some(ts), Some(interval)) => ts.sub_interval(interval),
+                        _ => ts,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_ts));
+                }
+
+                Ok(result.to_vector())
+            }
+            ConcreteDataType::Date(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let date = left.get(i).as_date();
+                    let interval = right.get(i).as_interval();
+                    let new_date = match (date, interval) {
+                        (Some(date), Some(interval)) => date.sub_interval(interval),
+                        _ => date,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_date));
+                }
+
+                Ok(result.to_vector())
+            }
+            ConcreteDataType::DateTime(_) => {
+                let mut result = left_datatype.create_mutable_vector(size);
+                for i in 0..size {
+                    let datetime = left.get(i).as_datetime();
+                    let interval = right.get(i).as_interval();
+                    let new_datetime = match (datetime, interval) {
+                        (Some(datetime), Some(interval)) => datetime.sub_interval(interval),
+                        _ => datetime,
+                    };
+
+                    result.push_value_ref(ValueRef::from(new_datetime));
+                }
+
+                Ok(result.to_vector())
+            }
+            _ => UnsupportedInputDataTypeSnafu {
+                function: NAME,
+                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+            }
+            .fail(),
+        }
+    }
+}
+
+impl fmt::Display for DateSubFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "DATE_SUB")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::{TypeSignature, Volatility};
+    use datatypes::prelude::ConcreteDataType;
+    use datatypes::value::Value;
+    use datatypes::vectors::{
+        DateTimeVector, DateVector, IntervalDayTimeVector, IntervalYearMonthVector,
+        TimestampSecondVector,
+    };
+
+    use super::{DateSubFunction, *};
+
+    #[test]
+    fn test_date_sub_misc() {
+        let f = DateSubFunction;
+        assert_eq!("date_sub", f.name());
+        assert_eq!(
+            ConcreteDataType::timestamp_microsecond_datatype(),
+            f.return_type(&[ConcreteDataType::timestamp_microsecond_datatype()])
+                .unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::timestamp_second_datatype(),
+            f.return_type(&[ConcreteDataType::timestamp_second_datatype()])
+                .unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::date_datatype(),
+            f.return_type(&[ConcreteDataType::date_datatype()]).unwrap()
+        );
+        assert_eq!(
+            ConcreteDataType::datetime_datatype(),
+            f.return_type(&[ConcreteDataType::datetime_datatype()])
+                .unwrap()
+        );
+        assert!(matches!(f.signature(),
+                         Signature {
+                             type_signature: TypeSignature::OneOf(sigs),
+                             volatility: Volatility::Immutable
+                         } if  sigs.len() == 18));
+    }
+
+    #[test]
+    fn test_timestamp_date_sub() {
+        let f = DateSubFunction;
+
+        let times = vec![Some(123), None, Some(42), None];
+        // Intervals in milliseconds
+        let intervals = vec![1000, 2000, 3000, 1000];
+        let results = [Some(122), None, Some(39), None];
+
+        let time_vector = TimestampSecondVector::from(times.clone());
+        let interval_vector = IntervalDayTimeVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(time_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in times.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Timestamp(ts) => {
+                    assert_eq!(ts.value(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_date_date_sub() {
+        let f = DateSubFunction;
+        let days_per_month = 30;
+
+        let dates = vec![
+            Some(123 * days_per_month),
+            None,
+            Some(42 * days_per_month),
+            None,
+        ];
+        // Intervals in months
+        let intervals = vec![1, 2, 3, 1];
+        let results = [Some(3659), None, Some(1168), None];
+
+        let date_vector = DateVector::from(dates.clone());
+        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in dates.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::Date(date) => {
+                    assert_eq!(date.val(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_datetime_date_sub() {
+        let f = DateSubFunction;
+        let millis_per_month = 3600 * 24 * 30 * 1000;
+
+        let dates = vec![
+            Some(123 * millis_per_month),
+            None,
+            Some(42 * millis_per_month),
+            None,
+        ];
+        // Intervals in months
+        let intervals = vec![1, 2, 3, 1];
+        let results = [Some(316137600000), None, Some(100915200000), None];
+
+        let date_vector = DateTimeVector::from(dates.clone());
+        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
+        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
+        let vector = f.eval(FunctionContext::default(), &args).unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, _t) in dates.iter().enumerate() {
+            let v = vector.get(i);
+            let result = results.get(i).unwrap();
+
+            if result.is_none() {
+                assert_eq!(Value::Null, v);
+                continue;
+            }
+            match v {
+                Value::DateTime(date) => {
+                    assert_eq!(date.val(), result.unwrap());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+}

src/common/function/src/scalars/math.rs

@@ -28,9 +28,8 @@ pub use pow::PowFunction;
 pub use rate::RateFunction;
 use snafu::ResultExt;
 
-use super::function::FunctionContext;
-use super::Function;
-use crate::scalars::function_registry::FunctionRegistry;
+use crate::function::{Function, FunctionContext};
+use crate::function_registry::FunctionRegistry;
 
 pub(crate) struct MathFunction;
 

src/common/function/src/scalars/math/pow.rs

@@ -25,8 +25,8 @@ use datatypes::with_match_primitive_type_id;
 use num::traits::Pow;
 use num_traits::AsPrimitive;
 
+use crate::function::{Function, FunctionContext};
 use crate::scalars::expression::{scalar_binary_op, EvalContext};
-use crate::scalars::function::{Function, FunctionContext};
 
 #[derive(Clone, Debug, Default)]
 pub struct PowFunction;
@@ -83,6 +83,7 @@ mod tests {
     use datatypes::vectors::{Float32Vector, Int8Vector};
 
     use super::*;
+    use crate::function::FunctionContext;
     #[test]
     fn test_pow_function() {
         let pow = PowFunction;

src/common/function/src/scalars/math/rate.rs

@@ -23,7 +23,7 @@ use datatypes::prelude::*;
 use datatypes::vectors::{Helper, VectorRef};
 use snafu::ResultExt;
 
-use crate::scalars::function::{Function, FunctionContext};
+use crate::function::{Function, FunctionContext};
 
 /// generates rates from a sequence of adjacent data points.
 #[derive(Clone, Debug, Default)]

src/common/function/src/scalars/numpy.rs

@@ -19,7 +19,7 @@ use std::sync::Arc;
 
 use clip::ClipFunction;
 
-use crate::scalars::function_registry::FunctionRegistry;
+use crate::function_registry::FunctionRegistry;
 
 pub(crate) struct NumpyFunction;
 

src/common/function/src/scalars/numpy/clip.rs

@@ -24,8 +24,8 @@ use datatypes::prelude::*;
 use datatypes::vectors::PrimitiveVector;
 use paste::paste;
 
+use crate::function::{Function, FunctionContext};
 use crate::scalars::expression::{scalar_binary_op, EvalContext};
-use crate::scalars::function::{Function, FunctionContext};
 
 /// numpy.clip function, <https://numpy.org/doc/stable/reference/generated/numpy.clip.html>
 #[derive(Clone, Debug, Default)]

src/common/function/src/scalars/test.rs

@@ -20,8 +20,8 @@ use common_query::prelude::{Signature, Volatility};
 use datatypes::data_type::ConcreteDataType;
 use datatypes::prelude::VectorRef;
 
+use crate::function::{Function, FunctionContext};
 use crate::scalars::expression::{scalar_binary_op, EvalContext};
-use crate::scalars::function::{Function, FunctionContext};
 
 #[derive(Clone, Default)]
 pub(crate) struct TestAndFunction;

src/common/function/src/scalars/timestamp.rs

@@ -19,7 +19,7 @@ mod to_unixtime;
 use greatest::GreatestFunction;
 use to_unixtime::ToUnixtimeFunction;
 
-use crate::scalars::function_registry::FunctionRegistry;
+use crate::function_registry::FunctionRegistry;
 
 pub(crate) struct TimestampFunction;
 

src/common/function/src/scalars/timestamp/greatest.rs

@@ -27,7 +27,7 @@ use datatypes::prelude::ConcreteDataType;
 use datatypes::vectors::{Helper, VectorRef};
 use snafu::{ensure, ResultExt};
 
-use crate::scalars::function::{Function, FunctionContext};
+use crate::function::{Function, FunctionContext};
 
 #[derive(Clone, Debug, Default)]
 pub struct GreatestFunction;
@@ -113,10 +113,7 @@ mod tests {
     use datatypes::value::Value;
     use datatypes::vectors::{DateVector, StringVector, Vector};
 
-    use super::GreatestFunction;
-    use crate::scalars::function::FunctionContext;
-    use crate::scalars::Function;
-
+    use super::*;
     #[test]
     fn test_greatest_takes_string_vector() {
         let function = GreatestFunction;

src/common/function/src/scalars/timestamp/to_unixtime.rs

@@ -23,7 +23,7 @@ use datatypes::prelude::ConcreteDataType;
 use datatypes::vectors::{Int64Vector, VectorRef};
 use snafu::ensure;
 
-use crate::scalars::function::{Function, FunctionContext};
+use crate::function::{Function, FunctionContext};
 
 /// A function to convert the column into the unix timestamp in seconds.
 #[derive(Clone, Debug, Default)]
@@ -152,7 +152,6 @@ mod tests {
     };
 
     use super::{ToUnixtimeFunction, *};
-    use crate::scalars::Function;
 
     #[test]
     fn test_string_to_unixtime() {

src/common/function/src/scalars/udf.rs

@@ -23,7 +23,7 @@ use datatypes::prelude::*;
 use datatypes::vectors::Helper;
 use snafu::ResultExt;
 
-use crate::scalars::function::{FunctionContext, FunctionRef};
+use crate::function::{FunctionContext, FunctionRef};
 
 /// Create a ScalarUdf from function.
 pub fn create_udf(func: FunctionRef) -> ScalarUdf {
@@ -72,7 +72,7 @@ mod tests {
     use datatypes::vectors::{BooleanVector, ConstantVector};
 
     use super::*;
-    use crate::scalars::function::Function;
+    use crate::function::Function;
     use crate::scalars::test::TestAndFunction;
 
     #[test]

src/common/function/src/system.rs

@@ -0,0 +1,29 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod build;
+
+use std::sync::Arc;
+
+use build::BuildFunction;
+
+use crate::function_registry::FunctionRegistry;
+
+pub(crate) struct SystemFunction;
+
+impl SystemFunction {
+    pub fn register(registry: &FunctionRegistry) {
+        registry.register(Arc::new(BuildFunction));
+    }
+}

src/common/function/src/system/build.rs

@@ -0,0 +1,102 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt;
+use std::sync::Arc;
+
+use common_query::error::Result;
+use common_query::prelude::{Signature, Volatility};
+use datatypes::prelude::*;
+use datatypes::vectors::{StringVector, VectorRef};
+
+use crate::function::{Function, FunctionContext};
+
+const DEFAULT_VALUE: &str = "unknown";
+
+/// Generates build information  
+#[derive(Clone, Debug, Default)]
+pub struct BuildFunction;
+
+impl fmt::Display for BuildFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "BUILD")
+    }
+}
+
+impl Function for BuildFunction {
+    fn name(&self) -> &str {
+        "build"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::uniform(
+            0,
+            vec![ConcreteDataType::string_datatype()],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
+        let build_info = format!(
+            "branch: {}\ncommit: {}\ncommit short: {}\ndirty: {}\nversion: {}",
+            build_data::get_git_branch().unwrap_or_else(|_| DEFAULT_VALUE.to_string()),
+            build_data::get_git_commit().unwrap_or_else(|_| DEFAULT_VALUE.to_string()),
+            build_data::get_git_commit_short().unwrap_or_else(|_| DEFAULT_VALUE.to_string()),
+            build_data::get_git_dirty().map_or(DEFAULT_VALUE.to_string(), |v| v.to_string()),
+            env!("CARGO_PKG_VERSION")
+        );
+
+        let v = Arc::new(StringVector::from(vec![build_info]));
+        Ok(v)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::TypeSignature;
+
+    use super::*;
+    #[test]
+    fn test_build_function() {
+        let build = BuildFunction;
+        assert_eq!("build", build.name());
+        assert_eq!(
+            ConcreteDataType::string_datatype(),
+            build.return_type(&[]).unwrap()
+        );
+        assert!(matches!(build.signature(),
+                         Signature {
+                             type_signature: TypeSignature::Uniform(0, valid_types),
+                             volatility: Volatility::Immutable
+                         } if  valid_types == vec![ConcreteDataType::string_datatype()]
+        ));
+        let build_info = format!(
+            "branch: {}\ncommit: {}\ncommit short: {}\ndirty: {}\nversion: {}",
+            build_data::get_git_branch().unwrap_or_else(|_| DEFAULT_VALUE.to_string()),
+            build_data::get_git_commit().unwrap_or_else(|_| DEFAULT_VALUE.to_string()),
+            build_data::get_git_commit_short().unwrap_or_else(|_| DEFAULT_VALUE.to_string()),
+            build_data::get_git_dirty().map_or(DEFAULT_VALUE.to_string(), |v| v.to_string()),
+            env!("CARGO_PKG_VERSION")
+        );
+        let vector = build.eval(FunctionContext::default(), &[]).unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec![build_info]));
+        assert_eq!(expect, vector);
+    }
+}

src/common/grpc-expr/src/insert.rs

@@ -98,7 +98,11 @@ mod tests {
 
     use api::helper::ColumnDataTypeWrapper;
     use api::v1::column::Values;
-    use api::v1::{Column, ColumnDataType, IntervalMonthDayNano, SemanticType};
+    use api::v1::column_data_type_extension::TypeExt;
+    use api::v1::{
+        Column, ColumnDataType, ColumnDataTypeExtension, Decimal128, DecimalTypeExtension,
+        IntervalMonthDayNano, SemanticType,
+    };
     use common_base::BitVec;
     use common_catalog::consts::MITO_ENGINE;
     use common_time::interval::IntervalUnit;
@@ -160,7 +164,7 @@ mod tests {
 
         let column_defs = create_expr.column_defs;
         assert_eq!(column_defs[6].name, create_expr.time_index);
-        assert_eq!(7, column_defs.len());
+        assert_eq!(8, column_defs.len());
 
         assert_eq!(
             ConcreteDataType::string_datatype(),
@@ -266,6 +270,18 @@ mod tests {
                 .unwrap()
             )
         );
+
+        let decimal_column = column_defs.iter().find(|c| c.name == "decimals").unwrap();
+        assert_eq!(
+            ConcreteDataType::decimal128_datatype(38, 10),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    decimal_column.data_type,
+                    decimal_column.datatype_extension.clone(),
+                )
+                .unwrap()
+            )
+        );
     }
 
     #[test]
@@ -286,7 +302,7 @@ mod tests {
 
         let add_columns = find_new_columns(&schema, &insert_batch.0).unwrap().unwrap();
 
-        assert_eq!(5, add_columns.add_columns.len());
+        assert_eq!(6, add_columns.add_columns.len());
         let host_column = &add_columns.add_columns[0];
         assert_eq!(
             ConcreteDataType::string_datatype(),
@@ -347,6 +363,23 @@ mod tests {
                 .unwrap()
             )
         );
+
+        let decimal_column = &add_columns.add_columns[5];
+        assert_eq!(
+            ConcreteDataType::decimal128_datatype(38, 10),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    decimal_column.column_def.as_ref().unwrap().data_type,
+                    decimal_column
+                        .column_def
+                        .as_ref()
+                        .unwrap()
+                        .datatype_extension
+                        .clone()
+                )
+                .unwrap()
+            )
+        );
     }
 
     #[test]
@@ -466,6 +499,23 @@ mod tests {
             datatype: ColumnDataType::TimestampMillisecond as i32,
             ..Default::default()
         };
+        let decimal_vals = Values {
+            decimal128_values: vec![Decimal128 { hi: 0, lo: 123 }, Decimal128 { hi: 0, lo: 456 }],
+            ..Default::default()
+        };
+        let decimal_column = Column {
+            column_name: "decimals".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(decimal_vals),
+            null_mask: vec![0],
+            datatype: ColumnDataType::Decimal128 as i32,
+            datatype_extension: Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::DecimalType(DecimalTypeExtension {
+                    precision: 38,
+                    scale: 10,
+                })),
+            }),
+        };
 
         (
             vec![
@@ -476,6 +526,7 @@ mod tests {
                 interval_column,
                 duration_column,
                 ts_column,
+                decimal_column,
             ],
             row_count,
         )

src/common/grpc-expr/src/util.rs

@@ -15,7 +15,8 @@
 use std::collections::HashSet;
 
 use api::v1::{
-    AddColumn, AddColumns, Column, ColumnDef, ColumnSchema, CreateTableExpr, SemanticType,
+    AddColumn, AddColumns, Column, ColumnDataTypeExtension, ColumnDef, ColumnSchema,
+    CreateTableExpr, SemanticType,
 };
 use datatypes::schema::Schema;
 use snafu::{ensure, OptionExt};
@@ -30,6 +31,7 @@ pub struct ColumnExpr<'a> {
     pub column_name: &'a str,
     pub datatype: i32,
     pub semantic_type: i32,
+    pub datatype_extension: &'a Option<ColumnDataTypeExtension>,
 }
 
 impl<'a> ColumnExpr<'a> {
@@ -50,6 +52,7 @@ impl<'a> From<&'a Column> for ColumnExpr<'a> {
             column_name: &column.column_name,
             datatype: column.datatype,
             semantic_type: column.semantic_type,
+            datatype_extension: &column.datatype_extension,
         }
     }
 }
@@ -60,6 +63,7 @@ impl<'a> From<&'a ColumnSchema> for ColumnExpr<'a> {
             column_name: &schema.column_name,
             datatype: schema.datatype,
             semantic_type: schema.semantic_type,
+            datatype_extension: &schema.datatype_extension,
         }
     }
 }
@@ -94,6 +98,7 @@ pub fn build_create_table_expr(
         column_name,
         datatype,
         semantic_type,
+        datatype_extension,
     } in column_exprs
     {
         let mut is_nullable = true;
@@ -121,7 +126,7 @@ pub fn build_create_table_expr(
             default_constraint: vec![],
             semantic_type,
             comment: String::new(),
-            ..Default::default()
+            datatype_extension: datatype_extension.clone(),
         };
         column_defs.push(column_def);
     }
@@ -162,7 +167,7 @@ pub fn extract_new_columns(
                 default_constraint: vec![],
                 semantic_type: expr.semantic_type,
                 comment: String::new(),
-                ..Default::default()
+                datatype_extension: expr.datatype_extension.clone(),
             });
             AddColumn {
                 column_def,

src/common/grpc/Cargo.toml

@@ -16,7 +16,7 @@ common-recordbatch.workspace = true
 common-runtime.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
-dashmap = "5.4"
+dashmap.workspace = true
 datafusion.workspace = true
 datatypes.workspace = true
 flatbuffers = "23.1"

src/common/meta/Cargo.toml

@@ -13,8 +13,9 @@ async-recursion = "1.0"
 async-stream.workspace = true
 async-trait.workspace = true
 base64.workspace = true
-bytes = "1.4"
+bytes.workspace = true
 common-catalog.workspace = true
+common-config.workspace = true
 common-error.workspace = true
 common-grpc-expr.workspace = true
 common-macro.workspace = true
@@ -24,20 +25,25 @@ common-runtime.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
 datatypes.workspace = true
+derive_builder.workspace = true
 etcd-client.workspace = true
 futures.workspace = true
 humantime-serde.workspace = true
 lazy_static.workspace = true
 prometheus.workspace = true
 prost.workspace = true
+rand.workspace = true
 regex.workspace = true
+rskafka.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+serde_with = "3"
 snafu.workspace = true
 store-api.workspace = true
 strum.workspace = true
 table.workspace = true
 tokio.workspace = true
+toml.workspace = true
 tonic.workspace = true
 
 [dev-dependencies]

src/common/meta/src/ddl.rs

@@ -12,19 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
-use api::v1::meta::Partition;
 use common_telemetry::tracing_context::W3cTrace;
-use store_api::storage::TableId;
-use table::metadata::RawTableInfo;
+use store_api::storage::{RegionNumber, TableId};
 
 use crate::cache_invalidator::CacheInvalidatorRef;
 use crate::datanode_manager::DatanodeManagerRef;
 use crate::error::Result;
+use crate::key::table_route::TableRouteValue;
 use crate::key::TableMetadataManagerRef;
-use crate::rpc::ddl::{SubmitDdlTaskRequest, SubmitDdlTaskResponse};
-use crate::rpc::router::RegionRoute;
+use crate::region_keeper::MemoryRegionKeeperRef;
+use crate::rpc::ddl::{CreateTableTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse};
 
 pub mod alter_table;
 pub mod create_table;
@@ -53,14 +53,24 @@ pub struct TableMetadataAllocatorContext {
     pub cluster_id: u64,
 }
 
+/// Metadata allocated to a table.
+pub struct TableMetadata {
+    /// Table id.
+    pub table_id: TableId,
+    /// Route information for each region of the table.
+    pub table_route: TableRouteValue,
+    /// The encoded wal options for regions of the table.
+    // If a region does not have an associated wal options, no key for the region would be found in the map.
+    pub region_wal_options: HashMap<RegionNumber, String>,
+}
+
 #[async_trait::async_trait]
 pub trait TableMetadataAllocator: Send + Sync {
     async fn create(
         &self,
         ctx: &TableMetadataAllocatorContext,
-        table_info: &mut RawTableInfo,
-        partitions: &[Partition],
-    ) -> Result<(TableId, Vec<RegionRoute>)>;
+        task: &CreateTableTask,
+    ) -> Result<TableMetadata>;
 }
 
 pub type TableMetadataAllocatorRef = Arc<dyn TableMetadataAllocator>;
@@ -70,4 +80,5 @@ pub struct DdlContext {
     pub datanode_manager: DatanodeManagerRef,
     pub cache_invalidator: CacheInvalidatorRef,
     pub table_metadata_manager: TableMetadataManagerRef,
+    pub memory_region_keeper: MemoryRegionKeeperRef,
 }

src/common/meta/src/ddl/alter_table.rs

@@ -45,7 +45,6 @@ use crate::error::{
 };
 use crate::key::table_info::TableInfoValue;
 use crate::key::table_name::TableNameKey;
-use crate::key::table_route::TableRouteValue;
 use crate::key::DeserializedValueWithBytes;
 use crate::metrics;
 use crate::rpc::ddl::AlterTableTask;
@@ -183,25 +182,23 @@ impl AlterTableProcedure {
 
     pub async fn submit_alter_region_requests(&mut self) -> Result<Status> {
         let table_id = self.data.table_id();
-        let table_ref = self.data.table_ref();
 
-        let TableRouteValue { region_routes, .. } = self
+        let table_route = self
             .context
             .table_metadata_manager
             .table_route_manager()
             .get(table_id)
             .await?
-            .with_context(|| TableRouteNotFoundSnafu {
-                table_name: table_ref.to_string(),
-            })?
+            .context(TableRouteNotFoundSnafu { table_id })?
             .into_inner();
+        let region_routes = table_route.region_routes();
 
-        let leaders = find_leaders(&region_routes);
+        let leaders = find_leaders(region_routes);
         let mut alter_region_tasks = Vec::with_capacity(leaders.len());
 
         for datanode in leaders {
             let requester = self.context.datanode_manager.datanode(&datanode).await;
-            let regions = find_leader_regions(&region_routes, &datanode);
+            let regions = find_leader_regions(region_routes, &datanode);
 
             for region in regions {
                 let region_id = RegionId::new(table_id, region);

src/common/meta/src/ddl/create_table.rs

@@ -12,31 +12,42 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
+
 use api::v1::region::region_request::Body as PbRegionRequest;
 use api::v1::region::{
     CreateRequest as PbCreateRegionRequest, RegionColumnDef, RegionRequest, RegionRequestHeader,
 };
 use api::v1::{ColumnDef, SemanticType};
 use async_trait::async_trait;
-use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
+use common_config::WAL_OPTIONS_KEY;
+use common_error::ext::BoxedError;
+use common_procedure::error::{
+    ExternalSnafu, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu,
+};
 use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status};
 use common_telemetry::info;
 use common_telemetry::tracing_context::TracingContext;
 use futures::future::join_all;
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
-use store_api::storage::RegionId;
+use store_api::metric_engine_consts::LOGICAL_TABLE_METADATA_KEY;
+use store_api::storage::{RegionId, RegionNumber};
 use strum::AsRefStr;
 use table::engine::TableReference;
 use table::metadata::{RawTableInfo, TableId};
 
 use crate::ddl::utils::{handle_operate_region_error, handle_retry_error, region_storage_path};
 use crate::ddl::DdlContext;
-use crate::error::{self, Result};
+use crate::error::{self, Result, TableRouteNotFoundSnafu};
 use crate::key::table_name::TableNameKey;
+use crate::key::table_route::TableRouteValue;
 use crate::metrics;
+use crate::region_keeper::OperatingRegionGuard;
 use crate::rpc::ddl::CreateTableTask;
-use crate::rpc::router::{find_leader_regions, find_leaders, RegionRoute};
+use crate::rpc::router::{
+    find_leader_regions, find_leaders, operating_leader_regions, RegionRoute,
+};
 
 pub struct CreateTableProcedure {
     pub context: DdlContext,
@@ -49,21 +60,32 @@ impl CreateTableProcedure {
     pub fn new(
         cluster_id: u64,
         task: CreateTableTask,
-        region_routes: Vec<RegionRoute>,
+        table_route: TableRouteValue,
+        region_wal_options: HashMap<RegionNumber, String>,
         context: DdlContext,
     ) -> Self {
         Self {
             context,
-            creator: TableCreator::new(cluster_id, task, region_routes),
+            creator: TableCreator::new(cluster_id, task, table_route, region_wal_options),
         }
     }
 
     pub fn from_json(json: &str, context: DdlContext) -> ProcedureResult<Self> {
         let data = serde_json::from_str(json).context(FromJsonSnafu)?;
-        Ok(CreateTableProcedure {
-            context,
-            creator: TableCreator { data },
-        })
+
+        let mut creator = TableCreator {
+            data,
+            opening_regions: vec![],
+        };
+
+        if let TableRouteValue::Physical(x) = &creator.data.table_route {
+            creator.opening_regions = creator
+                .register_opening_regions(&context, &x.region_routes)
+                .map_err(BoxedError::new)
+                .context(ExternalSnafu)?;
+        }
+
+        Ok(CreateTableProcedure { context, creator })
     }
 
     pub fn table_info(&self) -> &RawTableInfo {
@@ -74,8 +96,8 @@ impl CreateTableProcedure {
         self.table_info().ident.table_id
     }
 
-    pub fn region_routes(&self) -> &Vec<RegionRoute> {
-        &self.creator.data.region_routes
+    pub fn region_wal_options(&self) -> &HashMap<RegionNumber, String> {
+        &self.creator.data.region_wal_options
     }
 
     /// Checks whether the table exists.
@@ -108,7 +130,10 @@ impl CreateTableProcedure {
         Ok(Status::executing(true))
     }
 
-    pub fn create_region_request_template(&self) -> Result<PbCreateRegionRequest> {
+    pub fn new_region_request_builder(
+        &self,
+        physical_table_id: Option<TableId>,
+    ) -> Result<CreateRequestBuilder> {
         let create_table_expr = &self.creator.data.task.create_table;
 
         let column_defs = create_table_expr
@@ -158,27 +183,70 @@ impl CreateTableProcedure {
             })
             .collect::<Result<_>>()?;
 
-        Ok(PbCreateRegionRequest {
+        let template = PbCreateRegionRequest {
             region_id: 0,
             engine: create_table_expr.engine.to_string(),
             column_defs,
             primary_key,
             path: String::new(),
             options: create_table_expr.table_options.clone(),
+        };
+
+        Ok(CreateRequestBuilder {
+            template,
+            physical_table_id,
         })
     }
 
     pub async fn on_datanode_create_regions(&mut self) -> Result<Status> {
+        match &self.creator.data.table_route {
+            TableRouteValue::Physical(x) => {
+                let region_routes = x.region_routes.clone();
+                let request_builder = self.new_region_request_builder(None)?;
+                self.create_regions(&region_routes, request_builder).await
+            }
+            TableRouteValue::Logical(x) => {
+                let physical_table_id = x.physical_table_id();
+
+                let physical_table_route = self
+                    .context
+                    .table_metadata_manager
+                    .table_route_manager()
+                    .get(physical_table_id)
+                    .await?
+                    .context(TableRouteNotFoundSnafu {
+                        table_id: physical_table_id,
+                    })?;
+                let region_routes = physical_table_route.region_routes();
+
+                let request_builder = self.new_region_request_builder(Some(physical_table_id))?;
+
+                self.create_regions(region_routes, request_builder).await
+            }
+        }
+    }
+
+    async fn create_regions(
+        &mut self,
+        region_routes: &[RegionRoute],
+        request_builder: CreateRequestBuilder,
+    ) -> Result<Status> {
+        // Registers opening regions
+        let guards = self
+            .creator
+            .register_opening_regions(&self.context, region_routes)?;
+        if !guards.is_empty() {
+            self.creator.opening_regions = guards;
+        }
+
         let create_table_data = &self.creator.data;
-        let region_routes = &create_table_data.region_routes;
+        let region_wal_options = &create_table_data.region_wal_options;
 
         let create_table_expr = &create_table_data.task.create_table;
         let catalog = &create_table_expr.catalog_name;
         let schema = &create_table_expr.schema_name;
         let storage_path = region_storage_path(catalog, schema);
 
-        let request_template = self.create_region_request_template()?;
-
         let leaders = find_leaders(region_routes);
         let mut create_region_tasks = Vec::with_capacity(leaders.len());
 
@@ -186,17 +254,15 @@ impl CreateTableProcedure {
             let requester = self.context.datanode_manager.datanode(&datanode).await;
 
             let regions = find_leader_regions(region_routes, &datanode);
-            let requests = regions
-                .iter()
-                .map(|region_number| {
-                    let region_id = RegionId::new(self.table_id(), *region_number);
+            let mut requests = Vec::with_capacity(regions.len());
+            for region_number in regions {
+                let region_id = RegionId::new(self.table_id(), region_number);
+                let create_region_request = request_builder
+                    .build_one(region_id, storage_path.clone(), region_wal_options)
+                    .await?;
 
-                    let mut create_region_request = request_template.clone();
-                    create_region_request.region_id = region_id.as_u64();
-                    create_region_request.path = storage_path.clone();
-                    PbRegionRequest::Create(create_region_request)
-                })
-                .collect::<Vec<_>>();
+                requests.push(PbRegionRequest::Create(create_region_request));
+            }
 
             for request in requests {
                 let request = RegionRequest {
@@ -226,7 +292,9 @@ impl CreateTableProcedure {
 
         self.creator.data.state = CreateTableState::CreateMetadata;
 
-        Ok(Status::executing(true))
+        // Ensures the procedures after the crash start from the `DatanodeCreateRegions` stage.
+        // TODO(weny): Add more tests.
+        Ok(Status::executing(false))
     }
 
     async fn on_create_metadata(&self) -> Result<Status> {
@@ -234,9 +302,13 @@ impl CreateTableProcedure {
         let manager = &self.context.table_metadata_manager;
 
         let raw_table_info = self.table_info().clone();
-        let region_routes = self.region_routes().clone();
+        let region_wal_options = self.region_wal_options().clone();
         manager
-            .create_table_metadata(raw_table_info, region_routes)
+            .create_table_metadata(
+                raw_table_info,
+                self.creator.data.table_route.clone(),
+                region_wal_options,
+            )
             .await?;
         info!("Created table metadata for table {table_id}");
 
@@ -282,20 +354,57 @@ impl Procedure for CreateTableProcedure {
 }
 
 pub struct TableCreator {
+    /// The serializable data.
     pub data: CreateTableData,
+    /// The guards of opening.
+    pub opening_regions: Vec<OperatingRegionGuard>,
 }
 
 impl TableCreator {
-    pub fn new(cluster_id: u64, task: CreateTableTask, region_routes: Vec<RegionRoute>) -> Self {
+    pub fn new(
+        cluster_id: u64,
+        task: CreateTableTask,
+        table_route: TableRouteValue,
+        region_wal_options: HashMap<RegionNumber, String>,
+    ) -> Self {
         Self {
             data: CreateTableData {
                 state: CreateTableState::Prepare,
                 cluster_id,
                 task,
-                region_routes,
+                table_route,
+                region_wal_options,
             },
+            opening_regions: vec![],
         }
     }
+
+    /// Registers and returns the guards of the opening region if they don't exist.
+    fn register_opening_regions(
+        &self,
+        context: &DdlContext,
+        region_routes: &[RegionRoute],
+    ) -> Result<Vec<OperatingRegionGuard>> {
+        let opening_regions = operating_leader_regions(region_routes);
+
+        if self.opening_regions.len() == opening_regions.len() {
+            return Ok(vec![]);
+        }
+
+        let mut opening_region_guards = Vec::with_capacity(opening_regions.len());
+
+        for (region_id, datanode_id) in opening_regions {
+            let guard = context
+                .memory_region_keeper
+                .register(datanode_id, region_id)
+                .context(error::RegionOperatingRaceSnafu {
+                    region_id,
+                    peer_id: datanode_id,
+                })?;
+            opening_region_guards.push(guard);
+        }
+        Ok(opening_region_guards)
+    }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, AsRefStr)]
@@ -312,7 +421,8 @@ pub enum CreateTableState {
 pub struct CreateTableData {
     pub state: CreateTableState,
     pub task: CreateTableTask,
-    pub region_routes: Vec<RegionRoute>,
+    table_route: TableRouteValue,
+    pub region_wal_options: HashMap<RegionNumber, String>,
     pub cluster_id: u64,
 }
 
@@ -321,3 +431,50 @@ impl CreateTableData {
         self.task.table_ref()
     }
 }
+
+/// Builder for [PbCreateRegionRequest].
+pub struct CreateRequestBuilder {
+    template: PbCreateRegionRequest,
+    /// Optional. Only for metric engine.
+    physical_table_id: Option<TableId>,
+}
+
+impl CreateRequestBuilder {
+    pub fn template(&self) -> &PbCreateRegionRequest {
+        &self.template
+    }
+
+    async fn build_one(
+        &self,
+        region_id: RegionId,
+        storage_path: String,
+        region_wal_options: &HashMap<RegionNumber, String>,
+    ) -> Result<PbCreateRegionRequest> {
+        let mut request = self.template.clone();
+
+        request.region_id = region_id.as_u64();
+        request.path = storage_path;
+        // Stores the encoded wal options into the request options.
+        region_wal_options
+            .get(&region_id.region_number())
+            .and_then(|wal_options| {
+                request
+                    .options
+                    .insert(WAL_OPTIONS_KEY.to_string(), wal_options.clone())
+            });
+
+        if let Some(physical_table_id) = self.physical_table_id {
+            // Logical table has the same region numbers with physical table, and they have a one-to-one mapping.
+            // For example, region 0 of logical table must resides with region 0 of physical table. So here we can
+            // simply concat the physical table id and the logical region number to get the physical region id.
+            let physical_region_id = RegionId::new(physical_table_id, region_id.region_number());
+
+            request.options.insert(
+                LOGICAL_TABLE_METADATA_KEY.to_string(),
+                physical_region_id.as_u64().to_string(),
+            );
+        }
+
+        Ok(request)
+    }
+}

src/common/meta/src/ddl/drop_table.rs

@@ -26,7 +26,7 @@ use common_telemetry::tracing_context::TracingContext;
 use common_telemetry::{debug, info};
 use futures::future::join_all;
 use serde::{Deserialize, Serialize};
-use snafu::{ensure, ResultExt};
+use snafu::{ensure, OptionExt, ResultExt};
 use store_api::storage::RegionId;
 use strum::AsRefStr;
 use table::engine::TableReference;
@@ -42,12 +42,19 @@ use crate::key::table_name::TableNameKey;
 use crate::key::table_route::TableRouteValue;
 use crate::key::DeserializedValueWithBytes;
 use crate::metrics;
+use crate::region_keeper::OperatingRegionGuard;
 use crate::rpc::ddl::DropTableTask;
-use crate::rpc::router::{find_leader_regions, find_leaders, RegionRoute};
+use crate::rpc::router::{
+    find_leader_regions, find_leaders, operating_leader_regions, RegionRoute,
+};
 
 pub struct DropTableProcedure {
+    /// The context of procedure runtime.
     pub context: DdlContext,
+    /// The serializable data.
     pub data: DropTableData,
+    /// The guards of opening regions.
+    pub dropping_regions: Vec<OperatingRegionGuard>,
 }
 
 #[allow(dead_code)]
@@ -64,12 +71,17 @@ impl DropTableProcedure {
         Self {
             context,
             data: DropTableData::new(cluster_id, task, table_route_value, table_info_value),
+            dropping_regions: vec![],
         }
     }
 
     pub fn from_json(json: &str, context: DdlContext) -> ProcedureResult<Self> {
         let data = serde_json::from_str(json).context(FromJsonSnafu)?;
-        Ok(Self { context, data })
+        Ok(Self {
+            context,
+            data,
+            dropping_regions: vec![],
+        })
     }
 
     async fn on_prepare(&mut self) -> Result<Status> {
@@ -86,6 +98,10 @@ impl DropTableProcedure {
             ))
             .await?;
 
+        if !exist && self.data.task.drop_if_exists {
+            return Ok(Status::Done);
+        }
+
         ensure!(
             exist,
             error::TableNotFoundSnafu {
@@ -98,8 +114,42 @@ impl DropTableProcedure {
         Ok(Status::executing(true))
     }
 
+    /// Register dropping regions if doesn't exist.
+    fn register_dropping_regions(&mut self) -> Result<()> {
+        let region_routes = self.data.region_routes();
+
+        let dropping_regions = operating_leader_regions(region_routes);
+
+        if self.dropping_regions.len() == dropping_regions.len() {
+            return Ok(());
+        }
+
+        let mut dropping_region_guards = Vec::with_capacity(dropping_regions.len());
+
+        for (region_id, datanode_id) in dropping_regions {
+            let guard = self
+                .context
+                .memory_region_keeper
+                .register(datanode_id, region_id)
+                .context(error::RegionOperatingRaceSnafu {
+                    region_id,
+                    peer_id: datanode_id,
+                })?;
+            dropping_region_guards.push(guard);
+        }
+
+        self.dropping_regions = dropping_region_guards;
+        Ok(())
+    }
+
     /// Removes the table metadata.
     async fn on_remove_metadata(&mut self) -> Result<Status> {
+        // NOTES: If the meta server is crashed after the `RemoveMetadata`,
+        // Corresponding regions of this table on the Datanode will be closed automatically.
+        // Then any future dropping operation will fail.
+
+        // TODO(weny): Considers introducing a RegionStatus to indicate the region is dropping.
+
         let table_metadata_manager = &self.context.table_metadata_manager;
         let table_info_value = &self.data.table_info_value;
         let table_route_value = &self.data.table_route_value;
@@ -257,7 +307,7 @@ impl DropTableData {
     }
 
     fn region_routes(&self) -> &Vec<RegionRoute> {
-        &self.table_route_value.region_routes
+        self.table_route_value.region_routes()
     }
 
     fn table_info(&self) -> &RawTableInfo {