fix: fix tests failed on windows (#3155)

* fix: fix tests failed on windows

* feat: add comments

* Update src/object-store/src/util.rs

Co-authored-by: Yingwen <realevenyag@gmail.com>

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>

.env.example

@@ -19,3 +19,5 @@ GT_GCS_BUCKET = GCS bucket
 GT_GCS_SCOPE  = GCS scope
 GT_GCS_CREDENTIAL_PATH = GCS credential path 
 GT_GCS_ENDPOINT = GCS end point
+# Settings for kafka wal test
+GT_KAFKA_ENDPOINTS = localhost:9092

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -21,6 +21,7 @@ body:
         - Locking issue
         - Performance issue
         - Unexpected error
+        - User Experience
         - Other
     validations:
       required: true
@@ -33,9 +34,14 @@ body:
       multiple: true
       options:
         - Standalone mode
+        - Distributed Cluster
+        - Storage Engine
+        - Query Engine
+        - Table Engine
+        - Write Protocols
+        - MetaSrv
         - Frontend
         - Datanode
-        - Meta
         - Other
     validations:
       required: true
@@ -77,6 +83,17 @@ body:
     validations:
       required: true
 
+  - type: input
+    id: greptimedb
+    attributes:
+      label: What version of GreptimeDB did you use?
+      description: |
+        Please provide the version of GreptimeDB. For example:
+        0.5.1 etc. You can get it by executing command line `greptime --version`.
+      placeholder: "0.5.1"
+    validations:
+      required: true
+
   - type: textarea
     id: logs
     attributes:

.github/pull_request_template.md

@@ -15,6 +15,6 @@ Please explain IN DETAIL what the changes are in this PR and why they are needed
 
 - [ ]  I have written the necessary rustdoc comments.
 - [ ]  I have added the necessary unit tests and integration tests.
-- [ ]  This PR does not require documentation updates.
+- [x]  This PR does not require documentation updates.
 
 ## Refer to a related PR or issue link (optional)

.github/workflows/apidoc.yml

@@ -1,7 +1,7 @@
 on:
   push:
     branches:
-      - develop
+      - main
     paths-ignore:
       - 'docs/**'
       - 'config/**'

.github/workflows/develop.yml

@@ -11,7 +11,6 @@ on:
       - '.gitignore'
   push:
     branches:
-      - develop
       - main
     paths-ignore:
       - 'docs/**'
@@ -105,6 +104,37 @@ jobs:
           path: ${{ runner.temp }}/greptime-*.log
           retention-days: 3
 
+  sqlness-kafka-wal:
+    name: Sqlness Test with Kafka Wal
+    if: github.event.pull_request.draft == false
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-20.04-8-cores ]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Setup kafka server
+        working-directory: tests-integration/fixtures/kafka
+        run: docker compose -f docker-compose-standalone.yml up -d --wait
+      - name: Run sqlness
+        run: cargo sqlness -w kafka -k 127.0.0.1:9092
+      - name: Upload sqlness logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: sqlness-logs
+          path: ${{ runner.temp }}/greptime-*.log
+          retention-days: 3
+
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false

.github/workflows/doc-label.yml

@@ -1,6 +1,6 @@
 name: "PR Doc Labeler"
 on:
-  pull_request:
+  pull_request_target:
     types: [opened, edited, synchronize, ready_for_review, auto_merge_enabled, labeled, unlabeled]
 
 permissions:
@@ -18,3 +18,14 @@ jobs:
         enable-versioned-regex: false
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: 1
+    - name: create an issue in doc repo
+      uses: dacbd/create-issue-action@main
+      if: ${{ github.event.action == 'opened' && contains(github.event.pull_request.body, '- [ ]  This PR does not require documentation updates.') }}
+      with:
+        owner: GreptimeTeam
+        repo: docs
+        token: ${{ secrets.DOCS_REPO_TOKEN }}
+        title: Update docs for ${{ github.event.issue.title || github.event.pull_request.title }}
+        body: |
+          A document change request is generated from
+          ${{ github.event.issue.html_url || github.event.pull_request.html_url }}

.github/workflows/docs.yml

@@ -11,7 +11,6 @@ on:
       - '.gitignore'
   push:
     branches:
-      - develop
       - main
     paths:
       - 'docs/**'

.github/workflows/license.yaml

@@ -3,7 +3,7 @@ name: License checker
 on:
   push:
     branches:
-    - develop
+    - main
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
 jobs:

.github/workflows/release.yml

@@ -91,7 +91,7 @@ env:
   # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
   # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.6.0
+  NEXT_RELEASE_VERSION: v0.7.0
 
 jobs:
   allocate-runners:

.github/workflows/size-label.yml

@@ -1,11 +1,14 @@
 name: size-labeler
 
-on: [pull_request]
+on: [pull_request_target]
 
 jobs:
   labeler:
     runs-on: ubuntu-latest
     name: Label the PR size
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
       - uses: codelytv/pr-size-labeler@v1
         with:
@@ -18,9 +21,5 @@ jobs:
           l_max_size: '1000'
           xl_label: 'Size: XL'
           fail_if_xl: 'false'
-          message_if_xl: >
-            This PR exceeds the recommended size of 1000 lines.
-            Please make sure you are NOT addressing multiple issues with one PR.
-            Note this PR might be rejected due to its size.
-          github_api_url: 'api.github.com'
+          message_if_xl: ""
           files_to_ignore: 'Cargo.lock'

CONTRIBUTING.md

@@ -10,7 +10,7 @@ Follow our [README](https://github.com/GreptimeTeam/greptimedb#readme) to get th
 
 It can feel intimidating to contribute to a complex project, but it can also be exciting and fun. These general notes will help everyone participate in this communal activity.
 
-- Follow the [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md)
+- Follow the [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/main/CODE_OF_CONDUCT.md)
 - Small changes make huge differences. We will happily accept a PR making a single character change if it helps move forward. Don't wait to have everything working.
 - Check the closed issues before opening your issue.
 - Try to follow the existing style of the code.
@@ -26,7 +26,7 @@ Pull requests are great, but we accept all kinds of other help if you like. Such
 
 ## Code of Conduct
 
-Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/develop/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
+Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/main/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
 
 ## License
 

Cargo.toml

@@ -58,7 +58,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.5.0"
+version = "0.6.0"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -111,7 +111,7 @@ prost = "0.12"
 raft-engine = { git = "https://github.com/tikv/raft-engine.git", rev = "22dfb426cd994602b57725ef080287d3e53db479" }
 rand = "0.8"
 regex = "1.8"
-regex-automata = { version = "0.1", features = ["transducer"] }
+regex-automata = { version = "0.2", features = ["transducer"] }
 reqwest = { version = "0.11", default-features = false, features = [
     "json",
     "rustls-tls-native-roots",
@@ -121,7 +121,7 @@ rskafka = "0.5"
 rust_decimal = "1.33"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-smallvec = "1"
+smallvec = { version = "1", features = ["serde"] }
 snafu = "0.7"
 # on branch v0.38.x
 sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "6a93567ae38d42be5c8d08b13c8ff4dde26502ef", features = [
@@ -169,6 +169,7 @@ datanode = { path = "src/datanode" }
 datatypes = { path = "src/datatypes" }
 file-engine = { path = "src/file-engine" }
 frontend = { path = "src/frontend" }
+index = { path = "src/index" }
 log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
 meta-srv = { path = "src/meta-srv" }
@@ -179,6 +180,7 @@ operator = { path = "src/operator" }
 partition = { path = "src/partition" }
 plugins = { path = "src/plugins" }
 promql = { path = "src/promql" }
+puffin = { path = "src/puffin" }
 query = { path = "src/query" }
 script = { path = "src/script" }
 servers = { path = "src/servers" }

README.md

@@ -1,8 +1,8 @@
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: light)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png">
-    <source media="(prefers-color-scheme: dark)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding-dark.png">
-    <img alt="GreptimeDB Logo" src="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@develop/docs/logo-text-padding.png" width="400px">
+    <source media="(prefers-color-scheme: light)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@main/docs/logo-text-padding.png">
+    <source media="(prefers-color-scheme: dark)" srcset="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@main/docs/logo-text-padding-dark.png">
+    <img alt="GreptimeDB Logo" src="https://cdn.jsdelivr.net/gh/GreptimeTeam/greptimedb@main/docs/logo-text-padding.png" width="400px">
   </picture>
 </p>
 
@@ -12,11 +12,11 @@
 </h3>
 
 <p align="center">
-    <a href="https://codecov.io/gh/GrepTimeTeam/greptimedb"><img src="https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/develop/graph/badge.svg?token=FITFDI3J3C"></img></a>
+    <a href="https://codecov.io/gh/GrepTimeTeam/greptimedb"><img src="https://codecov.io/gh/GrepTimeTeam/greptimedb/branch/main/graph/badge.svg?token=FITFDI3J3C"></img></a>
     &nbsp;
     <a href="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml"><img src="https://github.com/GreptimeTeam/greptimedb/actions/workflows/develop.yml/badge.svg" alt="CI"></img></a>
     &nbsp;
-    <a href="https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE"><img src="https://img.shields.io/github/license/greptimeTeam/greptimedb"></a>
+    <a href="https://github.com/greptimeTeam/greptimedb/blob/main/LICENSE"><img src="https://img.shields.io/github/license/greptimeTeam/greptimedb"></a>
 </p>
 
 <p align="center">
@@ -29,21 +29,17 @@
 
 ## What is GreptimeDB
 
-GreptimeDB is an open-source time-series database with a special focus on
-scalability, analytical capabilities and efficiency. It's designed to work on
-infrastructure of the cloud era, and users benefit from its elasticity and commodity
-storage.
+GreptimeDB is an open-source time-series database focusing on efficiency, scalability, and analytical capabilities.
+It's designed to work on infrastructure of the cloud era, and users benefit from its elasticity and commodity storage.
 
-Our core developers have been building time-series data platform
-for years. Based on their best-practices, GreptimeDB is born to give you:
+Our core developers have been building time-series data platforms for years. Based on their best-practices, GreptimeDB is born to give you:
 
-- A standalone binary that scales to highly-available distributed cluster, providing a transparent experience for cluster users
-- Optimized columnar layout for handling time-series data; compacted, compressed, and stored on various storage backends
-- Flexible indexes, tackling high cardinality issues down
-- Distributed, parallel query execution, leveraging elastic computing resource
-- Native SQL, and Python scripting for advanced analytical scenarios
-- Widely adopted database protocols and APIs, native PromQL supports
-- Extensible table engine architecture for extensive workloads
+- Optimized columnar layout for handling time-series data; compacted, compressed, and stored on various storage backends, particularly cloud object storage with 50x cost efficiency.
+- Fully open-source distributed cluster architecture that harnesses the power of cloud-native elastic computing resources.
+- Seamless scalability from a standalone binary at edge to a robust, highly available distributed cluster in cloud, with a transparent experience for both developers and administrators.
+- Native SQL and PromQL for queries, and Python scripting to facilitate complex analytical tasks.
+- Flexible indexing capabilities and distributed, parallel-processing query engine, tackling high cardinality issues down.
+- Widely adopted database protocols and APIs, including MySQL, PostgreSQL, and Prometheus Remote Storage, etc.
 
 ## Quick Start
 
@@ -168,7 +164,7 @@ In addition, you may:
 GreptimeDB uses the [Apache 2.0 license][1] to strike a balance between
 open contributions and allowing you to use the software however you want.
 
-[1]: <https://github.com/greptimeTeam/greptimedb/blob/develop/LICENSE>
+[1]: <https://github.com/greptimeTeam/greptimedb/blob/main/LICENSE>
 
 ## Contributing
 

config/datanode.example.toml

@@ -34,11 +34,7 @@ connect_timeout = "1s"
 tcp_nodelay = true
 
 # WAL options.
-# Currently, users are expected to choose the wal through the provider field. 
-# When a wal provider is chose, the user should comment out all other wal config 
-# except those corresponding to the chosen one.
 [wal]
-# WAL data directory
 provider = "raft_engine"
 
 # Raft-engine wal options, see `standalone.example.toml`.
@@ -51,9 +47,10 @@ sync_write = false
 
 # Kafka wal options, see `standalone.example.toml`.
 # broker_endpoints = ["127.0.0.1:9092"]
-# max_batch_size = "4MB"
+# Warning: Kafka has a default limit of 1MB per message in a topic.
+# max_batch_size = "1MB"
 # linger = "200ms"
-# produce_record_timeout = "100ms"
+# consumer_wait_timeout = "100ms"
 # backoff_init = "500ms"
 # backoff_max = "10s"
 # backoff_base = 2
@@ -116,6 +113,8 @@ sst_write_buffer_size = "8MB"
 scan_parallelism = 0
 # Capacity of the channel to send data from parallel scan tasks to the main task (default 32).
 parallel_scan_channel_size = 32
+# Whether to allow stale WAL entries read during replay.
+allow_stale_entries = false
 
 # Log options, see `standalone.example.toml`
 # [logging]
@@ -129,11 +128,10 @@ parallel_scan_channel_size = 32
 # [export_metrics]
 # whether enable export metrics, default is false
 # enable = false
-# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
-# endpoint = "127.0.0.1:4000"
-# The database name of exported metrics stores, user needs to specify a valid database
-# db = ""
 # The interval of export metrics
 # write_interval = "30s"
+# [export_metrics.remote_write]
+# The url the metrics send to. The url is empty by default, url example: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`
+# url = ""
 # HTTP headers of Prometheus remote-write carry
 # headers = {}

config/frontend.example.toml

@@ -1,5 +1,7 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
+# The default timezone of the server
+# default_timezone = "UTC"
 
 [heartbeat]
 # Interval for sending heartbeat task to the Metasrv, 5 seconds by default.
@@ -85,11 +87,8 @@ tcp_nodelay = true
 # [export_metrics]
 # whether enable export metrics, default is false
 # enable = false
-# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
-# endpoint = "127.0.0.1:4000"
-# The database name of exported metrics stores, user needs to specify a valid database
-# db = ""
 # The interval of export metrics
 # write_interval = "30s"
-# HTTP headers of Prometheus remote-write carry
-# headers = {}
+# for `frontend`, `self_import` is recommend to collect metrics generated by itself
+# [export_metrics.self_import]
+# db = "information_schema"

config/metasrv.example.toml

@@ -15,6 +15,8 @@ selector = "lease_based"
 use_memory_store = false
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
+# If it's not empty, the metasrv will store all data with this key prefix.
+store_key_prefix = ""
 
 # Log options, see `standalone.example.toml`
 # [logging]
@@ -62,8 +64,6 @@ provider = "raft_engine"
 # selector_type = "round_robin"
 # A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
 # topic_name_prefix = "greptimedb_wal_topic"
-# Number of partitions per topic.
-# num_partitions = 1
 # Expected number of replicas of each partition.
 # replication_factor = 1
 # Above which a topic creation operation will be cancelled.
@@ -84,11 +84,10 @@ provider = "raft_engine"
 # [export_metrics]
 # whether enable export metrics, default is false
 # enable = false
-# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
-# endpoint = "127.0.0.1:4000"
-# The database name of exported metrics stores, user needs to specify a valid database
-# db = ""
 # The interval of export metrics
 # write_interval = "30s"
+# [export_metrics.remote_write]
+# The url the metrics send to. The url is empty by default, url example: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`
+# url = ""
 # HTTP headers of Prometheus remote-write carry
 # headers = {}

config/standalone.example.toml

@@ -2,6 +2,8 @@
 mode = "standalone"
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
+# The default timezone of the server
+# default_timezone = "UTC"
 
 # HTTP server options.
 [http]
@@ -98,29 +100,29 @@ provider = "raft_engine"
 # Available selector types: 
 # - "round_robin" (default)
 # selector_type = "round_robin"
-# A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
+# The prefix of topic name.
 # topic_name_prefix = "greptimedb_wal_topic"
-# Number of partitions per topic.
-# num_partitions = 1
-# Expected number of replicas of each partition.
+# The number of replicas of each partition.
+# Warning: the replication factor must be positive and must not be greater than the number of broker endpoints.
 # replication_factor = 1
 
-# The maximum log size a kafka batch producer could buffer.
-# max_batch_size = "4MB"
-# The linger duration of a kafka batch producer.
+# The max size of a single producer batch.
+# Warning: Kafka has a default limit of 1MB per message in a topic.
+# max_batch_size = "1MB"
+# The linger duration.
 # linger = "200ms"
-# The maximum amount of time (in milliseconds) to wait for Kafka records to be returned.
-# produce_record_timeout = "100ms"
-# Above which a topic creation operation will be cancelled.
+# The consumer wait timeout.
+# consumer_wait_timeout = "100ms"
+# Create topic timeout.
 # create_topic_timeout = "30s"
 
-# The initial backoff for kafka clients.
+# The initial backoff delay.
 # backoff_init = "500ms"
-# The maximum backoff for kafka clients.
+# The maximum backoff delay.
 # backoff_max = "10s"
 # Exponential backoff rate, i.e. next backoff = base * current backoff.
 # backoff_base = 2
-# Stop reconnecting if the total wait time reaches the deadline. If this config is missing, the reconnecting won't terminate.
+# The deadline of retries.
 # backoff_deadline = "5mins"
 
 # WAL data directory
@@ -135,6 +137,12 @@ purge_interval = "10m"
 read_batch_size = 128
 # Whether to sync log file after every write.
 sync_write = false
+# Whether to reuse logically truncated log files.
+enable_log_recycle = true
+# Whether to pre-create log files on start up
+prefill_log_files = false
+# Duration for fsyncing log files.
+sync_period = "1000ms"
 
 # Metadata storage options.
 [metadata_store]
@@ -205,6 +213,8 @@ sst_write_buffer_size = "8MB"
 scan_parallelism = 0
 # Capacity of the channel to send data from parallel scan tasks to the main task (default 32).
 parallel_scan_channel_size = 32
+# Whether to allow stale WAL entries read during replay.
+allow_stale_entries = false
 
 # Log options
 # [logging]
@@ -228,11 +238,8 @@ parallel_scan_channel_size = 32
 # [export_metrics]
 # whether enable export metrics, default is false
 # enable = false
-# The url of metrics export endpoint, default is `frontend` default HTTP endpoint.
-# endpoint = "127.0.0.1:4000"
-# The database name of exported metrics stores, user needs to specify a valid database
-# db = ""
 # The interval of export metrics
 # write_interval = "30s"
-# HTTP headers of Prometheus remote-write carry
-# headers = {}
+# for `standalone`, `self_import` is recommend to collect metrics generated by itself
+# [export_metrics.self_import]
+# db = "information_schema"

docs/rfcs/2023-12-22-enclose-column-id.md

@@ -0,0 +1,44 @@
+---
+Feature Name: Enclose Column Id
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/2982
+Date: 2023-12-22
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+# Summary
+This RFC proposes to enclose the usage of `ColumnId` into the region engine only.
+
+# Motivation
+`ColumnId` is an identifier for columns. It's assigned by meta server, stored in `TableInfo` and `RegionMetadata` and used in region engine to distinguish columns.
+
+At present, Both Frontend, Datanode and Metasrv are aware of `ColumnId` but it's only used in region engine. Thus this RFC proposes to remove it from Frontend (mainly used in `TableInfo`) and Metasrv.
+
+# Details
+
+`ColumnId` is used widely on both read and write paths. Removing it from Frontend and Metasrv implies several things:
+
+- A column may have different column id in different regions.
+- A column is identified by its name in all components.
+- Column order in the region engine is not restricted, i.e., no need to be in the same order with table info.
+
+The first thing doesn't matter IMO. This concept doesn't exist anymore outside of region server, and each region is autonomous and independent -- the only guarantee it should hold is those columns exist. But if we consider region repartition, where the SST file would be re-assign to different regions, things would become a bit more complicated. A possible solution is store the relation between name and ColumnId in the manifest, but it's out of the scope of this RFC. We can likely give a workaround by introducing a indirection mapping layer of different version of partitions.
+
+And more importantly, we can still assume columns have the same column ids across regions. We have procedure to maintain consistency between regions and the region engine should ensure alterations are idempotent. So it is possible that region repartition doesn't need to consider column ids or other region metadata in the future.
+
+Users write and query column by their names, not by ColumnId or something else. The second point also means to change the column reference in ScanRequest from index to name. This change can hugely alleviate the misuse of the column index, which has given us many surprises.
+
+And for the last one, column order only matters in table info. This order is used in user-faced table structure operation, like add column, describe column or as the default order of INSERT clause. None of them is connected with the order in storage.
+
+# Drawback
+Firstly, this is a breaking change. Delivering this change requires a full upgrade of the cluster. Secondly, this change may introduce some performance regression. For example, we have to pass the full table name in the `ScanRequest` instead of the `ColumnId`. But this influence is very limited, since the column index is only used in the region engine.
+
+# Alternatives
+
+There are two alternatives from the perspective of "what can be used as the column identifier":
+
+- Index of column to the table schema
+- `ColumnId` of that column
+
+The first one is what we are using now. By choosing this way, it's required to keep the column order in the region engine the same as the table info. This is not hard to achieve, but it's a bit annoying. And things become tricky when there is internal column or different schemas like those stored in file format. And this is the initial purpose of this RFC, which is trying to decouple the table schema and region schema.
+
+The second one, in other hand, requires the `ColumnId` should be identical in all regions and `TableInfo`. It has the same drawback with the previous alternative, that the `TableInfo` and `RegionMetadata` are tighted together. Another point is that the `ColumnId` is assigned by the Metasrv, who doesn't need it but have to maintain it. And this also limits the functionality of `ColumnId`, by taking the ability of assigning it from concrete region engine.

src/catalog/Cargo.toml

@@ -11,6 +11,7 @@ testing = []
 api.workspace = true
 arc-swap = "1.0"
 arrow-schema.workspace = true
+arrow.workspace = true
 async-stream.workspace = true
 async-trait = "0.1"
 build-data = "0.1"
@@ -29,6 +30,7 @@ datafusion.workspace = true
 datatypes.workspace = true
 futures = "0.3"
 futures-util.workspace = true
+itertools.workspace = true
 lazy_static.workspace = true
 meta-client.workspace = true
 moka = { workspace = true, features = ["future"] }

src/catalog/src/information_schema.rs

@@ -13,20 +13,25 @@
 // limitations under the License.
 
 mod columns;
+mod key_column_usage;
 mod memory_table;
+mod predicate;
+mod runtime_metrics;
+mod schemata;
 mod table_names;
 mod tables;
 
 use std::collections::HashMap;
 use std::sync::{Arc, Weak};
 
-use common_catalog::consts::{self, INFORMATION_SCHEMA_NAME};
+use common_catalog::consts::{self, DEFAULT_CATALOG_NAME, INFORMATION_SCHEMA_NAME};
 use common_error::ext::BoxedError;
 use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
 use datatypes::schema::SchemaRef;
 use futures_util::StreamExt;
 use lazy_static::lazy_static;
 use paste::paste;
+pub(crate) use predicate::Predicates;
 use snafu::ResultExt;
 use store_api::data_source::DataSource;
 use store_api::storage::{ScanRequest, TableId};
@@ -40,7 +45,10 @@ pub use table_names::*;
 
 use self::columns::InformationSchemaColumns;
 use crate::error::Result;
+use crate::information_schema::key_column_usage::InformationSchemaKeyColumnUsage;
 use crate::information_schema::memory_table::{get_schema_columns, MemoryTable};
+use crate::information_schema::runtime_metrics::InformationSchemaMetrics;
+use crate::information_schema::schemata::InformationSchemaSchemata;
 use crate::information_schema::tables::InformationSchemaTables;
 use crate::CatalogManager;
 
@@ -50,7 +58,22 @@ lazy_static! {
         ENGINES,
         COLUMN_PRIVILEGES,
         COLUMN_STATISTICS,
-        BUILD_INFO,
+        CHARACTER_SETS,
+        COLLATIONS,
+        COLLATION_CHARACTER_SET_APPLICABILITY,
+        CHECK_CONSTRAINTS,
+        EVENTS,
+        FILES,
+        OPTIMIZER_TRACE,
+        PARAMETERS,
+        PROFILING,
+        REFERENTIAL_CONSTRAINTS,
+        ROUTINES,
+        SCHEMA_PRIVILEGES,
+        TABLE_PRIVILEGES,
+        TRIGGERS,
+        GLOBAL_STATUS,
+        SESSION_STATUS,
     ];
 }
 
@@ -120,12 +143,32 @@ impl InformationSchemaProvider {
 
     fn build_tables(&mut self) {
         let mut tables = HashMap::new();
+
+        // Carefully consider the tables that may expose sensitive cluster configurations,
+        // authentication details, and other critical information.
+        // Only put these tables under `greptime` catalog to prevent info leak.
+        if self.catalog_name == DEFAULT_CATALOG_NAME {
+            tables.insert(
+                RUNTIME_METRICS.to_string(),
+                self.build_table(RUNTIME_METRICS).unwrap(),
+            );
+            tables.insert(
+                BUILD_INFO.to_string(),
+                self.build_table(BUILD_INFO).unwrap(),
+            );
+        }
+
         tables.insert(TABLES.to_string(), self.build_table(TABLES).unwrap());
+        tables.insert(SCHEMATA.to_string(), self.build_table(SCHEMATA).unwrap());
         tables.insert(COLUMNS.to_string(), self.build_table(COLUMNS).unwrap());
+        tables.insert(
+            KEY_COLUMN_USAGE.to_string(),
+            self.build_table(KEY_COLUMN_USAGE).unwrap(),
+        );
 
         // Add memory tables
         for name in MEMORY_TABLES.iter() {
-            tables.insert((*name).to_string(), self.build_table(name).unwrap());
+            tables.insert((*name).to_string(), self.build_table(name).expect(name));
         }
 
         self.tables = tables;
@@ -134,7 +177,7 @@ impl InformationSchemaProvider {
     fn build_table(&self, name: &str) -> Option<TableRef> {
         self.information_table(name).map(|table| {
             let table_info = Self::table_info(self.catalog_name.clone(), &table);
-            let filter_pushdown = FilterPushDownType::Unsupported;
+            let filter_pushdown = FilterPushDownType::Inexact;
             let thin_table = ThinTable::new(table_info, filter_pushdown);
 
             let data_source = Arc::new(InformationTableDataSource::new(table));
@@ -156,6 +199,33 @@ impl InformationSchemaProvider {
             COLUMN_PRIVILEGES => setup_memory_table!(COLUMN_PRIVILEGES),
             COLUMN_STATISTICS => setup_memory_table!(COLUMN_STATISTICS),
             BUILD_INFO => setup_memory_table!(BUILD_INFO),
+            CHARACTER_SETS => setup_memory_table!(CHARACTER_SETS),
+            COLLATIONS => setup_memory_table!(COLLATIONS),
+            COLLATION_CHARACTER_SET_APPLICABILITY => {
+                setup_memory_table!(COLLATION_CHARACTER_SET_APPLICABILITY)
+            }
+            CHECK_CONSTRAINTS => setup_memory_table!(CHECK_CONSTRAINTS),
+            EVENTS => setup_memory_table!(EVENTS),
+            FILES => setup_memory_table!(FILES),
+            OPTIMIZER_TRACE => setup_memory_table!(OPTIMIZER_TRACE),
+            PARAMETERS => setup_memory_table!(PARAMETERS),
+            PROFILING => setup_memory_table!(PROFILING),
+            REFERENTIAL_CONSTRAINTS => setup_memory_table!(REFERENTIAL_CONSTRAINTS),
+            ROUTINES => setup_memory_table!(ROUTINES),
+            SCHEMA_PRIVILEGES => setup_memory_table!(SCHEMA_PRIVILEGES),
+            TABLE_PRIVILEGES => setup_memory_table!(TABLE_PRIVILEGES),
+            TRIGGERS => setup_memory_table!(TRIGGERS),
+            GLOBAL_STATUS => setup_memory_table!(GLOBAL_STATUS),
+            SESSION_STATUS => setup_memory_table!(SESSION_STATUS),
+            KEY_COLUMN_USAGE => Some(Arc::new(InformationSchemaKeyColumnUsage::new(
+                self.catalog_name.clone(),
+                self.catalog_manager.clone(),
+            )) as _),
+            SCHEMATA => Some(Arc::new(InformationSchemaSchemata::new(
+                self.catalog_name.clone(),
+                self.catalog_manager.clone(),
+            )) as _),
+            RUNTIME_METRICS => Some(Arc::new(InformationSchemaMetrics::new())),
             _ => None,
         }
     }
@@ -187,7 +257,7 @@ trait InformationTable {
 
     fn schema(&self) -> SchemaRef;
 
-    fn to_stream(&self) -> Result<SendableRecordBatchStream>;
+    fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream>;
 
     fn table_type(&self) -> TableType {
         TableType::Temporary
@@ -221,7 +291,7 @@ impl DataSource for InformationTableDataSource {
         &self,
         request: ScanRequest,
     ) -> std::result::Result<SendableRecordBatchStream, BoxedError> {
-        let projection = request.projection;
+        let projection = request.projection.clone();
         let projected_schema = match &projection {
             Some(projection) => self.try_project(projection)?,
             None => self.table.schema(),
@@ -229,7 +299,7 @@ impl DataSource for InformationTableDataSource {
 
         let stream = self
             .table
-            .to_stream()
+            .to_stream(request)
             .map_err(BoxedError::new)
             .context(TablesRecordBatchSnafu)
             .map_err(BoxedError::new)?

src/catalog/src/information_schema/columns.rs

@@ -29,14 +29,16 @@ use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatc
 use datatypes::prelude::{ConcreteDataType, DataType};
 use datatypes::scalars::ScalarVectorBuilder;
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::value::Value;
 use datatypes::vectors::{StringVectorBuilder, VectorRef};
 use snafu::{OptionExt, ResultExt};
-use store_api::storage::TableId;
+use store_api::storage::{ScanRequest, TableId};
 
 use super::{InformationTable, COLUMNS};
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
+use crate::information_schema::Predicates;
 use crate::CatalogManager;
 
 pub(super) struct InformationSchemaColumns {
@@ -51,6 +53,10 @@ const TABLE_NAME: &str = "table_name";
 const COLUMN_NAME: &str = "column_name";
 const DATA_TYPE: &str = "data_type";
 const SEMANTIC_TYPE: &str = "semantic_type";
+const COLUMN_DEFAULT: &str = "column_default";
+const IS_NULLABLE: &str = "is_nullable";
+const COLUMN_TYPE: &str = "column_type";
+const COLUMN_COMMENT: &str = "column_comment";
 
 impl InformationSchemaColumns {
     pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
@@ -69,6 +75,10 @@ impl InformationSchemaColumns {
             ColumnSchema::new(COLUMN_NAME, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(DATA_TYPE, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(SEMANTIC_TYPE, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(COLUMN_DEFAULT, ConcreteDataType::string_datatype(), true),
+            ColumnSchema::new(IS_NULLABLE, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(COLUMN_TYPE, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(COLUMN_COMMENT, ConcreteDataType::string_datatype(), true),
         ]))
     }
 
@@ -94,14 +104,14 @@ impl InformationTable for InformationSchemaColumns {
         self.schema.clone()
     }
 
-    fn to_stream(&self) -> Result<SendableRecordBatchStream> {
+    fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream> {
         let schema = self.schema.arrow_schema().clone();
         let mut builder = self.builder();
         let stream = Box::pin(DfRecordBatchStreamAdapter::new(
             schema,
             futures::stream::once(async move {
                 builder
-                    .make_columns()
+                    .make_columns(Some(request))
                     .await
                     .map(|x| x.into_df_record_batch())
                     .map_err(Into::into)
@@ -126,6 +136,11 @@ struct InformationSchemaColumnsBuilder {
     column_names: StringVectorBuilder,
     data_types: StringVectorBuilder,
     semantic_types: StringVectorBuilder,
+
+    column_defaults: StringVectorBuilder,
+    is_nullables: StringVectorBuilder,
+    column_types: StringVectorBuilder,
+    column_comments: StringVectorBuilder,
 }
 
 impl InformationSchemaColumnsBuilder {
@@ -144,16 +159,21 @@ impl InformationSchemaColumnsBuilder {
             column_names: StringVectorBuilder::with_capacity(42),
             data_types: StringVectorBuilder::with_capacity(42),
             semantic_types: StringVectorBuilder::with_capacity(42),
+            column_defaults: StringVectorBuilder::with_capacity(42),
+            is_nullables: StringVectorBuilder::with_capacity(42),
+            column_types: StringVectorBuilder::with_capacity(42),
+            column_comments: StringVectorBuilder::with_capacity(42),
         }
     }
 
     /// Construct the `information_schema.columns` virtual table
-    async fn make_columns(&mut self) -> Result<RecordBatch> {
+    async fn make_columns(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
         let catalog_name = self.catalog_name.clone();
         let catalog_manager = self
             .catalog_manager
             .upgrade()
             .context(UpgradeWeakCatalogManagerRefSnafu)?;
+        let predicates = Predicates::from_scan_request(&request);
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
@@ -184,12 +204,12 @@ impl InformationSchemaColumnsBuilder {
                         };
 
                         self.add_column(
+                            &predicates,
                             &catalog_name,
                             &schema_name,
                             &table_name,
-                            &column.name,
-                            &column.data_type.name(),
                             semantic_type,
+                            column,
                         );
                     }
                 } else {
@@ -203,19 +223,48 @@ impl InformationSchemaColumnsBuilder {
 
     fn add_column(
         &mut self,
+        predicates: &Predicates,
         catalog_name: &str,
         schema_name: &str,
         table_name: &str,
-        column_name: &str,
-        data_type: &str,
         semantic_type: &str,
+        column_schema: &ColumnSchema,
     ) {
+        let data_type = &column_schema.data_type.name();
+
+        let row = [
+            (TABLE_CATALOG, &Value::from(catalog_name)),
+            (TABLE_SCHEMA, &Value::from(schema_name)),
+            (TABLE_NAME, &Value::from(table_name)),
+            (COLUMN_NAME, &Value::from(column_schema.name.as_str())),
+            (DATA_TYPE, &Value::from(data_type.as_str())),
+            (SEMANTIC_TYPE, &Value::from(semantic_type)),
+        ];
+
+        if !predicates.eval(&row) {
+            return;
+        }
+
         self.catalog_names.push(Some(catalog_name));
         self.schema_names.push(Some(schema_name));
         self.table_names.push(Some(table_name));
-        self.column_names.push(Some(column_name));
+        self.column_names.push(Some(&column_schema.name));
         self.data_types.push(Some(data_type));
         self.semantic_types.push(Some(semantic_type));
+        self.column_defaults.push(
+            column_schema
+                .default_constraint()
+                .map(|s| format!("{}", s))
+                .as_deref(),
+        );
+        if column_schema.is_nullable() {
+            self.is_nullables.push(Some("Yes"));
+        } else {
+            self.is_nullables.push(Some("No"));
+        }
+        self.column_types.push(Some(data_type));
+        self.column_comments
+            .push(column_schema.column_comment().map(|x| x.as_ref()));
     }
 
     fn finish(&mut self) -> Result<RecordBatch> {
@@ -226,6 +275,10 @@ impl InformationSchemaColumnsBuilder {
             Arc::new(self.column_names.finish()),
             Arc::new(self.data_types.finish()),
             Arc::new(self.semantic_types.finish()),
+            Arc::new(self.column_defaults.finish()),
+            Arc::new(self.is_nullables.finish()),
+            Arc::new(self.column_types.finish()),
+            Arc::new(self.column_comments.finish()),
         ];
 
         RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
@@ -244,7 +297,7 @@ impl DfPartitionStream for InformationSchemaColumns {
             schema,
             futures::stream::once(async move {
                 builder
-                    .make_columns()
+                    .make_columns(None)
                     .await
                     .map(|x| x.into_df_record_batch())
                     .map_err(Into::into)

src/catalog/src/information_schema/key_column_usage.rs

@@ -0,0 +1,347 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::{Arc, Weak};
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_catalog::consts::INFORMATION_SCHEMA_KEY_COLUMN_USAGE_TABLE_ID;
+use common_error::ext::BoxedError;
+use common_query::physical_plan::TaskContext;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
+use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
+use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::value::Value;
+use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder};
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::{ScanRequest, TableId};
+
+use super::KEY_COLUMN_USAGE;
+use crate::error::{
+    CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
+};
+use crate::information_schema::{InformationTable, Predicates};
+use crate::CatalogManager;
+
+const CONSTRAINT_SCHEMA: &str = "constraint_schema";
+const CONSTRAINT_NAME: &str = "constraint_name";
+const TABLE_CATALOG: &str = "table_catalog";
+const TABLE_SCHEMA: &str = "table_schema";
+const TABLE_NAME: &str = "table_name";
+const COLUMN_NAME: &str = "column_name";
+const ORDINAL_POSITION: &str = "ordinal_position";
+
+/// The virtual table implementation for `information_schema.KEY_COLUMN_USAGE`.
+pub(super) struct InformationSchemaKeyColumnUsage {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_manager: Weak<dyn CatalogManager>,
+}
+
+impl InformationSchemaKeyColumnUsage {
+    pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
+        Self {
+            schema: Self::schema(),
+            catalog_name,
+            catalog_manager,
+        }
+    }
+
+    pub(crate) fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
+            ColumnSchema::new(
+                "constraint_catalog",
+                ConcreteDataType::string_datatype(),
+                false,
+            ),
+            ColumnSchema::new(
+                CONSTRAINT_SCHEMA,
+                ConcreteDataType::string_datatype(),
+                false,
+            ),
+            ColumnSchema::new(CONSTRAINT_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_CATALOG, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_SCHEMA, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(COLUMN_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(ORDINAL_POSITION, ConcreteDataType::uint32_datatype(), false),
+            ColumnSchema::new(
+                "position_in_unique_constraint",
+                ConcreteDataType::uint32_datatype(),
+                true,
+            ),
+            ColumnSchema::new(
+                "referenced_table_schema",
+                ConcreteDataType::string_datatype(),
+                true,
+            ),
+            ColumnSchema::new(
+                "referenced_table_name",
+                ConcreteDataType::string_datatype(),
+                true,
+            ),
+            ColumnSchema::new(
+                "referenced_column_name",
+                ConcreteDataType::string_datatype(),
+                true,
+            ),
+        ]))
+    }
+
+    fn builder(&self) -> InformationSchemaKeyColumnUsageBuilder {
+        InformationSchemaKeyColumnUsageBuilder::new(
+            self.schema.clone(),
+            self.catalog_name.clone(),
+            self.catalog_manager.clone(),
+        )
+    }
+}
+
+impl InformationTable for InformationSchemaKeyColumnUsage {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_KEY_COLUMN_USAGE_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        KEY_COLUMN_USAGE
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_key_column_usage(Some(request))
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+/// Builds the `information_schema.KEY_COLUMN_USAGE` table row by row
+///
+/// Columns are based on <https://dev.mysql.com/doc/refman/8.2/en/information-schema-key-column-usage-table.html>
+struct InformationSchemaKeyColumnUsageBuilder {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_manager: Weak<dyn CatalogManager>,
+
+    constraint_catalog: StringVectorBuilder,
+    constraint_schema: StringVectorBuilder,
+    constraint_name: StringVectorBuilder,
+    table_catalog: StringVectorBuilder,
+    table_schema: StringVectorBuilder,
+    table_name: StringVectorBuilder,
+    column_name: StringVectorBuilder,
+    ordinal_position: UInt32VectorBuilder,
+    position_in_unique_constraint: UInt32VectorBuilder,
+    referenced_table_schema: StringVectorBuilder,
+    referenced_table_name: StringVectorBuilder,
+    referenced_column_name: StringVectorBuilder,
+}
+
+impl InformationSchemaKeyColumnUsageBuilder {
+    fn new(
+        schema: SchemaRef,
+        catalog_name: String,
+        catalog_manager: Weak<dyn CatalogManager>,
+    ) -> Self {
+        Self {
+            schema,
+            catalog_name,
+            catalog_manager,
+            constraint_catalog: StringVectorBuilder::with_capacity(42),
+            constraint_schema: StringVectorBuilder::with_capacity(42),
+            constraint_name: StringVectorBuilder::with_capacity(42),
+            table_catalog: StringVectorBuilder::with_capacity(42),
+            table_schema: StringVectorBuilder::with_capacity(42),
+            table_name: StringVectorBuilder::with_capacity(42),
+            column_name: StringVectorBuilder::with_capacity(42),
+            ordinal_position: UInt32VectorBuilder::with_capacity(42),
+            position_in_unique_constraint: UInt32VectorBuilder::with_capacity(42),
+            referenced_table_schema: StringVectorBuilder::with_capacity(42),
+            referenced_table_name: StringVectorBuilder::with_capacity(42),
+            referenced_column_name: StringVectorBuilder::with_capacity(42),
+        }
+    }
+
+    /// Construct the `information_schema.KEY_COLUMN_USAGE` virtual table
+    async fn make_key_column_usage(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
+        let catalog_name = self.catalog_name.clone();
+        let catalog_manager = self
+            .catalog_manager
+            .upgrade()
+            .context(UpgradeWeakCatalogManagerRefSnafu)?;
+        let predicates = Predicates::from_scan_request(&request);
+
+        let mut primary_constraints = vec![];
+
+        for schema_name in catalog_manager.schema_names(&catalog_name).await? {
+            if !catalog_manager
+                .schema_exists(&catalog_name, &schema_name)
+                .await?
+            {
+                continue;
+            }
+
+            for table_name in catalog_manager
+                .table_names(&catalog_name, &schema_name)
+                .await?
+            {
+                if let Some(table) = catalog_manager
+                    .table(&catalog_name, &schema_name, &table_name)
+                    .await?
+                {
+                    let keys = &table.table_info().meta.primary_key_indices;
+                    let schema = table.schema();
+
+                    for (idx, column) in schema.column_schemas().iter().enumerate() {
+                        if column.is_time_index() {
+                            self.add_key_column_usage(
+                                &predicates,
+                                &schema_name,
+                                "TIME INDEX",
+                                &schema_name,
+                                &table_name,
+                                &column.name,
+                                1, //always 1 for time index
+                            );
+                        }
+                        if keys.contains(&idx) {
+                            primary_constraints.push((
+                                schema_name.clone(),
+                                table_name.clone(),
+                                column.name.clone(),
+                            ));
+                        }
+                        // TODO(dimbtp): foreign key constraint not supported yet
+                    }
+                } else {
+                    unreachable!();
+                }
+            }
+        }
+
+        for (i, (schema_name, table_name, column_name)) in
+            primary_constraints.into_iter().enumerate()
+        {
+            self.add_key_column_usage(
+                &predicates,
+                &schema_name,
+                "PRIMARY",
+                &schema_name,
+                &table_name,
+                &column_name,
+                i as u32 + 1,
+            );
+        }
+
+        self.finish()
+    }
+
+    // TODO(dimbtp): Foreign key constraint has not `None` value for last 4
+    // fields, but it is not supported yet.
+    #[allow(clippy::too_many_arguments)]
+    fn add_key_column_usage(
+        &mut self,
+        predicates: &Predicates,
+        constraint_schema: &str,
+        constraint_name: &str,
+        table_schema: &str,
+        table_name: &str,
+        column_name: &str,
+        ordinal_position: u32,
+    ) {
+        let row = [
+            (CONSTRAINT_SCHEMA, &Value::from(constraint_schema)),
+            (CONSTRAINT_NAME, &Value::from(constraint_name)),
+            (TABLE_SCHEMA, &Value::from(table_schema)),
+            (TABLE_NAME, &Value::from(table_name)),
+            (COLUMN_NAME, &Value::from(column_name)),
+            (ORDINAL_POSITION, &Value::from(ordinal_position)),
+        ];
+
+        if !predicates.eval(&row) {
+            return;
+        }
+
+        self.constraint_catalog.push(Some("def"));
+        self.constraint_schema.push(Some(constraint_schema));
+        self.constraint_name.push(Some(constraint_name));
+        self.table_catalog.push(Some("def"));
+        self.table_schema.push(Some(table_schema));
+        self.table_name.push(Some(table_name));
+        self.column_name.push(Some(column_name));
+        self.ordinal_position.push(Some(ordinal_position));
+        self.position_in_unique_constraint.push(None);
+        self.referenced_table_schema.push(None);
+        self.referenced_table_name.push(None);
+        self.referenced_column_name.push(None);
+    }
+
+    fn finish(&mut self) -> Result<RecordBatch> {
+        let columns: Vec<VectorRef> = vec![
+            Arc::new(self.constraint_catalog.finish()),
+            Arc::new(self.constraint_schema.finish()),
+            Arc::new(self.constraint_name.finish()),
+            Arc::new(self.table_catalog.finish()),
+            Arc::new(self.table_schema.finish()),
+            Arc::new(self.table_name.finish()),
+            Arc::new(self.column_name.finish()),
+            Arc::new(self.ordinal_position.finish()),
+            Arc::new(self.position_in_unique_constraint.finish()),
+            Arc::new(self.referenced_table_schema.finish()),
+            Arc::new(self.referenced_table_name.finish()),
+            Arc::new(self.referenced_column_name.finish()),
+        ];
+        RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
+    }
+}
+
+impl DfPartitionStream for InformationSchemaKeyColumnUsage {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_key_column_usage(None)
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}

src/catalog/src/information_schema/memory_table.rs

@@ -26,7 +26,7 @@ use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatc
 use datatypes::schema::SchemaRef;
 use datatypes::vectors::VectorRef;
 use snafu::ResultExt;
-use store_api::storage::TableId;
+use store_api::storage::{ScanRequest, TableId};
 pub use tables::get_schema_columns;
 
 use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
@@ -74,7 +74,7 @@ impl InformationTable for MemoryTable {
         self.schema.clone()
     }
 
-    fn to_stream(&self) -> Result<SendableRecordBatchStream> {
+    fn to_stream(&self, _request: ScanRequest) -> Result<SendableRecordBatchStream> {
         let schema = self.schema.arrow_schema().clone();
         let mut builder = self.builder();
         let stream = Box::pin(DfRecordBatchStreamAdapter::new(
@@ -169,7 +169,7 @@ mod tests {
         assert_eq!("test", table.table_name());
         assert_eq!(schema, InformationTable::schema(&table));
 
-        let stream = table.to_stream().unwrap();
+        let stream = table.to_stream(ScanRequest::default()).unwrap();
 
         let batches = RecordBatches::try_collect(stream).await.unwrap();
 
@@ -198,7 +198,7 @@ mod tests {
         assert_eq!("test", table.table_name());
         assert_eq!(schema, InformationTable::schema(&table));
 
-        let stream = table.to_stream().unwrap();
+        let stream = table.to_stream(ScanRequest::default()).unwrap();
 
         let batches = RecordBatches::try_collect(stream).await.unwrap();
 

src/catalog/src/information_schema/memory_table/tables.rs

@@ -17,7 +17,7 @@ use std::sync::Arc;
 use common_catalog::consts::MITO_ENGINE;
 use datatypes::prelude::{ConcreteDataType, VectorRef};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
-use datatypes::vectors::StringVector;
+use datatypes::vectors::{Int64Vector, StringVector};
 
 use crate::information_schema::table_names::*;
 
@@ -97,6 +97,320 @@ pub fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec<VectorRef>) {
             ],
         ),
 
+        CHARACTER_SETS => (
+            vec![
+                string_column("CHARACTER_SET_NAME"),
+                string_column("DEFAULT_COLLATE_NAME"),
+                string_column("DESCRIPTION"),
+                bigint_column("MAXLEN"),
+            ],
+            vec![
+                Arc::new(StringVector::from(vec!["utf8"])),
+                Arc::new(StringVector::from(vec!["utf8_bin"])),
+                Arc::new(StringVector::from(vec!["UTF-8 Unicode"])),
+                Arc::new(Int64Vector::from_slice([4])),
+            ],
+        ),
+
+        COLLATIONS => (
+            vec![
+                string_column("COLLATION_NAME"),
+                string_column("CHARACTER_SET_NAME"),
+                bigint_column("ID"),
+                string_column("IS_DEFAULT"),
+                string_column("IS_COMPILED"),
+                bigint_column("SORTLEN"),
+            ],
+            vec![
+                Arc::new(StringVector::from(vec!["utf8_bin"])),
+                Arc::new(StringVector::from(vec!["utf8"])),
+                Arc::new(Int64Vector::from_slice([1])),
+                Arc::new(StringVector::from(vec!["Yes"])),
+                Arc::new(StringVector::from(vec!["Yes"])),
+                Arc::new(Int64Vector::from_slice([1])),
+            ],
+        ),
+
+        COLLATION_CHARACTER_SET_APPLICABILITY => (
+            vec![
+                string_column("COLLATION_NAME"),
+                string_column("CHARACTER_SET_NAME"),
+            ],
+            vec![
+                Arc::new(StringVector::from(vec!["utf8_bin"])),
+                Arc::new(StringVector::from(vec!["utf8"])),
+            ],
+        ),
+
+        CHECK_CONSTRAINTS => (
+            string_columns(&[
+                "CONSTRAINT_CATALOG",
+                "CONSTRAINT_SCHEMA",
+                "CONSTRAINT_NAME",
+                "CHECK_CLAUSE",
+            ]),
+            // Not support check constraints yet
+            vec![],
+        ),
+
+        EVENTS => (
+            vec![
+                string_column("EVENT_CATALOG"),
+                string_column("EVENT_SCHEMA"),
+                string_column("EVENT_NAME"),
+                string_column("DEFINER"),
+                string_column("TIME_ZONE"),
+                string_column("EVENT_BODY"),
+                string_column("EVENT_DEFINITION"),
+                string_column("EVENT_TYPE"),
+                datetime_column("EXECUTE_AT"),
+                bigint_column("INTERVAL_VALUE"),
+                string_column("INTERVAL_FIELD"),
+                string_column("SQL_MODE"),
+                datetime_column("STARTS"),
+                datetime_column("ENDS"),
+                string_column("STATUS"),
+                string_column("ON_COMPLETION"),
+                datetime_column("CREATED"),
+                datetime_column("LAST_ALTERED"),
+                datetime_column("LAST_EXECUTED"),
+                string_column("EVENT_COMMENT"),
+                bigint_column("ORIGINATOR"),
+                string_column("CHARACTER_SET_CLIENT"),
+                string_column("COLLATION_CONNECTION"),
+                string_column("DATABASE_COLLATION"),
+            ],
+            vec![],
+        ),
+
+        FILES => (
+            vec![
+                bigint_column("FILE_ID"),
+                string_column("FILE_NAME"),
+                string_column("FILE_TYPE"),
+                string_column("TABLESPACE_NAME"),
+                string_column("TABLE_CATALOG"),
+                string_column("TABLE_SCHEMA"),
+                string_column("TABLE_NAME"),
+                string_column("LOGFILE_GROUP_NAME"),
+                bigint_column("LOGFILE_GROUP_NUMBER"),
+                string_column("ENGINE"),
+                string_column("FULLTEXT_KEYS"),
+                bigint_column("DELETED_ROWS"),
+                bigint_column("UPDATE_COUNT"),
+                bigint_column("FREE_EXTENTS"),
+                bigint_column("TOTAL_EXTENTS"),
+                bigint_column("EXTENT_SIZE"),
+                bigint_column("INITIAL_SIZE"),
+                bigint_column("MAXIMUM_SIZE"),
+                bigint_column("AUTOEXTEND_SIZE"),
+                datetime_column("CREATION_TIME"),
+                datetime_column("LAST_UPDATE_TIME"),
+                datetime_column("LAST_ACCESS_TIME"),
+                datetime_column("RECOVER_TIME"),
+                bigint_column("TRANSACTION_COUNTER"),
+                string_column("VERSION"),
+                string_column("ROW_FORMAT"),
+                bigint_column("TABLE_ROWS"),
+                bigint_column("AVG_ROW_LENGTH"),
+                bigint_column("DATA_LENGTH"),
+                bigint_column("MAX_DATA_LENGTH"),
+                bigint_column("INDEX_LENGTH"),
+                bigint_column("DATA_FREE"),
+                datetime_column("CREATE_TIME"),
+                datetime_column("UPDATE_TIME"),
+                datetime_column("CHECK_TIME"),
+                string_column("CHECKSUM"),
+                string_column("STATUS"),
+                string_column("EXTRA"),
+            ],
+            vec![],
+        ),
+
+        OPTIMIZER_TRACE => (
+            vec![
+                string_column("QUERY"),
+                string_column("TRACE"),
+                bigint_column("MISSING_BYTES_BEYOND_MAX_MEM_SIZE"),
+                bigint_column("INSUFFICIENT_PRIVILEGES"),
+            ],
+            vec![],
+        ),
+
+        // MySQL(https://dev.mysql.com/doc/refman/8.2/en/information-schema-parameters-table.html)
+        // has the spec that is different from
+        // PostgreSQL(https://www.postgresql.org/docs/current/infoschema-parameters.html).
+        // Follow `MySQL` spec here.
+        PARAMETERS => (
+            vec![
+                string_column("SPECIFIC_CATALOG"),
+                string_column("SPECIFIC_SCHEMA"),
+                string_column("SPECIFIC_NAME"),
+                bigint_column("ORDINAL_POSITION"),
+                string_column("PARAMETER_MODE"),
+                string_column("PARAMETER_NAME"),
+                string_column("DATA_TYPE"),
+                bigint_column("CHARACTER_MAXIMUM_LENGTH"),
+                bigint_column("CHARACTER_OCTET_LENGTH"),
+                bigint_column("NUMERIC_PRECISION"),
+                bigint_column("NUMERIC_SCALE"),
+                bigint_column("DATETIME_PRECISION"),
+                string_column("CHARACTER_SET_NAME"),
+                string_column("COLLATION_NAME"),
+                string_column("DTD_IDENTIFIER"),
+                string_column("ROUTINE_TYPE"),
+            ],
+            vec![],
+        ),
+
+        PROFILING => (
+            vec![
+                bigint_column("QUERY_ID"),
+                bigint_column("SEQ"),
+                string_column("STATE"),
+                bigint_column("DURATION"),
+                bigint_column("CPU_USER"),
+                bigint_column("CPU_SYSTEM"),
+                bigint_column("CONTEXT_VOLUNTARY"),
+                bigint_column("CONTEXT_INVOLUNTARY"),
+                bigint_column("BLOCK_OPS_IN"),
+                bigint_column("BLOCK_OPS_OUT"),
+                bigint_column("MESSAGES_SENT"),
+                bigint_column("MESSAGES_RECEIVED"),
+                bigint_column("PAGE_FAULTS_MAJOR"),
+                bigint_column("PAGE_FAULTS_MINOR"),
+                bigint_column("SWAPS"),
+                string_column("SOURCE_FUNCTION"),
+                string_column("SOURCE_FILE"),
+                bigint_column("SOURCE_LINE"),
+            ],
+            vec![],
+        ),
+
+        // TODO: _Must_ reimplement this table when foreign key constraint is supported.
+        REFERENTIAL_CONSTRAINTS => (
+            vec![
+                string_column("CONSTRAINT_CATALOG"),
+                string_column("CONSTRAINT_SCHEMA"),
+                string_column("CONSTRAINT_NAME"),
+                string_column("UNIQUE_CONSTRAINT_CATALOG"),
+                string_column("UNIQUE_CONSTRAINT_SCHEMA"),
+                string_column("UNIQUE_CONSTRAINT_NAME"),
+                string_column("MATCH_OPTION"),
+                string_column("UPDATE_RULE"),
+                string_column("DELETE_RULE"),
+                string_column("TABLE_NAME"),
+                string_column("REFERENCED_TABLE_NAME"),
+            ],
+            vec![],
+        ),
+
+        ROUTINES => (
+            vec![
+                string_column("SPECIFIC_NAME"),
+                string_column("ROUTINE_CATALOG"),
+                string_column("ROUTINE_SCHEMA"),
+                string_column("ROUTINE_NAME"),
+                string_column("ROUTINE_TYPE"),
+                string_column("DATA_TYPE"),
+                bigint_column("CHARACTER_MAXIMUM_LENGTH"),
+                bigint_column("CHARACTER_OCTET_LENGTH"),
+                bigint_column("NUMERIC_PRECISION"),
+                bigint_column("NUMERIC_SCALE"),
+                bigint_column("DATETIME_PRECISION"),
+                string_column("CHARACTER_SET_NAME"),
+                string_column("COLLATION_NAME"),
+                string_column("DTD_IDENTIFIER"),
+                string_column("ROUTINE_BODY"),
+                string_column("ROUTINE_DEFINITION"),
+                string_column("EXTERNAL_NAME"),
+                string_column("EXTERNAL_LANGUAGE"),
+                string_column("PARAMETER_STYLE"),
+                string_column("IS_DETERMINISTIC"),
+                string_column("SQL_DATA_ACCESS"),
+                string_column("SQL_PATH"),
+                string_column("SECURITY_TYPE"),
+                datetime_column("CREATED"),
+                datetime_column("LAST_ALTERED"),
+                string_column("SQL_MODE"),
+                string_column("ROUTINE_COMMENT"),
+                string_column("DEFINER"),
+                string_column("CHARACTER_SET_CLIENT"),
+                string_column("COLLATION_CONNECTION"),
+                string_column("DATABASE_COLLATION"),
+            ],
+            vec![],
+        ),
+
+        SCHEMA_PRIVILEGES => (
+            vec![
+                string_column("GRANTEE"),
+                string_column("TABLE_CATALOG"),
+                string_column("TABLE_SCHEMA"),
+                string_column("PRIVILEGE_TYPE"),
+                string_column("IS_GRANTABLE"),
+            ],
+            vec![],
+        ),
+
+        TABLE_PRIVILEGES => (
+            vec![
+                string_column("GRANTEE"),
+                string_column("TABLE_CATALOG"),
+                string_column("TABLE_SCHEMA"),
+                string_column("TABLE_NAME"),
+                string_column("PRIVILEGE_TYPE"),
+                string_column("IS_GRANTABLE"),
+            ],
+            vec![],
+        ),
+
+        TRIGGERS => (
+            vec![
+                string_column("TRIGGER_CATALOG"),
+                string_column("TRIGGER_SCHEMA"),
+                string_column("TRIGGER_NAME"),
+                string_column("EVENT_MANIPULATION"),
+                string_column("EVENT_OBJECT_CATALOG"),
+                string_column("EVENT_OBJECT_SCHEMA"),
+                string_column("EVENT_OBJECT_TABLE"),
+                bigint_column("ACTION_ORDER"),
+                string_column("ACTION_CONDITION"),
+                string_column("ACTION_STATEMENT"),
+                string_column("ACTION_ORIENTATION"),
+                string_column("ACTION_TIMING"),
+                string_column("ACTION_REFERENCE_OLD_TABLE"),
+                string_column("ACTION_REFERENCE_NEW_TABLE"),
+                string_column("ACTION_REFERENCE_OLD_ROW"),
+                string_column("ACTION_REFERENCE_NEW_ROW"),
+                datetime_column("CREATED"),
+                string_column("SQL_MODE"),
+                string_column("DEFINER"),
+                string_column("CHARACTER_SET_CLIENT"),
+                string_column("COLLATION_CONNECTION"),
+                string_column("DATABASE_COLLATION"),
+            ],
+            vec![],
+        ),
+
+        // TODO: Considering store internal metrics in `global_status` and
+        // `session_status` tables.
+        GLOBAL_STATUS => (
+            vec![
+                string_column("VARIABLE_NAME"),
+                string_column("VARIABLE_VALUE"),
+            ],
+            vec![],
+        ),
+
+        SESSION_STATUS => (
+            vec![
+                string_column("VARIABLE_NAME"),
+                string_column("VARIABLE_VALUE"),
+            ],
+            vec![],
+        ),
+
         _ => unreachable!("Unknown table in information_schema: {}", table_name),
     };
 
@@ -115,6 +429,22 @@ fn string_column(name: &str) -> ColumnSchema {
     )
 }
 
+fn bigint_column(name: &str) -> ColumnSchema {
+    ColumnSchema::new(
+        str::to_lowercase(name),
+        ConcreteDataType::int64_datatype(),
+        false,
+    )
+}
+
+fn datetime_column(name: &str) -> ColumnSchema {
+    ColumnSchema::new(
+        str::to_lowercase(name),
+        ConcreteDataType::datetime_datatype(),
+        false,
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/catalog/src/information_schema/predicate.rs

@@ -0,0 +1,609 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use arrow::array::StringArray;
+use arrow::compute::kernels::comparison;
+use common_query::logical_plan::DfExpr;
+use datafusion::common::ScalarValue;
+use datafusion::logical_expr::expr::Like;
+use datafusion::logical_expr::Operator;
+use datatypes::value::Value;
+use store_api::storage::ScanRequest;
+
+type ColumnName = String;
+/// Predicate to filter `information_schema` tables stream,
+/// we only support these simple predicates currently.
+/// TODO(dennis): supports more predicate types.
+#[derive(Clone, PartialEq, Eq, Debug)]
+enum Predicate {
+    Eq(ColumnName, Value),
+    Like(ColumnName, String, bool),
+    NotEq(ColumnName, Value),
+    InList(ColumnName, Vec<Value>),
+    And(Box<Predicate>, Box<Predicate>),
+    Or(Box<Predicate>, Box<Predicate>),
+    Not(Box<Predicate>),
+}
+
+impl Predicate {
+    /// Evaluate the predicate with the row, returns:
+    /// - `None` when the predicate can't evaluate with the row.
+    /// - `Some(true)` when the predicate is satisfied,
+    /// - `Some(false)` when the predicate is not satisfied,
+    fn eval(&self, row: &[(&str, &Value)]) -> Option<bool> {
+        match self {
+            Predicate::Eq(c, v) => {
+                for (column, value) in row {
+                    if c != column {
+                        continue;
+                    }
+                    return Some(v == *value);
+                }
+            }
+            Predicate::Like(c, pattern, case_insensitive) => {
+                for (column, value) in row {
+                    if c != column {
+                        continue;
+                    }
+
+                    let Value::String(bs) = value else {
+                        continue;
+                    };
+
+                    return like_utf8(bs.as_utf8(), pattern, case_insensitive);
+                }
+            }
+            Predicate::NotEq(c, v) => {
+                for (column, value) in row {
+                    if c != column {
+                        continue;
+                    }
+                    return Some(v != *value);
+                }
+            }
+            Predicate::InList(c, values) => {
+                for (column, value) in row {
+                    if c != column {
+                        continue;
+                    }
+                    return Some(values.iter().any(|v| v == *value));
+                }
+            }
+            Predicate::And(left, right) => {
+                let left = left.eval(row);
+
+                // short-circuit
+                if matches!(left, Some(false)) {
+                    return Some(false);
+                }
+
+                return match (left, right.eval(row)) {
+                    (Some(left), Some(right)) => Some(left && right),
+                    (None, Some(false)) => Some(false),
+                    _ => None,
+                };
+            }
+            Predicate::Or(left, right) => {
+                let left = left.eval(row);
+
+                // short-circuit
+                if matches!(left, Some(true)) {
+                    return Some(true);
+                }
+
+                return match (left, right.eval(row)) {
+                    (Some(left), Some(right)) => Some(left || right),
+                    (None, Some(true)) => Some(true),
+                    _ => None,
+                };
+            }
+            Predicate::Not(p) => {
+                let Some(b) = p.eval(row) else {
+                    return None;
+                };
+
+                return Some(!b);
+            }
+        }
+
+        // Can't evaluate predicate with the row
+        None
+    }
+
+    /// Try to create a predicate from datafusion [`Expr`], return None if fails.
+    fn from_expr(expr: DfExpr) -> Option<Predicate> {
+        match expr {
+            // NOT expr
+            DfExpr::Not(expr) => {
+                let Some(p) = Self::from_expr(*expr) else {
+                    return None;
+                };
+
+                Some(Predicate::Not(Box::new(p)))
+            }
+            // expr LIKE pattern
+            DfExpr::Like(Like {
+                negated,
+                expr,
+                pattern,
+                case_insensitive,
+                ..
+            }) if is_column(&expr) && is_string_literal(&pattern) => {
+                // Safety: ensured by gurad
+                let DfExpr::Column(c) = *expr else {
+                    unreachable!();
+                };
+                let DfExpr::Literal(ScalarValue::Utf8(Some(pattern))) = *pattern else {
+                    unreachable!();
+                };
+
+                let p = Predicate::Like(c.name, pattern, case_insensitive);
+
+                if negated {
+                    Some(Predicate::Not(Box::new(p)))
+                } else {
+                    Some(p)
+                }
+            }
+            // left OP right
+            DfExpr::BinaryExpr(bin) => match (*bin.left, bin.op, *bin.right) {
+                // left == right
+                (DfExpr::Literal(scalar), Operator::Eq, DfExpr::Column(c))
+                | (DfExpr::Column(c), Operator::Eq, DfExpr::Literal(scalar)) => {
+                    let Ok(v) = Value::try_from(scalar) else {
+                        return None;
+                    };
+
+                    Some(Predicate::Eq(c.name, v))
+                }
+                // left != right
+                (DfExpr::Literal(scalar), Operator::NotEq, DfExpr::Column(c))
+                | (DfExpr::Column(c), Operator::NotEq, DfExpr::Literal(scalar)) => {
+                    let Ok(v) = Value::try_from(scalar) else {
+                        return None;
+                    };
+
+                    Some(Predicate::NotEq(c.name, v))
+                }
+                // left AND right
+                (left, Operator::And, right) => {
+                    let Some(left) = Self::from_expr(left) else {
+                        return None;
+                    };
+
+                    let Some(right) = Self::from_expr(right) else {
+                        return None;
+                    };
+
+                    Some(Predicate::And(Box::new(left), Box::new(right)))
+                }
+                // left OR right
+                (left, Operator::Or, right) => {
+                    let Some(left) = Self::from_expr(left) else {
+                        return None;
+                    };
+
+                    let Some(right) = Self::from_expr(right) else {
+                        return None;
+                    };
+
+                    Some(Predicate::Or(Box::new(left), Box::new(right)))
+                }
+                _ => None,
+            },
+            // [NOT] IN (LIST)
+            DfExpr::InList(list) => {
+                match (*list.expr, list.list, list.negated) {
+                    // column [NOT] IN (v1, v2, v3, ...)
+                    (DfExpr::Column(c), list, negated) if is_all_scalars(&list) => {
+                        let mut values = Vec::with_capacity(list.len());
+                        for scalar in list {
+                            // Safety: checked by `is_all_scalars`
+                            let DfExpr::Literal(scalar) = scalar else {
+                                unreachable!();
+                            };
+
+                            let Ok(value) = Value::try_from(scalar) else {
+                                return None;
+                            };
+
+                            values.push(value);
+                        }
+
+                        let predicate = Predicate::InList(c.name, values);
+
+                        if negated {
+                            Some(Predicate::Not(Box::new(predicate)))
+                        } else {
+                            Some(predicate)
+                        }
+                    }
+                    _ => None,
+                }
+            }
+            _ => None,
+        }
+    }
+}
+
+/// Perform SQL left LIKE right, return `None` if fail to evaluate.
+/// - `s` the target string
+/// - `pattern` the pattern just like '%abc'
+/// - `case_insensitive` whether to perform case-insensitive like or not.
+fn like_utf8(s: &str, pattern: &str, case_insensitive: &bool) -> Option<bool> {
+    let array = StringArray::from(vec![s]);
+    let patterns = StringArray::new_scalar(pattern);
+
+    let Ok(booleans) = (if *case_insensitive {
+        comparison::ilike(&array, &patterns)
+    } else {
+        comparison::like(&array, &patterns)
+    }) else {
+        return None;
+    };
+
+    // Safety: at least one value in result
+    Some(booleans.value(0))
+}
+
+fn is_string_literal(expr: &DfExpr) -> bool {
+    matches!(expr, DfExpr::Literal(ScalarValue::Utf8(Some(_))))
+}
+
+fn is_column(expr: &DfExpr) -> bool {
+    matches!(expr, DfExpr::Column(_))
+}
+
+/// A list of predicate
+pub struct Predicates {
+    predicates: Vec<Predicate>,
+}
+
+impl Predicates {
+    /// Try its best to create predicates from [`ScanRequest`].
+    pub fn from_scan_request(request: &Option<ScanRequest>) -> Predicates {
+        if let Some(request) = request {
+            let mut predicates = Vec::with_capacity(request.filters.len());
+
+            for filter in &request.filters {
+                if let Some(predicate) = Predicate::from_expr(filter.df_expr().clone()) {
+                    predicates.push(predicate);
+                }
+            }
+
+            Self { predicates }
+        } else {
+            Self {
+                predicates: Vec::new(),
+            }
+        }
+    }
+
+    /// Evaluate the predicates with the row.
+    /// returns true when all the predicates are satisfied or can't be evaluated.
+    pub fn eval(&self, row: &[(&str, &Value)]) -> bool {
+        // fast path
+        if self.predicates.is_empty() {
+            return true;
+        }
+
+        self.predicates
+            .iter()
+            .filter_map(|p| p.eval(row))
+            .all(|b| b)
+    }
+}
+
+/// Returns true when the values are all [`DfExpr::Literal`].
+fn is_all_scalars(list: &[DfExpr]) -> bool {
+    list.iter().all(|v| matches!(v, DfExpr::Literal(_)))
+}
+
+#[cfg(test)]
+mod tests {
+    use datafusion::common::{Column, ScalarValue};
+    use datafusion::logical_expr::expr::InList;
+    use datafusion::logical_expr::BinaryExpr;
+
+    use super::*;
+
+    #[test]
+    fn test_predicate_eval() {
+        let a_col = "a".to_string();
+        let b_col = "b".to_string();
+        let a_value = Value::from("a_value");
+        let b_value = Value::from("b_value");
+        let wrong_value = Value::from("wrong_value");
+
+        let a_row = [(a_col.as_str(), &a_value)];
+        let b_row = [("b", &wrong_value)];
+        let wrong_row = [(a_col.as_str(), &wrong_value)];
+
+        // Predicate::Eq
+        let p = Predicate::Eq(a_col.clone(), a_value.clone());
+        assert!(p.eval(&a_row).unwrap());
+        assert!(p.eval(&b_row).is_none());
+        assert!(!p.eval(&wrong_row).unwrap());
+
+        // Predicate::NotEq
+        let p = Predicate::NotEq(a_col.clone(), a_value.clone());
+        assert!(!p.eval(&a_row).unwrap());
+        assert!(p.eval(&b_row).is_none());
+        assert!(p.eval(&wrong_row).unwrap());
+
+        // Predicate::InList
+        let p = Predicate::InList(a_col.clone(), vec![a_value.clone(), b_value.clone()]);
+        assert!(p.eval(&a_row).unwrap());
+        assert!(p.eval(&b_row).is_none());
+        assert!(!p.eval(&wrong_row).unwrap());
+        assert!(p.eval(&[(&a_col, &b_value)]).unwrap());
+
+        let p1 = Predicate::Eq(a_col.clone(), a_value.clone());
+        let p2 = Predicate::Eq(b_col.clone(), b_value.clone());
+        let row = [(a_col.as_str(), &a_value), (b_col.as_str(), &b_value)];
+        let wrong_row = [(a_col.as_str(), &a_value), (b_col.as_str(), &wrong_value)];
+
+        //Predicate::And
+        let p = Predicate::And(Box::new(p1.clone()), Box::new(p2.clone()));
+        assert!(p.eval(&row).unwrap());
+        assert!(!p.eval(&wrong_row).unwrap());
+        assert!(p.eval(&[]).is_none());
+        assert!(p.eval(&[("c", &a_value)]).is_none());
+        assert!(!p
+            .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &a_value)])
+            .unwrap());
+        assert!(!p
+            .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &b_value)])
+            .unwrap());
+        assert!(p
+            .eval(&[(a_col.as_ref(), &a_value), ("c", &a_value)])
+            .is_none());
+        assert!(!p
+            .eval(&[(a_col.as_ref(), &b_value), ("c", &a_value)])
+            .unwrap());
+
+        //Predicate::Or
+        let p = Predicate::Or(Box::new(p1), Box::new(p2));
+        assert!(p.eval(&row).unwrap());
+        assert!(p.eval(&wrong_row).unwrap());
+        assert!(p.eval(&[]).is_none());
+        assert!(p.eval(&[("c", &a_value)]).is_none());
+        assert!(!p
+            .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &a_value)])
+            .unwrap());
+        assert!(p
+            .eval(&[(a_col.as_str(), &b_value), (b_col.as_str(), &b_value)])
+            .unwrap());
+        assert!(p
+            .eval(&[(a_col.as_ref(), &a_value), ("c", &a_value)])
+            .unwrap());
+        assert!(p
+            .eval(&[(a_col.as_ref(), &b_value), ("c", &a_value)])
+            .is_none());
+    }
+
+    #[test]
+    fn test_predicate_like() {
+        // case insensitive
+        let expr = DfExpr::Like(Like {
+            negated: false,
+            expr: Box::new(column("a")),
+            pattern: Box::new(string_literal("%abc")),
+            case_insensitive: true,
+            escape_char: None,
+        });
+
+        let p = Predicate::from_expr(expr).unwrap();
+        assert!(
+            matches!(&p, Predicate::Like(c, pattern, case_insensitive) if
+                         c == "a"
+                         && pattern == "%abc"
+                         && *case_insensitive)
+        );
+
+        let match_row = [
+            ("a", &Value::from("hello AbC")),
+            ("b", &Value::from("b value")),
+        ];
+        let unmatch_row = [("a", &Value::from("bca")), ("b", &Value::from("b value"))];
+
+        assert!(p.eval(&match_row).unwrap());
+        assert!(!p.eval(&unmatch_row).unwrap());
+        assert!(p.eval(&[]).is_none());
+
+        // case sensitive
+        let expr = DfExpr::Like(Like {
+            negated: false,
+            expr: Box::new(column("a")),
+            pattern: Box::new(string_literal("%abc")),
+            case_insensitive: false,
+            escape_char: None,
+        });
+
+        let p = Predicate::from_expr(expr).unwrap();
+        assert!(
+            matches!(&p, Predicate::Like(c, pattern, case_insensitive) if
+                         c == "a"
+                         && pattern == "%abc"
+                         && !*case_insensitive)
+        );
+        assert!(!p.eval(&match_row).unwrap());
+        assert!(!p.eval(&unmatch_row).unwrap());
+        assert!(p.eval(&[]).is_none());
+
+        // not like
+        let expr = DfExpr::Like(Like {
+            negated: true,
+            expr: Box::new(column("a")),
+            pattern: Box::new(string_literal("%abc")),
+            case_insensitive: true,
+            escape_char: None,
+        });
+
+        let p = Predicate::from_expr(expr).unwrap();
+        assert!(!p.eval(&match_row).unwrap());
+        assert!(p.eval(&unmatch_row).unwrap());
+        assert!(p.eval(&[]).is_none());
+    }
+
+    fn column(name: &str) -> DfExpr {
+        DfExpr::Column(Column {
+            relation: None,
+            name: name.to_string(),
+        })
+    }
+
+    fn string_literal(v: &str) -> DfExpr {
+        DfExpr::Literal(ScalarValue::Utf8(Some(v.to_string())))
+    }
+
+    fn match_string_value(v: &Value, expected: &str) -> bool {
+        matches!(v, Value::String(bs) if bs.as_utf8() == expected)
+    }
+
+    fn match_string_values(vs: &[Value], expected: &[&str]) -> bool {
+        assert_eq!(vs.len(), expected.len());
+
+        let mut result = true;
+        for (i, v) in vs.iter().enumerate() {
+            result = result && match_string_value(v, expected[i]);
+        }
+
+        result
+    }
+
+    fn mock_exprs() -> (DfExpr, DfExpr) {
+        let expr1 = DfExpr::BinaryExpr(BinaryExpr {
+            left: Box::new(column("a")),
+            op: Operator::Eq,
+            right: Box::new(string_literal("a_value")),
+        });
+
+        let expr2 = DfExpr::BinaryExpr(BinaryExpr {
+            left: Box::new(column("b")),
+            op: Operator::NotEq,
+            right: Box::new(string_literal("b_value")),
+        });
+
+        (expr1, expr2)
+    }
+
+    #[test]
+    fn test_predicate_from_expr() {
+        let (expr1, expr2) = mock_exprs();
+
+        let p1 = Predicate::from_expr(expr1.clone()).unwrap();
+        assert!(matches!(&p1, Predicate::Eq(column, v) if column == "a"
+                         && match_string_value(v, "a_value")));
+
+        let p2 = Predicate::from_expr(expr2.clone()).unwrap();
+        assert!(matches!(&p2, Predicate::NotEq(column, v) if column == "b"
+                         && match_string_value(v, "b_value")));
+
+        let and_expr = DfExpr::BinaryExpr(BinaryExpr {
+            left: Box::new(expr1.clone()),
+            op: Operator::And,
+            right: Box::new(expr2.clone()),
+        });
+        let or_expr = DfExpr::BinaryExpr(BinaryExpr {
+            left: Box::new(expr1.clone()),
+            op: Operator::Or,
+            right: Box::new(expr2.clone()),
+        });
+        let not_expr = DfExpr::Not(Box::new(expr1.clone()));
+
+        let and_p = Predicate::from_expr(and_expr).unwrap();
+        assert!(matches!(and_p, Predicate::And(left, right) if *left == p1 && *right == p2));
+        let or_p = Predicate::from_expr(or_expr).unwrap();
+        assert!(matches!(or_p, Predicate::Or(left, right) if *left == p1 && *right == p2));
+        let not_p = Predicate::from_expr(not_expr).unwrap();
+        assert!(matches!(not_p, Predicate::Not(p) if *p == p1));
+
+        let inlist_expr = DfExpr::InList(InList {
+            expr: Box::new(column("a")),
+            list: vec![string_literal("a1"), string_literal("a2")],
+            negated: false,
+        });
+
+        let inlist_p = Predicate::from_expr(inlist_expr).unwrap();
+        assert!(matches!(&inlist_p, Predicate::InList(c, values) if c == "a"
+                         && match_string_values(values, &["a1", "a2"])));
+
+        let inlist_expr = DfExpr::InList(InList {
+            expr: Box::new(column("a")),
+            list: vec![string_literal("a1"), string_literal("a2")],
+            negated: true,
+        });
+        let inlist_p = Predicate::from_expr(inlist_expr).unwrap();
+        assert!(matches!(inlist_p, Predicate::Not(p) if
+                         matches!(&*p,
+                                  Predicate::InList(c, values) if c == "a"
+                                  && match_string_values(values, &["a1", "a2"]))));
+    }
+
+    #[test]
+    fn test_predicates_from_scan_request() {
+        let predicates = Predicates::from_scan_request(&None);
+        assert!(predicates.predicates.is_empty());
+
+        let (expr1, expr2) = mock_exprs();
+
+        let request = ScanRequest {
+            filters: vec![expr1.into(), expr2.into()],
+            ..Default::default()
+        };
+        let predicates = Predicates::from_scan_request(&Some(request));
+
+        assert_eq!(2, predicates.predicates.len());
+        assert!(
+            matches!(&predicates.predicates[0], Predicate::Eq(column, v) if column == "a"
+                     && match_string_value(v, "a_value"))
+        );
+        assert!(
+            matches!(&predicates.predicates[1], Predicate::NotEq(column, v) if column == "b"
+                     && match_string_value(v, "b_value"))
+        );
+    }
+
+    #[test]
+    fn test_predicates_eval_row() {
+        let wrong_row = [
+            ("a", &Value::from("a_value")),
+            ("b", &Value::from("b_value")),
+            ("c", &Value::from("c_value")),
+        ];
+        let row = [
+            ("a", &Value::from("a_value")),
+            ("b", &Value::from("not_b_value")),
+            ("c", &Value::from("c_value")),
+        ];
+        let c_row = [("c", &Value::from("c_value"))];
+
+        // test empty predicates, always returns true
+        let predicates = Predicates::from_scan_request(&None);
+        assert!(predicates.eval(&row));
+        assert!(predicates.eval(&wrong_row));
+        assert!(predicates.eval(&c_row));
+
+        let (expr1, expr2) = mock_exprs();
+        let request = ScanRequest {
+            filters: vec![expr1.into(), expr2.into()],
+            ..Default::default()
+        };
+        let predicates = Predicates::from_scan_request(&Some(request));
+        assert!(predicates.eval(&row));
+        assert!(!predicates.eval(&wrong_row));
+        assert!(predicates.eval(&c_row));
+    }
+}

src/catalog/src/information_schema/runtime_metrics.rs

@@ -0,0 +1,250 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_catalog::consts::INFORMATION_SCHEMA_RUNTIME_METRICS_TABLE_ID;
+use common_error::ext::BoxedError;
+use common_query::physical_plan::TaskContext;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use common_time::util::current_time_millis;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
+use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
+use datatypes::prelude::{ConcreteDataType, MutableVector};
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::vectors::{
+    ConstantVector, Float64VectorBuilder, StringVector, StringVectorBuilder,
+    TimestampMillisecondVector, VectorRef,
+};
+use itertools::Itertools;
+use snafu::ResultExt;
+use store_api::storage::{ScanRequest, TableId};
+
+use super::{InformationTable, RUNTIME_METRICS};
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
+
+pub(super) struct InformationSchemaMetrics {
+    schema: SchemaRef,
+}
+
+const METRIC_NAME: &str = "metric_name";
+const METRIC_VALUE: &str = "value";
+const METRIC_LABELS: &str = "labels";
+const NODE: &str = "node";
+const NODE_TYPE: &str = "node_type";
+const TIMESTAMP: &str = "timestamp";
+
+/// The `information_schema.runtime_metrics` virtual table.
+/// It provides the GreptimeDB runtime metrics for the users by SQL.
+impl InformationSchemaMetrics {
+    pub(super) fn new() -> Self {
+        Self {
+            schema: Self::schema(),
+        }
+    }
+
+    fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
+            ColumnSchema::new(METRIC_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(METRIC_VALUE, ConcreteDataType::float64_datatype(), false),
+            ColumnSchema::new(METRIC_LABELS, ConcreteDataType::string_datatype(), true),
+            ColumnSchema::new(NODE, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(NODE_TYPE, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(
+                TIMESTAMP,
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                false,
+            ),
+        ]))
+    }
+
+    fn builder(&self) -> InformationSchemaMetricsBuilder {
+        InformationSchemaMetricsBuilder::new(self.schema.clone())
+    }
+}
+
+impl InformationTable for InformationSchemaMetrics {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_RUNTIME_METRICS_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        RUNTIME_METRICS
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_metrics(Some(request))
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+struct InformationSchemaMetricsBuilder {
+    schema: SchemaRef,
+
+    metric_names: StringVectorBuilder,
+    metric_values: Float64VectorBuilder,
+    metric_labels: StringVectorBuilder,
+}
+
+impl InformationSchemaMetricsBuilder {
+    fn new(schema: SchemaRef) -> Self {
+        Self {
+            schema,
+            metric_names: StringVectorBuilder::with_capacity(42),
+            metric_values: Float64VectorBuilder::with_capacity(42),
+            metric_labels: StringVectorBuilder::with_capacity(42),
+        }
+    }
+
+    fn add_metric(&mut self, metric_name: &str, labels: String, metric_value: f64) {
+        self.metric_names.push(Some(metric_name));
+        self.metric_values.push(Some(metric_value));
+        self.metric_labels.push(Some(&labels));
+    }
+
+    async fn make_metrics(&mut self, _request: Option<ScanRequest>) -> Result<RecordBatch> {
+        let metric_families = prometheus::gather();
+
+        let write_request =
+            common_telemetry::metric::convert_metric_to_write_request(metric_families, None, 0);
+
+        for ts in write_request.timeseries {
+            //Safety: always has `__name__` label
+            let metric_name = ts
+                .labels
+                .iter()
+                .find_map(|label| {
+                    if label.name == "__name__" {
+                        Some(label.value.clone())
+                    } else {
+                        None
+                    }
+                })
+                .unwrap();
+
+            self.add_metric(
+                &metric_name,
+                ts.labels
+                    .into_iter()
+                    .filter_map(|label| {
+                        if label.name == "__name__" {
+                            None
+                        } else {
+                            Some(format!("{}={}", label.name, label.value))
+                        }
+                    })
+                    .join(", "),
+                // Safety: always has a sample
+                ts.samples[0].value,
+            );
+        }
+
+        self.finish()
+    }
+
+    fn finish(&mut self) -> Result<RecordBatch> {
+        let rows_num = self.metric_names.len();
+        let unknowns = Arc::new(ConstantVector::new(
+            Arc::new(StringVector::from(vec!["unknown"])),
+            rows_num,
+        ));
+        let timestamps = Arc::new(ConstantVector::new(
+            Arc::new(TimestampMillisecondVector::from_slice([
+                current_time_millis(),
+            ])),
+            rows_num,
+        ));
+
+        let columns: Vec<VectorRef> = vec![
+            Arc::new(self.metric_names.finish()),
+            Arc::new(self.metric_values.finish()),
+            Arc::new(self.metric_labels.finish()),
+            // TODO(dennis): supports node and node_type for cluster
+            unknowns.clone(),
+            unknowns,
+            timestamps,
+        ];
+
+        RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
+    }
+}
+
+impl DfPartitionStream for InformationSchemaMetrics {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_metrics(None)
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use common_recordbatch::RecordBatches;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_make_metrics() {
+        let metrics = InformationSchemaMetrics::new();
+
+        let stream = metrics.to_stream(ScanRequest::default()).unwrap();
+
+        let batches = RecordBatches::try_collect(stream).await.unwrap();
+
+        let result_literal = batches.pretty_print().unwrap();
+
+        assert!(result_literal.contains(METRIC_NAME));
+        assert!(result_literal.contains(METRIC_VALUE));
+        assert!(result_literal.contains(METRIC_LABELS));
+        assert!(result_literal.contains(NODE));
+        assert!(result_literal.contains(NODE_TYPE));
+        assert!(result_literal.contains(TIMESTAMP));
+    }
+}

src/catalog/src/information_schema/schemata.rs

@@ -0,0 +1,228 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::{Arc, Weak};
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_catalog::consts::INFORMATION_SCHEMA_SCHEMATA_TABLE_ID;
+use common_error::ext::BoxedError;
+use common_query::physical_plan::TaskContext;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
+use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
+use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::value::Value;
+use datatypes::vectors::StringVectorBuilder;
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::{ScanRequest, TableId};
+
+use super::SCHEMATA;
+use crate::error::{
+    CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
+};
+use crate::information_schema::{InformationTable, Predicates};
+use crate::CatalogManager;
+
+const CATALOG_NAME: &str = "catalog_name";
+const SCHEMA_NAME: &str = "schema_name";
+const DEFAULT_CHARACTER_SET_NAME: &str = "default_character_set_name";
+const DEFAULT_COLLATION_NAME: &str = "default_collation_name";
+
+/// The `information_schema.schemata` table implementation.
+pub(super) struct InformationSchemaSchemata {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_manager: Weak<dyn CatalogManager>,
+}
+
+impl InformationSchemaSchemata {
+    pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
+        Self {
+            schema: Self::schema(),
+            catalog_name,
+            catalog_manager,
+        }
+    }
+
+    pub(crate) fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
+            ColumnSchema::new(CATALOG_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(SCHEMA_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(
+                DEFAULT_CHARACTER_SET_NAME,
+                ConcreteDataType::string_datatype(),
+                false,
+            ),
+            ColumnSchema::new(
+                DEFAULT_COLLATION_NAME,
+                ConcreteDataType::string_datatype(),
+                false,
+            ),
+            ColumnSchema::new("sql_path", ConcreteDataType::string_datatype(), true),
+        ]))
+    }
+
+    fn builder(&self) -> InformationSchemaSchemataBuilder {
+        InformationSchemaSchemataBuilder::new(
+            self.schema.clone(),
+            self.catalog_name.clone(),
+            self.catalog_manager.clone(),
+        )
+    }
+}
+
+impl InformationTable for InformationSchemaSchemata {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_SCHEMATA_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        SCHEMATA
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_schemata(Some(request))
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+/// Builds the `information_schema.schemata` table row by row
+///
+/// Columns are based on <https://docs.pingcap.com/tidb/stable/information-schema-schemata>
+struct InformationSchemaSchemataBuilder {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_manager: Weak<dyn CatalogManager>,
+
+    catalog_names: StringVectorBuilder,
+    schema_names: StringVectorBuilder,
+    charset_names: StringVectorBuilder,
+    collation_names: StringVectorBuilder,
+    sql_paths: StringVectorBuilder,
+}
+
+impl InformationSchemaSchemataBuilder {
+    fn new(
+        schema: SchemaRef,
+        catalog_name: String,
+        catalog_manager: Weak<dyn CatalogManager>,
+    ) -> Self {
+        Self {
+            schema,
+            catalog_name,
+            catalog_manager,
+            catalog_names: StringVectorBuilder::with_capacity(42),
+            schema_names: StringVectorBuilder::with_capacity(42),
+            charset_names: StringVectorBuilder::with_capacity(42),
+            collation_names: StringVectorBuilder::with_capacity(42),
+            sql_paths: StringVectorBuilder::with_capacity(42),
+        }
+    }
+
+    /// Construct the `information_schema.schemata` virtual table
+    async fn make_schemata(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
+        let catalog_name = self.catalog_name.clone();
+        let catalog_manager = self
+            .catalog_manager
+            .upgrade()
+            .context(UpgradeWeakCatalogManagerRefSnafu)?;
+        let predicates = Predicates::from_scan_request(&request);
+
+        for schema_name in catalog_manager.schema_names(&catalog_name).await? {
+            if !catalog_manager
+                .schema_exists(&catalog_name, &schema_name)
+                .await?
+            {
+                continue;
+            }
+
+            self.add_schema(&predicates, &catalog_name, &schema_name);
+        }
+
+        self.finish()
+    }
+
+    fn add_schema(&mut self, predicates: &Predicates, catalog_name: &str, schema_name: &str) {
+        let row = [
+            (CATALOG_NAME, &Value::from(catalog_name)),
+            (SCHEMA_NAME, &Value::from(schema_name)),
+            (DEFAULT_CHARACTER_SET_NAME, &Value::from("utf8")),
+            (DEFAULT_COLLATION_NAME, &Value::from("utf8_bin")),
+        ];
+
+        if !predicates.eval(&row) {
+            return;
+        }
+
+        self.catalog_names.push(Some(catalog_name));
+        self.schema_names.push(Some(schema_name));
+        self.charset_names.push(Some("utf8"));
+        self.collation_names.push(Some("utf8_bin"));
+        self.sql_paths.push(None);
+    }
+
+    fn finish(&mut self) -> Result<RecordBatch> {
+        let columns: Vec<VectorRef> = vec![
+            Arc::new(self.catalog_names.finish()),
+            Arc::new(self.schema_names.finish()),
+            Arc::new(self.charset_names.finish()),
+            Arc::new(self.collation_names.finish()),
+            Arc::new(self.sql_paths.finish()),
+        ];
+        RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
+    }
+}
+
+impl DfPartitionStream for InformationSchemaSchemata {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_schemata(None)
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}

src/catalog/src/information_schema/table_names.rs

@@ -20,3 +20,22 @@ pub const ENGINES: &str = "engines";
 pub const COLUMN_PRIVILEGES: &str = "column_privileges";
 pub const COLUMN_STATISTICS: &str = "column_statistics";
 pub const BUILD_INFO: &str = "build_info";
+pub const CHARACTER_SETS: &str = "character_sets";
+pub const COLLATIONS: &str = "collations";
+pub const COLLATION_CHARACTER_SET_APPLICABILITY: &str = "collation_character_set_applicability";
+pub const CHECK_CONSTRAINTS: &str = "check_constraints";
+pub const EVENTS: &str = "events";
+pub const FILES: &str = "files";
+pub const SCHEMATA: &str = "schemata";
+pub const KEY_COLUMN_USAGE: &str = "key_column_usage";
+pub const OPTIMIZER_TRACE: &str = "optimizer_trace";
+pub const PARAMETERS: &str = "parameters";
+pub const PROFILING: &str = "profiling";
+pub const REFERENTIAL_CONSTRAINTS: &str = "referential_constraints";
+pub const ROUTINES: &str = "routines";
+pub const SCHEMA_PRIVILEGES: &str = "schema_privileges";
+pub const TABLE_PRIVILEGES: &str = "table_privileges";
+pub const TRIGGERS: &str = "triggers";
+pub const GLOBAL_STATUS: &str = "global_status";
+pub const SESSION_STATUS: &str = "session_status";
+pub const RUNTIME_METRICS: &str = "runtime_metrics";

src/catalog/src/information_schema/tables.rs

@@ -25,18 +25,26 @@ use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
 use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::value::Value;
 use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder};
 use snafu::{OptionExt, ResultExt};
-use store_api::storage::TableId;
+use store_api::storage::{ScanRequest, TableId};
 use table::metadata::TableType;
 
 use super::TABLES;
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
-use crate::information_schema::InformationTable;
+use crate::information_schema::{InformationTable, Predicates};
 use crate::CatalogManager;
 
+const TABLE_CATALOG: &str = "table_catalog";
+const TABLE_SCHEMA: &str = "table_schema";
+const TABLE_NAME: &str = "table_name";
+const TABLE_TYPE: &str = "table_type";
+const TABLE_ID: &str = "table_id";
+const ENGINE: &str = "engine";
+
 pub(super) struct InformationSchemaTables {
     schema: SchemaRef,
     catalog_name: String,
@@ -54,12 +62,12 @@ impl InformationSchemaTables {
 
     pub(crate) fn schema() -> SchemaRef {
         Arc::new(Schema::new(vec![
-            ColumnSchema::new("table_catalog", ConcreteDataType::string_datatype(), false),
-            ColumnSchema::new("table_schema", ConcreteDataType::string_datatype(), false),
-            ColumnSchema::new("table_name", ConcreteDataType::string_datatype(), false),
-            ColumnSchema::new("table_type", ConcreteDataType::string_datatype(), false),
-            ColumnSchema::new("table_id", ConcreteDataType::uint32_datatype(), true),
-            ColumnSchema::new("engine", ConcreteDataType::string_datatype(), true),
+            ColumnSchema::new(TABLE_CATALOG, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_SCHEMA, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_TYPE, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(TABLE_ID, ConcreteDataType::uint32_datatype(), true),
+            ColumnSchema::new(ENGINE, ConcreteDataType::string_datatype(), true),
         ]))
     }
 
@@ -85,14 +93,14 @@ impl InformationTable for InformationSchemaTables {
         self.schema.clone()
     }
 
-    fn to_stream(&self) -> Result<SendableRecordBatchStream> {
+    fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream> {
         let schema = self.schema.arrow_schema().clone();
         let mut builder = self.builder();
         let stream = Box::pin(DfRecordBatchStreamAdapter::new(
             schema,
             futures::stream::once(async move {
                 builder
-                    .make_tables()
+                    .make_tables(Some(request))
                     .await
                     .map(|x| x.into_df_record_batch())
                     .map_err(Into::into)
@@ -142,12 +150,13 @@ impl InformationSchemaTablesBuilder {
     }
 
     /// Construct the `information_schema.tables` virtual table
-    async fn make_tables(&mut self) -> Result<RecordBatch> {
+    async fn make_tables(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
         let catalog_name = self.catalog_name.clone();
         let catalog_manager = self
             .catalog_manager
             .upgrade()
             .context(UpgradeWeakCatalogManagerRefSnafu)?;
+        let predicates = Predicates::from_scan_request(&request);
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
@@ -167,6 +176,7 @@ impl InformationSchemaTablesBuilder {
                 {
                     let table_info = table.table_info();
                     self.add_table(
+                        &predicates,
                         &catalog_name,
                         &schema_name,
                         &table_name,
@@ -183,8 +193,10 @@ impl InformationSchemaTablesBuilder {
         self.finish()
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn add_table(
         &mut self,
+        predicates: &Predicates,
         catalog_name: &str,
         schema_name: &str,
         table_name: &str,
@@ -192,14 +204,27 @@ impl InformationSchemaTablesBuilder {
         table_id: Option<u32>,
         engine: Option<&str>,
     ) {
-        self.catalog_names.push(Some(catalog_name));
-        self.schema_names.push(Some(schema_name));
-        self.table_names.push(Some(table_name));
-        self.table_types.push(Some(match table_type {
+        let table_type = match table_type {
             TableType::Base => "BASE TABLE",
             TableType::View => "VIEW",
             TableType::Temporary => "LOCAL TEMPORARY",
-        }));
+        };
+
+        let row = [
+            (TABLE_CATALOG, &Value::from(catalog_name)),
+            (TABLE_SCHEMA, &Value::from(schema_name)),
+            (TABLE_NAME, &Value::from(table_name)),
+            (TABLE_TYPE, &Value::from(table_type)),
+        ];
+
+        if !predicates.eval(&row) {
+            return;
+        }
+
+        self.catalog_names.push(Some(catalog_name));
+        self.schema_names.push(Some(schema_name));
+        self.table_names.push(Some(table_name));
+        self.table_types.push(Some(table_type));
         self.table_ids.push(table_id);
         self.engines.push(engine);
     }
@@ -229,7 +254,7 @@ impl DfPartitionStream for InformationSchemaTables {
             schema,
             futures::stream::once(async move {
                 builder
-                    .make_tables()
+                    .make_tables(None)
                     .await
                     .map(|x| x.into_df_record_batch())
                     .map_err(Into::into)

src/catalog/src/metrics.rs

@@ -19,17 +19,17 @@ use prometheus::*;
 
 lazy_static! {
     pub static ref METRIC_CATALOG_MANAGER_CATALOG_COUNT: IntGauge =
-        register_int_gauge!("catalog_catalog_count", "catalog catalog count").unwrap();
+        register_int_gauge!("greptime_catalog_catalog_count", "catalog catalog count").unwrap();
     pub static ref METRIC_CATALOG_MANAGER_SCHEMA_COUNT: IntGauge =
-        register_int_gauge!("catalog_schema_count", "catalog schema count").unwrap();
+        register_int_gauge!("greptime_catalog_schema_count", "catalog schema count").unwrap();
     pub static ref METRIC_CATALOG_MANAGER_TABLE_COUNT: IntGaugeVec = register_int_gauge_vec!(
-        "catalog_table_count",
+        "greptime_catalog_table_count",
         "catalog table count",
         &[METRIC_DB_LABEL]
     )
     .unwrap();
     pub static ref METRIC_CATALOG_KV_REMOTE_GET: Histogram =
-        register_histogram!("catalog_kv_get_remote", "catalog kv get remote").unwrap();
+        register_histogram!("greptime_catalog_kv_get_remote", "catalog kv get remote").unwrap();
     pub static ref METRIC_CATALOG_KV_GET: Histogram =
-        register_histogram!("catalog_kv_get", "catalog kv get").unwrap();
+        register_histogram!("greptime_catalog_kv_get", "catalog kv get").unwrap();
 }

src/catalog/src/table_source.rs

@@ -15,7 +15,6 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use common_catalog::consts::INFORMATION_SCHEMA_NAME;
 use common_catalog::format_full_table_name;
 use datafusion::common::{ResolvedTableReference, TableReference};
 use datafusion::datasource::provider_as_source;
@@ -30,7 +29,7 @@ use crate::CatalogManagerRef;
 pub struct DfTableSourceProvider {
     catalog_manager: CatalogManagerRef,
     resolved_tables: HashMap<String, Arc<dyn TableSource>>,
-    disallow_cross_schema_query: bool,
+    disallow_cross_catalog_query: bool,
     default_catalog: String,
     default_schema: String,
 }
@@ -38,12 +37,12 @@ pub struct DfTableSourceProvider {
 impl DfTableSourceProvider {
     pub fn new(
         catalog_manager: CatalogManagerRef,
-        disallow_cross_schema_query: bool,
+        disallow_cross_catalog_query: bool,
         query_ctx: &QueryContext,
     ) -> Self {
         Self {
             catalog_manager,
-            disallow_cross_schema_query,
+            disallow_cross_catalog_query,
             resolved_tables: HashMap::new(),
             default_catalog: query_ctx.current_catalog().to_owned(),
             default_schema: query_ctx.current_schema().to_owned(),
@@ -54,29 +53,18 @@ impl DfTableSourceProvider {
         &'a self,
         table_ref: TableReference<'a>,
     ) -> Result<ResolvedTableReference<'a>> {
-        if self.disallow_cross_schema_query {
+        if self.disallow_cross_catalog_query {
             match &table_ref {
                 TableReference::Bare { .. } => (),
-                TableReference::Partial { schema, .. } => {
-                    ensure!(
-                        schema.as_ref() == self.default_schema
-                            || schema.as_ref() == INFORMATION_SCHEMA_NAME,
-                        QueryAccessDeniedSnafu {
-                            catalog: &self.default_catalog,
-                            schema: schema.as_ref(),
-                        }
-                    );
-                }
+                TableReference::Partial { .. } => {}
                 TableReference::Full {
                     catalog, schema, ..
                 } => {
                     ensure!(
-                        catalog.as_ref() == self.default_catalog
-                            && (schema.as_ref() == self.default_schema
-                                || schema.as_ref() == INFORMATION_SCHEMA_NAME),
+                        catalog.as_ref() == self.default_catalog,
                         QueryAccessDeniedSnafu {
                             catalog: catalog.as_ref(),
-                            schema: schema.as_ref()
+                            schema: schema.as_ref(),
                         }
                     );
                 }
@@ -136,21 +124,21 @@ mod tests {
             table: Cow::Borrowed("table_name"),
         };
         let result = table_provider.resolve_table_ref(table_ref);
-        let _ = result.unwrap();
+        assert!(result.is_ok());
 
         let table_ref = TableReference::Partial {
             schema: Cow::Borrowed("public"),
             table: Cow::Borrowed("table_name"),
         };
         let result = table_provider.resolve_table_ref(table_ref);
-        let _ = result.unwrap();
+        assert!(result.is_ok());
 
         let table_ref = TableReference::Partial {
             schema: Cow::Borrowed("wrong_schema"),
             table: Cow::Borrowed("table_name"),
         };
         let result = table_provider.resolve_table_ref(table_ref);
-        assert!(result.is_err());
+        assert!(result.is_ok());
 
         let table_ref = TableReference::Full {
             catalog: Cow::Borrowed("greptime"),
@@ -158,7 +146,7 @@ mod tests {
             table: Cow::Borrowed("table_name"),
         };
         let result = table_provider.resolve_table_ref(table_ref);
-        let _ = result.unwrap();
+        assert!(result.is_ok());
 
         let table_ref = TableReference::Full {
             catalog: Cow::Borrowed("wrong_catalog"),
@@ -172,14 +160,15 @@ mod tests {
             schema: Cow::Borrowed("information_schema"),
             table: Cow::Borrowed("columns"),
         };
-        let _ = table_provider.resolve_table_ref(table_ref).unwrap();
+        let result = table_provider.resolve_table_ref(table_ref);
+        assert!(result.is_ok());
 
         let table_ref = TableReference::Full {
             catalog: Cow::Borrowed("greptime"),
             schema: Cow::Borrowed("information_schema"),
             table: Cow::Borrowed("columns"),
         };
-        let _ = table_provider.resolve_table_ref(table_ref).unwrap();
+        assert!(table_provider.resolve_table_ref(table_ref).is_ok());
 
         let table_ref = TableReference::Full {
             catalog: Cow::Borrowed("dummy"),
@@ -187,5 +176,12 @@ mod tests {
             table: Cow::Borrowed("columns"),
         };
         assert!(table_provider.resolve_table_ref(table_ref).is_err());
+
+        let table_ref = TableReference::Full {
+            catalog: Cow::Borrowed("greptime"),
+            schema: Cow::Borrowed("greptime_private"),
+            table: Cow::Borrowed("columns"),
+        };
+        assert!(table_provider.resolve_table_ref(table_ref).is_ok());
     }
 }

src/client/src/error.rs

@@ -16,7 +16,7 @@ use std::any::Any;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
+use common_error::{GREPTIME_DB_HEADER_ERROR_CODE, GREPTIME_DB_HEADER_ERROR_MSG};
 use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tonic::{Code, Status};
@@ -115,7 +115,7 @@ impl From<Status> for Error {
                 .and_then(|v| String::from_utf8(v.as_bytes().to_vec()).ok())
         }
 
-        let code = get_metadata_value(&e, GREPTIME_ERROR_CODE)
+        let code = get_metadata_value(&e, GREPTIME_DB_HEADER_ERROR_CODE)
             .and_then(|s| {
                 if let Ok(code) = s.parse::<u32>() {
                     StatusCode::from_u32(code)
@@ -125,8 +125,8 @@ impl From<Status> for Error {
             })
             .unwrap_or(StatusCode::Unknown);
 
-        let msg =
-            get_metadata_value(&e, GREPTIME_ERROR_MSG).unwrap_or_else(|| e.message().to_string());
+        let msg = get_metadata_value(&e, GREPTIME_DB_HEADER_ERROR_MSG)
+            .unwrap_or_else(|| e.message().to_string());
 
         Self::Server { code, msg }
     }

src/client/src/metrics.rs

@@ -17,27 +17,30 @@ use prometheus::*;
 
 lazy_static! {
     pub static ref METRIC_GRPC_CREATE_TABLE: Histogram =
-        register_histogram!("grpc_create_table", "grpc create table").unwrap();
-    pub static ref METRIC_GRPC_PROMQL_RANGE_QUERY: Histogram =
-        register_histogram!("grpc_promql_range_query", "grpc promql range query").unwrap();
+        register_histogram!("greptime_grpc_create_table", "grpc create table").unwrap();
+    pub static ref METRIC_GRPC_PROMQL_RANGE_QUERY: Histogram = register_histogram!(
+        "greptime_grpc_promql_range_query",
+        "grpc promql range query"
+    )
+    .unwrap();
     pub static ref METRIC_GRPC_INSERT: Histogram =
-        register_histogram!("grpc_insert", "grpc insert").unwrap();
+        register_histogram!("greptime_grpc_insert", "grpc insert").unwrap();
     pub static ref METRIC_GRPC_DELETE: Histogram =
-        register_histogram!("grpc_delete", "grpc delete").unwrap();
+        register_histogram!("greptime_grpc_delete", "grpc delete").unwrap();
     pub static ref METRIC_GRPC_SQL: Histogram =
-        register_histogram!("grpc_sql", "grpc sql").unwrap();
+        register_histogram!("greptime_grpc_sql", "grpc sql").unwrap();
     pub static ref METRIC_GRPC_LOGICAL_PLAN: Histogram =
-        register_histogram!("grpc_logical_plan", "grpc logical plan").unwrap();
+        register_histogram!("greptime_grpc_logical_plan", "grpc logical plan").unwrap();
     pub static ref METRIC_GRPC_ALTER: Histogram =
-        register_histogram!("grpc_alter", "grpc alter").unwrap();
+        register_histogram!("greptime_grpc_alter", "grpc alter").unwrap();
     pub static ref METRIC_GRPC_DROP_TABLE: Histogram =
-        register_histogram!("grpc_drop_table", "grpc drop table").unwrap();
+        register_histogram!("greptime_grpc_drop_table", "grpc drop table").unwrap();
     pub static ref METRIC_GRPC_TRUNCATE_TABLE: Histogram =
-        register_histogram!("grpc_truncate_table", "grpc truncate table").unwrap();
+        register_histogram!("greptime_grpc_truncate_table", "grpc truncate table").unwrap();
     pub static ref METRIC_GRPC_DO_GET: Histogram =
-        register_histogram!("grpc_do_get", "grpc do get").unwrap();
+        register_histogram!("greptime_grpc_do_get", "grpc do get").unwrap();
     pub static ref METRIC_REGION_REQUEST_GRPC: HistogramVec = register_histogram_vec!(
-        "grpc_region_request",
+        "greptime_grpc_region_request",
         "grpc region request",
         &["request_type"]
     )

src/client/src/stream_insert.rs

@@ -39,7 +39,7 @@ use crate::from_grpc_response;
 /// ```
 ///
 /// If you want to see a concrete usage example, please see
-/// [stream_inserter.rs](https://github.com/GreptimeTeam/greptimedb/blob/develop/src/client/examples/stream_ingest.rs).
+/// [stream_inserter.rs](https://github.com/GreptimeTeam/greptimedb/blob/main/src/client/examples/stream_ingest.rs).
 pub struct StreamInserter {
     sender: mpsc::Sender<GreptimeRequest>,
 

src/cmd/Cargo.toml

@@ -32,6 +32,7 @@ common-recordbatch.workspace = true
 common-telemetry = { workspace = true, features = [
     "deadlock_detection",
 ] }
+common-time.workspace = true
 config = "0.13"
 datanode.workspace = true
 datatypes.workspace = true

src/cmd/src/cli/bench/metadata.rs

@@ -14,6 +14,7 @@
 
 use std::time::Instant;
 
+use common_meta::key::table_route::TableRouteValue;
 use common_meta::key::TableMetadataManagerRef;
 use common_meta::table_name::TableName;
 
@@ -53,7 +54,11 @@ impl TableMetadataBencher {
                 let start = Instant::now();
 
                 self.table_metadata_manager
-                    .create_table_metadata(table_info, region_routes, region_wal_options)
+                    .create_table_metadata(
+                        table_info,
+                        TableRouteValue::physical(region_routes),
+                        region_wal_options,
+                    )
                     .await
                     .unwrap();
 

src/cmd/src/cli/upgrade.rs

@@ -27,7 +27,7 @@ use common_meta::key::table_info::{TableInfoKey, TableInfoValue};
 use common_meta::key::table_name::{TableNameKey, TableNameValue};
 use common_meta::key::table_region::{TableRegionKey, TableRegionValue};
 use common_meta::key::table_route::{TableRouteKey, TableRouteValue as NextTableRouteValue};
-use common_meta::key::{RegionDistribution, TableMetaKey};
+use common_meta::key::{RegionDistribution, TableMetaKey, TableMetaValue};
 use common_meta::kv_backend::etcd::EtcdStore;
 use common_meta::kv_backend::KvBackendRef;
 use common_meta::range_stream::PaginationStream;
@@ -153,7 +153,7 @@ impl MigrateTableMetadata {
         )
         .unwrap();
 
-        let new_table_value = NextTableRouteValue::new(table_route.region_routes);
+        let new_table_value = NextTableRouteValue::physical(table_route.region_routes);
 
         let table_id = table_route.table.id as u32;
         let new_key = TableRouteKey::new(table_id);

src/cmd/src/error.rs

@@ -43,6 +43,12 @@ pub enum Error {
         source: common_meta::error::Error,
     },
 
+    #[snafu(display("Failed to init default timezone"))]
+    InitTimezone {
+        location: Location,
+        source: common_time::error::Error,
+    },
+
     #[snafu(display("Failed to start procedure manager"))]
     StartProcedureManager {
         location: Location,
@@ -268,6 +274,7 @@ impl ErrorExt for Error {
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
             | Error::InvalidReplCommand { .. }
+            | Error::InitTimezone { .. }
             | Error::ConnectEtcd { .. }
             | Error::NotDataFromOutput { .. }
             | Error::CreateDir { .. }

src/cmd/src/frontend.rs

@@ -22,17 +22,19 @@ use client::client_manager::DatanodeClients;
 use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
 use common_meta::heartbeat::handler::HandlerGroupExecutor;
 use common_telemetry::logging;
+use common_time::timezone::set_default_timezone;
 use frontend::frontend::FrontendOptions;
 use frontend::heartbeat::handler::invalidate_table_cache::InvalidateTableCacheHandler;
 use frontend::heartbeat::HeartbeatTask;
 use frontend::instance::builder::FrontendBuilder;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
+use frontend::server::Services;
 use meta_client::MetaClientOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::{OptionExt, ResultExt};
 
-use crate::error::{self, MissingConfigSnafu, Result, StartFrontendSnafu};
+use crate::error::{self, InitTimezoneSnafu, MissingConfigSnafu, Result, StartFrontendSnafu};
 use crate::options::{CliOptions, Options};
 use crate::App;
 
@@ -217,6 +219,8 @@ impl StartCommand {
         logging::info!("Frontend start command: {:#?}", self);
         logging::info!("Frontend options: {:#?}", opts);
 
+        set_default_timezone(opts.default_timezone.as_deref()).context(InitTimezoneSnafu)?;
+
         let meta_client_options = opts.meta_client.as_ref().context(MissingConfigSnafu {
             msg: "'meta_client'",
         })?;
@@ -243,18 +247,18 @@ impl StartCommand {
             meta_client,
         )
         .with_cache_invalidator(meta_backend)
-        .with_plugin(plugins)
+        .with_plugin(plugins.clone())
         .with_heartbeat_task(heartbeat_task)
         .try_build()
         .await
         .context(StartFrontendSnafu)?;
 
-        instance
-            .build_export_metrics_task(&opts.export_metrics)
+        let servers = Services::new(plugins)
+            .build(opts.clone(), Arc::new(instance.clone()))
+            .await
             .context(StartFrontendSnafu)?;
-
         instance
-            .build_servers(opts)
+            .build_servers(opts, servers)
             .await
             .context(StartFrontendSnafu)?;
 

src/cmd/src/lib.rs

@@ -28,7 +28,7 @@ pub mod standalone;
 
 lazy_static::lazy_static! {
     static ref APP_VERSION: prometheus::IntGaugeVec =
-        prometheus::register_int_gauge_vec!("app_version", "app version", &["short_version", "version"]).unwrap();
+        prometheus::register_int_gauge_vec!("greptime_app_version", "app version", &["short_version", "version"]).unwrap();
 }
 
 #[async_trait]

src/cmd/src/metasrv.rs

@@ -119,8 +119,8 @@ struct StartCommand {
     data_home: Option<String>,
 
     /// If it's not empty, the metasrv will store all data with this key prefix.
-    #[clap(long)]
-    store_key_prefix: Option<String>,
+    #[clap(long, default_value = "")]
+    store_key_prefix: String,
 }
 
 impl StartCommand {
@@ -128,7 +128,7 @@ impl StartCommand {
         let mut opts: MetaSrvOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
-            None,
+            MetaSrvOptions::env_list_keys(),
         )?;
 
         if let Some(dir) = &cli_options.log_dir {
@@ -177,7 +177,9 @@ impl StartCommand {
             opts.data_home = data_home.clone();
         }
 
-        opts.store_key_prefix = self.store_key_prefix.clone();
+        if !self.store_key_prefix.is_empty() {
+            opts.store_key_prefix = self.store_key_prefix.clone()
+        }
 
         // Disable dashboard in metasrv.
         opts.http.disable_dashboard = true;

src/cmd/src/standalone.rs

@@ -22,7 +22,8 @@ use common_config::wal::StandaloneWalConfig;
 use common_config::{metadata_store_dir, KvBackendConfig};
 use common_meta::cache_invalidator::DummyCacheInvalidator;
 use common_meta::datanode_manager::DatanodeManagerRef;
-use common_meta::ddl::{DdlTaskExecutorRef, TableMetadataAllocatorRef};
+use common_meta::ddl::table_meta::TableMetadataAllocator;
+use common_meta::ddl::DdlTaskExecutorRef;
 use common_meta::ddl_manager::DdlManager;
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
@@ -32,13 +33,14 @@ use common_meta::wal::{WalOptionsAllocator, WalOptionsAllocatorRef};
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
+use common_time::timezone::set_default_timezone;
 use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::builder::FrontendBuilder;
-use frontend::instance::standalone::StandaloneTableMetadataAllocator;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
+use frontend::server::Services;
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
@@ -51,8 +53,8 @@ use servers::Mode;
 use snafu::ResultExt;
 
 use crate::error::{
-    CreateDirSnafu, IllegalConfigSnafu, InitDdlManagerSnafu, InitMetadataSnafu, Result,
-    ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu,
+    CreateDirSnafu, IllegalConfigSnafu, InitDdlManagerSnafu, InitMetadataSnafu, InitTimezoneSnafu,
+    Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu,
     StartProcedureManagerSnafu, StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
 };
 use crate::options::{CliOptions, MixOptions, Options};
@@ -98,6 +100,7 @@ impl SubCommand {
 pub struct StandaloneOptions {
     pub mode: Mode,
     pub enable_telemetry: bool,
+    pub default_timezone: Option<String>,
     pub http: HttpOptions,
     pub grpc: GrpcOptions,
     pub mysql: MysqlOptions,
@@ -116,11 +119,18 @@ pub struct StandaloneOptions {
     pub export_metrics: ExportMetricsOption,
 }
 
+impl StandaloneOptions {
+    pub fn env_list_keys() -> Option<&'static [&'static str]> {
+        Some(&["wal.broker_endpoints"])
+    }
+}
+
 impl Default for StandaloneOptions {
     fn default() -> Self {
         Self {
             mode: Mode::Standalone,
             enable_telemetry: true,
+            default_timezone: None,
             http: HttpOptions::default(),
             grpc: GrpcOptions::default(),
             mysql: MysqlOptions::default(),
@@ -147,6 +157,7 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
+            default_timezone: self.default_timezone,
             http: self.http,
             grpc: self.grpc,
             mysql: self.mysql,
@@ -263,7 +274,7 @@ impl StartCommand {
         let opts: StandaloneOptions = Options::load_layered_options(
             self.config_file.as_deref(),
             self.env_prefix.as_ref(),
-            None,
+            StandaloneOptions::env_list_keys(),
         )?;
 
         self.convert_options(cli_options, opts)
@@ -369,6 +380,9 @@ impl StartCommand {
 
         info!("Building standalone instance with {opts:#?}");
 
+        set_default_timezone(opts.frontend.default_timezone.as_deref())
+            .context(InitTimezoneSnafu)?;
+
         // Ensure the data_home directory exists.
         fs::create_dir_all(path::Path::new(&opts.data_home)).context(CreateDirSnafu {
             dir: &opts.data_home,
@@ -399,13 +413,18 @@ impl StartCommand {
             opts.wal_meta.clone(),
             kv_backend.clone(),
         ));
-        let table_meta_allocator = Arc::new(StandaloneTableMetadataAllocator::new(
+
+        let table_metadata_manager =
+            Self::create_table_metadata_manager(kv_backend.clone()).await?;
+
+        let table_meta_allocator = TableMetadataAllocator::new(
             table_id_sequence,
             wal_options_allocator.clone(),
-        ));
+            table_metadata_manager.clone(),
+        );
 
         let ddl_task_executor = Self::create_ddl_task_executor(
-            kv_backend.clone(),
+            table_metadata_manager,
             procedure_manager.clone(),
             datanode_manager.clone(),
             table_meta_allocator,
@@ -413,17 +432,17 @@ impl StartCommand {
         .await?;
 
         let mut frontend = FrontendBuilder::new(kv_backend, datanode_manager, ddl_task_executor)
-            .with_plugin(fe_plugins)
+            .with_plugin(fe_plugins.clone())
             .try_build()
             .await
             .context(StartFrontendSnafu)?;
 
-        frontend
-            .build_export_metrics_task(&opts.frontend.export_metrics)
+        let servers = Services::new(fe_plugins)
+            .build(opts.clone(), Arc::new(frontend.clone()))
+            .await
             .context(StartFrontendSnafu)?;
-
         frontend
-            .build_servers(opts)
+            .build_servers(opts, servers)
             .await
             .context(StartFrontendSnafu)?;
 
@@ -436,14 +455,11 @@ impl StartCommand {
     }
 
     pub async fn create_ddl_task_executor(
-        kv_backend: KvBackendRef,
+        table_metadata_manager: TableMetadataManagerRef,
         procedure_manager: ProcedureManagerRef,
         datanode_manager: DatanodeManagerRef,
-        table_meta_allocator: TableMetadataAllocatorRef,
+        table_meta_allocator: TableMetadataAllocator,
     ) -> Result<DdlTaskExecutorRef> {
-        let table_metadata_manager =
-            Self::create_table_metadata_manager(kv_backend.clone()).await?;
-
         let ddl_task_executor: DdlTaskExecutorRef = Arc::new(
             DdlManager::try_new(
                 procedure_manager,
@@ -459,7 +475,7 @@ impl StartCommand {
         Ok(ddl_task_executor)
     }
 
-    async fn create_table_metadata_manager(
+    pub async fn create_table_metadata_manager(
         kv_backend: KvBackendRef,
     ) -> Result<TableMetadataManagerRef> {
         let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend));

src/common/catalog/src/consts.rs

@@ -44,6 +44,44 @@ pub const INFORMATION_SCHEMA_COLUMN_PRIVILEGES_TABLE_ID: u32 = 6;
 pub const INFORMATION_SCHEMA_COLUMN_STATISTICS_TABLE_ID: u32 = 7;
 /// id for information_schema.build_info
 pub const INFORMATION_SCHEMA_BUILD_INFO_TABLE_ID: u32 = 8;
+/// id for information_schema.CHARACTER_SETS
+pub const INFORMATION_SCHEMA_CHARACTER_SETS_TABLE_ID: u32 = 9;
+/// id for information_schema.COLLATIONS
+pub const INFORMATION_SCHEMA_COLLATIONS_TABLE_ID: u32 = 10;
+/// id for information_schema.COLLATIONS
+pub const INFORMATION_SCHEMA_COLLATION_CHARACTER_SET_APPLICABILITY_TABLE_ID: u32 = 11;
+/// id for information_schema.CHECK_CONSTRAINTS
+pub const INFORMATION_SCHEMA_CHECK_CONSTRAINTS_TABLE_ID: u32 = 12;
+/// id for information_schema.EVENTS
+pub const INFORMATION_SCHEMA_EVENTS_TABLE_ID: u32 = 13;
+/// id for information_schema.FILES
+pub const INFORMATION_SCHEMA_FILES_TABLE_ID: u32 = 14;
+/// id for information_schema.SCHEMATA
+pub const INFORMATION_SCHEMA_SCHEMATA_TABLE_ID: u32 = 15;
+/// id for information_schema.KEY_COLUMN_USAGE
+pub const INFORMATION_SCHEMA_KEY_COLUMN_USAGE_TABLE_ID: u32 = 16;
+/// id for information_schema.OPTIMIZER_TRACE
+pub const INFORMATION_SCHEMA_OPTIMIZER_TRACE_TABLE_ID: u32 = 17;
+/// id for information_schema.PARAMETERS
+pub const INFORMATION_SCHEMA_PARAMETERS_TABLE_ID: u32 = 18;
+/// id for information_schema.PROFILING
+pub const INFORMATION_SCHEMA_PROFILING_TABLE_ID: u32 = 19;
+/// id for information_schema.REFERENTIAL_CONSTRAINTS
+pub const INFORMATION_SCHEMA_REFERENTIAL_CONSTRAINTS_TABLE_ID: u32 = 20;
+/// id for information_schema.ROUTINES
+pub const INFORMATION_SCHEMA_ROUTINES_TABLE_ID: u32 = 21;
+/// id for information_schema.SCHEMA_PRIVILEGES
+pub const INFORMATION_SCHEMA_SCHEMA_PRIVILEGES_TABLE_ID: u32 = 22;
+/// id for information_schema.TABLE_PRIVILEGES
+pub const INFORMATION_SCHEMA_TABLE_PRIVILEGES_TABLE_ID: u32 = 23;
+/// id for information_schema.TRIGGERS
+pub const INFORMATION_SCHEMA_TRIGGERS_TABLE_ID: u32 = 24;
+/// id for information_schema.GLOBAL_STATUS
+pub const INFORMATION_SCHEMA_GLOBAL_STATUS_TABLE_ID: u32 = 25;
+/// id for information_schema.SESSION_STATUS
+pub const INFORMATION_SCHEMA_SESSION_STATUS_TABLE_ID: u32 = 26;
+/// id for information_schema.RUNTIME_METRICS
+pub const INFORMATION_SCHEMA_RUNTIME_METRICS_TABLE_ID: u32 = 27;
 /// ----- End of information_schema tables -----
 
 pub const MITO_ENGINE: &str = "mito";

src/common/catalog/src/lib.rs

@@ -17,6 +17,11 @@ use consts::DEFAULT_CATALOG_NAME;
 pub mod consts;
 pub mod error;
 
+#[inline]
+pub fn format_schema_name(catalog: &str, schema: &str) -> String {
+    format!("{catalog}.{schema}")
+}
+
 /// Formats table fully-qualified name
 #[inline]
 pub fn format_full_table_name(catalog: &str, schema: &str, table: &str) -> String {

src/common/config/src/wal.rs

@@ -18,9 +18,7 @@ pub mod raft_engine;
 use serde::{Deserialize, Serialize};
 use serde_with::with_prefix;
 
-pub use crate::wal::kafka::{
-    KafkaConfig, KafkaOptions as KafkaWalOptions, StandaloneKafkaConfig, Topic as KafkaWalTopic,
-};
+pub use crate::wal::kafka::{KafkaConfig, KafkaOptions as KafkaWalOptions, StandaloneKafkaConfig};
 pub use crate::wal::raft_engine::RaftEngineConfig;
 
 /// An encoded wal options will be wrapped into a (WAL_OPTIONS_KEY, encoded wal options) key-value pair
@@ -90,11 +88,12 @@ mod tests {
 
     #[test]
     fn test_serde_kafka_config() {
+        // With all fields.
         let toml_str = r#"
             broker_endpoints = ["127.0.0.1:9092"]
-            max_batch_size = "4MB"
+            max_batch_size = "1MB"
             linger = "200ms"
-            produce_record_timeout = "100ms"
+            consumer_wait_timeout = "100ms"
             backoff_init = "500ms"
             backoff_max = "10s"
             backoff_base = 2
@@ -104,9 +103,9 @@ mod tests {
         let expected = KafkaConfig {
             broker_endpoints: vec!["127.0.0.1:9092".to_string()],
             compression: RsKafkaCompression::default(),
-            max_batch_size: ReadableSize::mb(4),
+            max_batch_size: ReadableSize::mb(1),
             linger: Duration::from_millis(200),
-            produce_record_timeout: Duration::from_millis(100),
+            consumer_wait_timeout: Duration::from_millis(100),
             backoff: KafkaBackoffConfig {
                 init: Duration::from_millis(500),
                 max: Duration::from_secs(10),
@@ -115,6 +114,19 @@ mod tests {
             },
         };
         assert_eq!(decoded, expected);
+
+        // With some fields missing.
+        let toml_str = r#"
+            broker_endpoints = ["127.0.0.1:9092"]
+            linger = "200ms"
+        "#;
+        let decoded: KafkaConfig = toml::from_str(toml_str).unwrap();
+        let expected = KafkaConfig {
+            broker_endpoints: vec!["127.0.0.1:9092".to_string()],
+            linger: Duration::from_millis(200),
+            ..Default::default()
+        };
+        assert_eq!(decoded, expected);
     }
 
     #[test]

src/common/config/src/wal/kafka.rs

@@ -19,11 +19,6 @@ use rskafka::client::partition::Compression as RsKafkaCompression;
 use serde::{Deserialize, Serialize};
 use serde_with::with_prefix;
 
-/// Topic name prefix.
-pub const TOPIC_NAME_PREFIX: &str = "greptimedb_wal_topic";
-/// Kafka wal topic.
-pub type Topic = String;
-
 /// The type of the topic selector, i.e. with which strategy to select a topic.
 #[derive(Default, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
@@ -40,16 +35,15 @@ pub struct KafkaConfig {
     pub broker_endpoints: Vec<String>,
     /// The compression algorithm used to compress log entries.
     #[serde(skip)]
-    #[serde(default)]
     pub compression: RsKafkaCompression,
-    /// The maximum log size a kakfa batch producer could buffer.
+    /// The max size of a single producer batch.
     pub max_batch_size: ReadableSize,
     /// The linger duration of a kafka batch producer.
     #[serde(with = "humantime_serde")]
     pub linger: Duration,
-    /// The maximum amount of time (in milliseconds) to wait for Kafka records to be returned.
+    /// The consumer wait timeout.
     #[serde(with = "humantime_serde")]
-    pub produce_record_timeout: Duration,
+    pub consumer_wait_timeout: Duration,
     /// The backoff config.
     #[serde(flatten, with = "kafka_backoff")]
     pub backoff: KafkaBackoffConfig,
@@ -60,9 +54,10 @@ impl Default for KafkaConfig {
         Self {
             broker_endpoints: vec!["127.0.0.1:9092".to_string()],
             compression: RsKafkaCompression::NoCompression,
-            max_batch_size: ReadableSize::mb(4),
+            // Warning: Kafka has a default limit of 1MB per message in a topic.
+            max_batch_size: ReadableSize::mb(1),
             linger: Duration::from_millis(200),
-            produce_record_timeout: Duration::from_millis(100),
+            consumer_wait_timeout: Duration::from_millis(100),
             backoff: KafkaBackoffConfig::default(),
         }
     }
@@ -73,17 +68,15 @@ with_prefix!(pub kafka_backoff "backoff_");
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct KafkaBackoffConfig {
-    /// The initial backoff for kafka clients.
+    /// The initial backoff delay.
     #[serde(with = "humantime_serde")]
     pub init: Duration,
-    /// The maximum backoff for kafka clients.
+    /// The maximum backoff delay.
     #[serde(with = "humantime_serde")]
     pub max: Duration,
     /// Exponential backoff rate, i.e. next backoff = base * current backoff.
-    // Sets to u32 type since some structs containing the KafkaConfig need to derive the Eq trait.
     pub base: u32,
-    /// Stop reconnecting if the total wait time reaches the deadline.
-    /// If it's None, the reconnecting won't terminate.
+    /// The deadline of retries. `None` stands for no deadline.
     #[serde(with = "humantime_serde")]
     pub deadline: Option<Duration>,
 }
@@ -114,7 +107,7 @@ pub struct StandaloneKafkaConfig {
     pub num_partitions: i32,
     /// The replication factor of each topic.
     pub replication_factor: i16,
-    /// Above which a topic creation operation will be cancelled.
+    /// The timeout of topic creation.
     #[serde(with = "humantime_serde")]
     pub create_topic_timeout: Duration,
 }
@@ -140,5 +133,5 @@ impl Default for StandaloneKafkaConfig {
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct KafkaOptions {
     /// Kafka wal topic.
-    pub topic: Topic,
+    pub topic: String,
 }

src/common/config/src/wal/raft_engine.rs

@@ -34,6 +34,13 @@ pub struct RaftEngineConfig {
     pub read_batch_size: usize,
     // whether to sync log file after every write
     pub sync_write: bool,
+    // whether to reuse logically truncated log files.
+    pub enable_log_recycle: bool,
+    // whether to pre-create log files on start up
+    pub prefill_log_files: bool,
+    // duration for fsyncing log files.
+    #[serde(with = "humantime_serde")]
+    pub sync_period: Option<Duration>,
 }
 
 impl Default for RaftEngineConfig {
@@ -45,6 +52,9 @@ impl Default for RaftEngineConfig {
             purge_interval: Duration::from_secs(600),
             read_batch_size: 128,
             sync_write: false,
+            enable_log_recycle: true,
+            prefill_log_files: false,
+            sync_period: None,
         }
     }
 }

src/common/error/src/lib.rs

@@ -19,7 +19,7 @@ pub mod format;
 pub mod mock;
 pub mod status_code;
 
-pub const GREPTIME_ERROR_CODE: &str = "x-greptime-err-code";
-pub const GREPTIME_ERROR_MSG: &str = "x-greptime-err-msg";
+pub const GREPTIME_DB_HEADER_ERROR_CODE: &str = "x-greptime-err-code";
+pub const GREPTIME_DB_HEADER_ERROR_MSG: &str = "x-greptime-err-msg";
 
 pub use snafu;

src/common/function/src/system.rs

@@ -13,10 +13,12 @@
 // limitations under the License.
 
 pub mod build;
+pub mod version;
 
 use std::sync::Arc;
 
 use build::BuildFunction;
+use version::VersionFunction;
 
 use crate::function_registry::FunctionRegistry;
 
@@ -25,5 +27,6 @@ pub(crate) struct SystemFunction;
 impl SystemFunction {
     pub fn register(registry: &FunctionRegistry) {
         registry.register(Arc::new(BuildFunction));
+        registry.register(Arc::new(VersionFunction));
     }
 }

src/common/function/src/system/version.rs

@@ -0,0 +1,54 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+use std::{env, fmt};
+
+use common_query::error::Result;
+use common_query::prelude::{Signature, Volatility};
+use datatypes::data_type::ConcreteDataType;
+use datatypes::vectors::{StringVector, VectorRef};
+
+use crate::function::{Function, FunctionContext};
+
+#[derive(Clone, Debug, Default)]
+pub(crate) struct VersionFunction;
+
+impl fmt::Display for VersionFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "VERSION")
+    }
+}
+
+impl Function for VersionFunction {
+    fn name(&self) -> &str {
+        "version"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::exact(vec![], Volatility::Immutable)
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
+        let result = StringVector::from(vec![format!(
+            "5.7.20-greptimedb-{}",
+            env!("CARGO_PKG_VERSION")
+        )]);
+        Ok(Arc::new(result))
+    }
+}

src/common/meta/Cargo.toml

@@ -14,6 +14,7 @@ async-stream.workspace = true
 async-trait.workspace = true
 base64.workspace = true
 bytes.workspace = true
+chrono.workspace = true
 common-catalog.workspace = true
 common-config.workspace = true
 common-error.workspace = true
@@ -27,6 +28,7 @@ common-time.workspace = true
 datatypes.workspace = true
 derive_builder.workspace = true
 etcd-client.workspace = true
+futures-util.workspace = true
 futures.workspace = true
 humantime-serde.workspace = true
 lazy_static.workspace = true
@@ -51,3 +53,4 @@ chrono.workspace = true
 common-procedure = { workspace = true, features = ["testing"] }
 datatypes.workspace = true
 hyper = { version = "0.14", features = ["full"] }
+uuid.workspace = true

src/common/meta/src/ddl.rs

@@ -21,14 +21,15 @@ use store_api::storage::{RegionNumber, TableId};
 use crate::cache_invalidator::CacheInvalidatorRef;
 use crate::datanode_manager::DatanodeManagerRef;
 use crate::error::Result;
+use crate::key::table_route::TableRouteValue;
 use crate::key::TableMetadataManagerRef;
 use crate::region_keeper::MemoryRegionKeeperRef;
-use crate::rpc::ddl::{CreateTableTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse};
-use crate::rpc::router::RegionRoute;
+use crate::rpc::ddl::{SubmitDdlTaskRequest, SubmitDdlTaskResponse};
 
 pub mod alter_table;
 pub mod create_table;
 pub mod drop_table;
+pub mod table_meta;
 pub mod truncate_table;
 pub mod utils;
 
@@ -58,23 +59,12 @@ pub struct TableMetadata {
     /// Table id.
     pub table_id: TableId,
     /// Route information for each region of the table.
-    pub region_routes: Vec<RegionRoute>,
+    pub table_route: TableRouteValue,
     /// The encoded wal options for regions of the table.
     // If a region does not have an associated wal options, no key for the region would be found in the map.
     pub region_wal_options: HashMap<RegionNumber, String>,
 }
 
-#[async_trait::async_trait]
-pub trait TableMetadataAllocator: Send + Sync {
-    async fn create(
-        &self,
-        ctx: &TableMetadataAllocatorContext,
-        task: &CreateTableTask,
-    ) -> Result<TableMetadata>;
-}
-
-pub type TableMetadataAllocatorRef = Arc<dyn TableMetadataAllocator>;
-
 #[derive(Clone)]
 pub struct DdlContext {
     pub datanode_manager: DatanodeManagerRef,

src/common/meta/src/ddl/alter_table.rs

@@ -24,7 +24,7 @@ use async_trait::async_trait;
 use common_grpc_expr::alter_expr_to_request;
 use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
 use common_procedure::{
-    Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure, Status,
+    Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure, Status, StringKey,
 };
 use common_telemetry::tracing_context::TracingContext;
 use common_telemetry::{debug, info};
@@ -40,13 +40,11 @@ use table::requests::AlterKind;
 use crate::cache_invalidator::Context;
 use crate::ddl::utils::handle_operate_region_error;
 use crate::ddl::DdlContext;
-use crate::error::{
-    self, ConvertAlterTableRequestSnafu, InvalidProtoMsgSnafu, Result, TableRouteNotFoundSnafu,
-};
+use crate::error::{self, ConvertAlterTableRequestSnafu, Error, InvalidProtoMsgSnafu, Result};
 use crate::key::table_info::TableInfoValue;
 use crate::key::table_name::TableNameKey;
-use crate::key::table_route::TableRouteValue;
 use crate::key::DeserializedValueWithBytes;
+use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
 use crate::metrics;
 use crate::rpc::ddl::AlterTableTask;
 use crate::rpc::router::{find_leader_regions, find_leaders};
@@ -66,6 +64,7 @@ impl AlterTableProcedure {
         cluster_id: u64,
         task: AlterTableTask,
         table_info_value: DeserializedValueWithBytes<TableInfoValue>,
+        physical_table_info: Option<(TableId, TableName)>,
         context: DdlContext,
     ) -> Result<Self> {
         let alter_kind = task
@@ -85,7 +84,13 @@ impl AlterTableProcedure {
 
         Ok(Self {
             context,
-            data: AlterTableData::new(task, table_info_value, cluster_id, next_column_id),
+            data: AlterTableData::new(
+                task,
+                table_info_value,
+                physical_table_info,
+                cluster_id,
+                next_column_id,
+            ),
             kind,
         })
     }
@@ -183,25 +188,19 @@ impl AlterTableProcedure {
 
     pub async fn submit_alter_region_requests(&mut self) -> Result<Status> {
         let table_id = self.data.table_id();
-        let table_ref = self.data.table_ref();
-
-        let TableRouteValue { region_routes, .. } = self
+        let (_, physical_table_route) = self
             .context
             .table_metadata_manager
             .table_route_manager()
-            .get(table_id)
-            .await?
-            .with_context(|| TableRouteNotFoundSnafu {
-                table_name: table_ref.to_string(),
-            })?
-            .into_inner();
+            .get_physical_table_route(table_id)
+            .await?;
 
-        let leaders = find_leaders(&region_routes);
+        let leaders = find_leaders(&physical_table_route.region_routes);
         let mut alter_region_tasks = Vec::with_capacity(leaders.len());
 
         for datanode in leaders {
             let requester = self.context.datanode_manager.datanode(&datanode).await;
-            let regions = find_leader_regions(&region_routes, &datanode);
+            let regions = find_leader_regions(&physical_table_route.region_routes, &datanode);
 
             for region in regions {
                 let region_id = RegionId::new(table_id, region);
@@ -337,21 +336,31 @@ impl AlterTableProcedure {
         Ok(Status::Done)
     }
 
-    fn lock_key_inner(&self) -> Vec<String> {
+    fn lock_key_inner(&self) -> Vec<StringKey> {
+        let mut lock_key = vec![];
+
+        if let Some((physical_table_id, physical_table_name)) = self.data.physical_table_info() {
+            lock_key.push(CatalogLock::Read(&physical_table_name.catalog_name).into());
+            lock_key.push(
+                SchemaLock::read(
+                    &physical_table_name.catalog_name,
+                    &physical_table_name.schema_name,
+                )
+                .into(),
+            );
+            lock_key.push(TableLock::Read(*physical_table_id).into())
+        }
+
         let table_ref = self.data.table_ref();
-        let table_key = common_catalog::format_full_table_name(
-            table_ref.catalog,
-            table_ref.schema,
-            table_ref.table,
-        );
-        let mut lock_key = vec![table_key];
+        let table_id = self.data.table_id();
+        lock_key.push(CatalogLock::Read(table_ref.catalog).into());
+        lock_key.push(SchemaLock::read(table_ref.catalog, table_ref.schema).into());
+        lock_key.push(TableLock::Write(table_id).into());
 
         if let Ok(Kind::RenameTable(RenameTable { new_table_name })) = self.alter_kind() {
-            lock_key.push(common_catalog::format_full_table_name(
-                table_ref.catalog,
-                table_ref.schema,
-                new_table_name,
-            ))
+            lock_key.push(
+                TableNameLock::new(table_ref.catalog, table_ref.schema, new_table_name).into(),
+            )
         }
 
         lock_key
@@ -365,8 +374,8 @@ impl Procedure for AlterTableProcedure {
     }
 
     async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
-        let error_handler = |e| {
-            if matches!(e, error::Error::RetryLater { .. }) {
+        let error_handler = |e: Error| {
+            if e.is_retry_later() {
                 ProcedureError::retry_later(e)
             } else {
                 ProcedureError::external(e)
@@ -414,11 +423,13 @@ enum AlterTableState {
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct AlterTableData {
+    cluster_id: u64,
     state: AlterTableState,
     task: AlterTableTask,
     /// Table info value before alteration.
     table_info_value: DeserializedValueWithBytes<TableInfoValue>,
-    cluster_id: u64,
+    /// Physical table name, if the table to alter is a logical table.
+    physical_table_info: Option<(TableId, TableName)>,
     /// Next column id of the table if the task adds columns to the table.
     next_column_id: Option<ColumnId>,
 }
@@ -427,6 +438,7 @@ impl AlterTableData {
     pub fn new(
         task: AlterTableTask,
         table_info_value: DeserializedValueWithBytes<TableInfoValue>,
+        physical_table_info: Option<(TableId, TableName)>,
         cluster_id: u64,
         next_column_id: Option<ColumnId>,
     ) -> Self {
@@ -434,6 +446,7 @@ impl AlterTableData {
             state: AlterTableState::Prepare,
             task,
             table_info_value,
+            physical_table_info,
             cluster_id,
             next_column_id,
         }
@@ -450,6 +463,10 @@ impl AlterTableData {
     fn table_info(&self) -> &RawTableInfo {
         &self.table_info_value.table_info
     }
+
+    fn physical_table_info(&self) -> Option<&(TableId, TableName)> {
+        self.physical_table_info.as_ref()
+    }
 }
 
 /// Creates region proto alter kind from `table_info` and `alter_kind`.

src/common/meta/src/ddl/create_table.rs

@@ -18,10 +18,8 @@ use api::v1::region::region_request::Body as PbRegionRequest;
 use api::v1::region::{
     CreateRequest as PbCreateRegionRequest, RegionColumnDef, RegionRequest, RegionRequestHeader,
 };
-use api::v1::{ColumnDef, CreateTableExpr, SemanticType};
+use api::v1::{ColumnDef, SemanticType};
 use async_trait::async_trait;
-use common_catalog::consts::METRIC_ENGINE;
-use common_config::WAL_OPTIONS_KEY;
 use common_error::ext::BoxedError;
 use common_procedure::error::{
     ExternalSnafu, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu,
@@ -40,14 +38,17 @@ use table::metadata::{RawTableInfo, TableId};
 
 use crate::ddl::utils::{handle_operate_region_error, handle_retry_error, region_storage_path};
 use crate::ddl::DdlContext;
-use crate::error::{self, Result, TableInfoNotFoundSnafu};
+use crate::error::{self, Result, TableRouteNotFoundSnafu};
 use crate::key::table_name::TableNameKey;
+use crate::key::table_route::TableRouteValue;
+use crate::lock_key::TableNameLock;
 use crate::metrics;
 use crate::region_keeper::OperatingRegionGuard;
 use crate::rpc::ddl::CreateTableTask;
 use crate::rpc::router::{
     find_leader_regions, find_leaders, operating_leader_regions, RegionRoute,
 };
+use crate::wal::prepare_wal_option;
 
 pub struct CreateTableProcedure {
     pub context: DdlContext,
@@ -60,13 +61,13 @@ impl CreateTableProcedure {
     pub fn new(
         cluster_id: u64,
         task: CreateTableTask,
-        region_routes: Vec<RegionRoute>,
+        table_route: TableRouteValue,
         region_wal_options: HashMap<RegionNumber, String>,
         context: DdlContext,
     ) -> Self {
         Self {
             context,
-            creator: TableCreator::new(cluster_id, task, region_routes, region_wal_options),
+            creator: TableCreator::new(cluster_id, task, table_route, region_wal_options),
         }
     }
 
@@ -78,10 +79,12 @@ impl CreateTableProcedure {
             opening_regions: vec![],
         };
 
-        creator
-            .register_opening_regions(&context)
-            .map_err(BoxedError::new)
-            .context(ExternalSnafu)?;
+        if let TableRouteValue::Physical(x) = &creator.data.table_route {
+            creator.opening_regions = creator
+                .register_opening_regions(&context, &x.region_routes)
+                .map_err(BoxedError::new)
+                .context(ExternalSnafu)?;
+        }
 
         Ok(CreateTableProcedure { context, creator })
     }
@@ -94,10 +97,6 @@ impl CreateTableProcedure {
         self.table_info().ident.table_id
     }
 
-    pub fn region_routes(&self) -> &Vec<RegionRoute> {
-        &self.creator.data.region_routes
-    }
-
     pub fn region_wal_options(&self) -> &HashMap<RegionNumber, String> {
         &self.creator.data.region_wal_options
     }
@@ -132,7 +131,10 @@ impl CreateTableProcedure {
         Ok(Status::executing(true))
     }
 
-    pub fn new_region_request_builder(&self) -> Result<CreateRequestBuilder> {
+    pub fn new_region_request_builder(
+        &self,
+        physical_table_id: Option<TableId>,
+    ) -> Result<CreateRequestBuilder> {
         let create_table_expr = &self.creator.data.task.create_table;
 
         let column_defs = create_table_expr
@@ -191,16 +193,54 @@ impl CreateTableProcedure {
             options: create_table_expr.table_options.clone(),
         };
 
-        let builder = CreateRequestBuilder::new_template(self.context.clone(), template);
-        Ok(builder)
+        Ok(CreateRequestBuilder {
+            template,
+            physical_table_id,
+        })
     }
 
     pub async fn on_datanode_create_regions(&mut self) -> Result<Status> {
+        match &self.creator.data.table_route {
+            TableRouteValue::Physical(x) => {
+                let region_routes = x.region_routes.clone();
+                let request_builder = self.new_region_request_builder(None)?;
+                self.create_regions(&region_routes, request_builder).await
+            }
+            TableRouteValue::Logical(x) => {
+                let physical_table_id = x.physical_table_id();
+
+                let physical_table_route = self
+                    .context
+                    .table_metadata_manager
+                    .table_route_manager()
+                    .get(physical_table_id)
+                    .await?
+                    .context(TableRouteNotFoundSnafu {
+                        table_id: physical_table_id,
+                    })?;
+                let region_routes = physical_table_route.region_routes()?;
+
+                let request_builder = self.new_region_request_builder(Some(physical_table_id))?;
+
+                self.create_regions(region_routes, request_builder).await
+            }
+        }
+    }
+
+    async fn create_regions(
+        &mut self,
+        region_routes: &[RegionRoute],
+        request_builder: CreateRequestBuilder,
+    ) -> Result<Status> {
         // Registers opening regions
-        self.creator.register_opening_regions(&self.context)?;
+        let guards = self
+            .creator
+            .register_opening_regions(&self.context, region_routes)?;
+        if !guards.is_empty() {
+            self.creator.opening_regions = guards;
+        }
 
         let create_table_data = &self.creator.data;
-        let region_routes = &create_table_data.region_routes;
         let region_wal_options = &create_table_data.region_wal_options;
 
         let create_table_expr = &create_table_data.task.create_table;
@@ -208,8 +248,6 @@ impl CreateTableProcedure {
         let schema = &create_table_expr.schema_name;
         let storage_path = region_storage_path(catalog, schema);
 
-        let mut request_builder = self.new_region_request_builder()?;
-
         let leaders = find_leaders(region_routes);
         let mut create_region_tasks = Vec::with_capacity(leaders.len());
 
@@ -221,12 +259,7 @@ impl CreateTableProcedure {
             for region_number in regions {
                 let region_id = RegionId::new(self.table_id(), region_number);
                 let create_region_request = request_builder
-                    .build_one(
-                        &self.creator.data.task.create_table,
-                        region_id,
-                        storage_path.clone(),
-                        region_wal_options,
-                    )
+                    .build_one(region_id, storage_path.clone(), region_wal_options)
                     .await?;
 
                 requests.push(PbRegionRequest::Create(create_region_request));
@@ -270,10 +303,13 @@ impl CreateTableProcedure {
         let manager = &self.context.table_metadata_manager;
 
         let raw_table_info = self.table_info().clone();
-        let region_routes = self.region_routes().clone();
         let region_wal_options = self.region_wal_options().clone();
         manager
-            .create_table_metadata(raw_table_info, region_routes, region_wal_options)
+            .create_table_metadata(
+                raw_table_info,
+                self.creator.data.table_route.clone(),
+                region_wal_options,
+            )
             .await?;
         info!("Created table metadata for table {table_id}");
 
@@ -308,13 +344,12 @@ impl Procedure for CreateTableProcedure {
 
     fn lock_key(&self) -> LockKey {
         let table_ref = &self.creator.data.table_ref();
-        let key = common_catalog::format_full_table_name(
+
+        LockKey::single(TableNameLock::new(
             table_ref.catalog,
             table_ref.schema,
             table_ref.table,
-        );
-
-        LockKey::single(key)
+        ))
     }
 }
 
@@ -329,7 +364,7 @@ impl TableCreator {
     pub fn new(
         cluster_id: u64,
         task: CreateTableTask,
-        region_routes: Vec<RegionRoute>,
+        table_route: TableRouteValue,
         region_wal_options: HashMap<RegionNumber, String>,
     ) -> Self {
         Self {
@@ -337,21 +372,23 @@ impl TableCreator {
                 state: CreateTableState::Prepare,
                 cluster_id,
                 task,
-                region_routes,
+                table_route,
                 region_wal_options,
             },
             opening_regions: vec![],
         }
     }
 
-    /// Register opening regions if doesn't exist.
-    pub fn register_opening_regions(&mut self, context: &DdlContext) -> Result<()> {
-        let region_routes = &self.data.region_routes;
-
+    /// Registers and returns the guards of the opening region if they don't exist.
+    fn register_opening_regions(
+        &self,
+        context: &DdlContext,
+        region_routes: &[RegionRoute],
+    ) -> Result<Vec<OperatingRegionGuard>> {
         let opening_regions = operating_leader_regions(region_routes);
 
         if self.opening_regions.len() == opening_regions.len() {
-            return Ok(());
+            return Ok(vec![]);
         }
 
         let mut opening_region_guards = Vec::with_capacity(opening_regions.len());
@@ -366,9 +403,7 @@ impl TableCreator {
                 })?;
             opening_region_guards.push(guard);
         }
-
-        self.opening_regions = opening_region_guards;
-        Ok(())
+        Ok(opening_region_guards)
     }
 }
 
@@ -386,7 +421,7 @@ pub enum CreateTableState {
 pub struct CreateTableData {
     pub state: CreateTableState,
     pub task: CreateTableTask,
-    pub region_routes: Vec<RegionRoute>,
+    table_route: TableRouteValue,
     pub region_wal_options: HashMap<RegionNumber, String>,
     pub cluster_id: u64,
 }
@@ -399,28 +434,18 @@ impl CreateTableData {
 
 /// Builder for [PbCreateRegionRequest].
 pub struct CreateRequestBuilder {
-    context: DdlContext,
     template: PbCreateRegionRequest,
     /// Optional. Only for metric engine.
     physical_table_id: Option<TableId>,
 }
 
 impl CreateRequestBuilder {
-    fn new_template(context: DdlContext, template: PbCreateRegionRequest) -> Self {
-        Self {
-            context,
-            template,
-            physical_table_id: None,
-        }
-    }
-
     pub fn template(&self) -> &PbCreateRegionRequest {
         &self.template
     }
 
     async fn build_one(
-        &mut self,
-        create_expr: &CreateTableExpr,
+        &self,
         region_id: RegionId,
         storage_path: String,
         region_wal_options: &HashMap<RegionNumber, String>,
@@ -430,57 +455,20 @@ impl CreateRequestBuilder {
         request.region_id = region_id.as_u64();
         request.path = storage_path;
         // Stores the encoded wal options into the request options.
-        region_wal_options
-            .get(&region_id.region_number())
-            .and_then(|wal_options| {
-                request
-                    .options
-                    .insert(WAL_OPTIONS_KEY.to_string(), wal_options.clone())
-            });
+        prepare_wal_option(&mut request.options, region_id, region_wal_options);
 
-        if self.template.engine == METRIC_ENGINE {
-            self.metric_engine_hook(create_expr, region_id, &mut request)
-                .await?;
-        }
+        if let Some(physical_table_id) = self.physical_table_id {
+            // Logical table has the same region numbers with physical table, and they have a one-to-one mapping.
+            // For example, region 0 of logical table must resides with region 0 of physical table. So here we can
+            // simply concat the physical table id and the logical region number to get the physical region id.
+            let physical_region_id = RegionId::new(physical_table_id, region_id.region_number());
 
-        Ok(request)
-    }
-
-    async fn metric_engine_hook(
-        &mut self,
-        create_expr: &CreateTableExpr,
-        region_id: RegionId,
-        request: &mut PbCreateRegionRequest,
-    ) -> Result<()> {
-        if let Some(physical_table_name) = request.options.get(LOGICAL_TABLE_METADATA_KEY) {
-            let table_id = if let Some(table_id) = self.physical_table_id {
-                table_id
-            } else {
-                let table_name_manager = self.context.table_metadata_manager.table_name_manager();
-                let table_name_key = TableNameKey::new(
-                    &create_expr.catalog_name,
-                    &create_expr.schema_name,
-                    physical_table_name,
-                );
-                let table_id = table_name_manager
-                    .get(table_name_key)
-                    .await?
-                    .context(TableInfoNotFoundSnafu {
-                        table_name: physical_table_name,
-                    })?
-                    .table_id();
-                self.physical_table_id = Some(table_id);
-                table_id
-            };
-            // Concat physical table's table id and corresponding region number to get
-            // the physical region id.
-            let physical_region_id = RegionId::new(table_id, region_id.region_number());
             request.options.insert(
                 LOGICAL_TABLE_METADATA_KEY.to_string(),
                 physical_region_id.as_u64().to_string(),
             );
         }
 
-        Ok(())
+        Ok(request)
     }
 }

src/common/meta/src/ddl/drop_table.rs

@@ -41,6 +41,7 @@ use crate::key::table_info::TableInfoValue;
 use crate::key::table_name::TableNameKey;
 use crate::key::table_route::TableRouteValue;
 use crate::key::DeserializedValueWithBytes;
+use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
 use crate::metrics;
 use crate::region_keeper::OperatingRegionGuard;
 use crate::rpc::ddl::DropTableTask;
@@ -116,7 +117,7 @@ impl DropTableProcedure {
 
     /// Register dropping regions if doesn't exist.
     fn register_dropping_regions(&mut self) -> Result<()> {
-        let region_routes = self.data.region_routes();
+        let region_routes = self.data.region_routes()?;
 
         let dropping_regions = operating_leader_regions(region_routes);
 
@@ -190,7 +191,7 @@ impl DropTableProcedure {
     pub async fn on_datanode_drop_regions(&self) -> Result<Status> {
         let table_id = self.data.table_id();
 
-        let region_routes = &self.data.region_routes();
+        let region_routes = &self.data.region_routes()?;
         let leaders = find_leaders(region_routes);
         let mut drop_region_tasks = Vec::with_capacity(leaders.len());
 
@@ -267,13 +268,14 @@ impl Procedure for DropTableProcedure {
 
     fn lock_key(&self) -> LockKey {
         let table_ref = &self.data.table_ref();
-        let key = common_catalog::format_full_table_name(
-            table_ref.catalog,
-            table_ref.schema,
-            table_ref.table,
-        );
+        let table_id = self.data.table_id();
+        let lock_key = vec![
+            CatalogLock::Read(table_ref.catalog).into(),
+            SchemaLock::read(table_ref.catalog, table_ref.schema).into(),
+            TableLock::Write(table_id).into(),
+        ];
 
-        LockKey::single(key)
+        LockKey::new(lock_key)
     }
 }
 
@@ -306,8 +308,8 @@ impl DropTableData {
         self.task.table_ref()
     }
 
-    fn region_routes(&self) -> &Vec<RegionRoute> {
-        &self.table_route_value.region_routes
+    fn region_routes(&self) -> Result<&Vec<RegionRoute>> {
+        self.table_route_value.region_routes()
     }
 
     fn table_info(&self) -> &RawTableInfo {

src/common/meta/src/ddl/table_meta.rs

@@ -0,0 +1,223 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_catalog::consts::METRIC_ENGINE;
+use common_telemetry::{debug, info};
+use snafu::{ensure, OptionExt};
+use store_api::metric_engine_consts::LOGICAL_TABLE_METADATA_KEY;
+use store_api::storage::{RegionId, RegionNumber, TableId};
+
+use crate::ddl::{TableMetadata, TableMetadataAllocatorContext};
+use crate::error::{Result, TableNotFoundSnafu, UnsupportedSnafu};
+use crate::key::table_name::TableNameKey;
+use crate::key::table_route::{LogicalTableRouteValue, PhysicalTableRouteValue, TableRouteValue};
+use crate::key::TableMetadataManagerRef;
+use crate::peer::Peer;
+use crate::rpc::ddl::CreateTableTask;
+use crate::rpc::router::{Region, RegionRoute};
+use crate::sequence::SequenceRef;
+use crate::wal::{allocate_region_wal_options, WalOptionsAllocatorRef};
+
+#[derive(Clone)]
+pub struct TableMetadataAllocator {
+    table_id_sequence: SequenceRef,
+    wal_options_allocator: WalOptionsAllocatorRef,
+    table_metadata_manager: TableMetadataManagerRef,
+    peer_allocator: PeerAllocatorRef,
+}
+
+impl TableMetadataAllocator {
+    pub fn new(
+        table_id_sequence: SequenceRef,
+        wal_options_allocator: WalOptionsAllocatorRef,
+        table_metadata_manager: TableMetadataManagerRef,
+    ) -> Self {
+        Self::with_peer_allocator(
+            table_id_sequence,
+            wal_options_allocator,
+            table_metadata_manager,
+            Arc::new(NoopPeerAllocator),
+        )
+    }
+
+    pub fn with_peer_allocator(
+        table_id_sequence: SequenceRef,
+        wal_options_allocator: WalOptionsAllocatorRef,
+        table_metadata_manager: TableMetadataManagerRef,
+        peer_allocator: PeerAllocatorRef,
+    ) -> Self {
+        Self {
+            table_id_sequence,
+            wal_options_allocator,
+            table_metadata_manager,
+            peer_allocator,
+        }
+    }
+
+    async fn allocate_table_id(&self, task: &CreateTableTask) -> Result<TableId> {
+        let table_id = if let Some(table_id) = &task.create_table.table_id {
+            let table_id = table_id.id;
+
+            ensure!(
+                !self
+                    .table_id_sequence
+                    .min_max()
+                    .await
+                    .contains(&(table_id as u64)),
+                UnsupportedSnafu {
+                    operation: format!(
+                        "create table by id {} that is reserved in this node",
+                        table_id
+                    )
+                }
+            );
+
+            info!(
+                "Received explicitly allocated table id {}, will use it directly.",
+                table_id
+            );
+
+            table_id
+        } else {
+            self.table_id_sequence.next().await? as TableId
+        };
+        Ok(table_id)
+    }
+
+    fn create_wal_options(
+        &self,
+        table_route: &TableRouteValue,
+    ) -> Result<HashMap<RegionNumber, String>> {
+        match table_route {
+            TableRouteValue::Physical(x) => {
+                let region_numbers = x
+                    .region_routes
+                    .iter()
+                    .map(|route| route.region.id.region_number())
+                    .collect();
+                allocate_region_wal_options(region_numbers, &self.wal_options_allocator)
+            }
+            TableRouteValue::Logical(_) => Ok(HashMap::new()),
+        }
+    }
+
+    async fn create_table_route(
+        &self,
+        ctx: &TableMetadataAllocatorContext,
+        table_id: TableId,
+        task: &CreateTableTask,
+    ) -> Result<TableRouteValue> {
+        let regions = task.partitions.len();
+
+        let table_route = if task.create_table.engine == METRIC_ENGINE
+            && let Some(physical_table_name) = task
+                .create_table
+                .table_options
+                .get(LOGICAL_TABLE_METADATA_KEY)
+        {
+            let physical_table_id = self
+                .table_metadata_manager
+                .table_name_manager()
+                .get(TableNameKey::new(
+                    &task.create_table.catalog_name,
+                    &task.create_table.schema_name,
+                    physical_table_name,
+                ))
+                .await?
+                .context(TableNotFoundSnafu {
+                    table_name: physical_table_name,
+                })?
+                .table_id();
+
+            let region_ids = (0..regions)
+                .map(|i| RegionId::new(table_id, i as RegionNumber))
+                .collect();
+
+            TableRouteValue::Logical(LogicalTableRouteValue::new(physical_table_id, region_ids))
+        } else {
+            let peers = self.peer_allocator.alloc(ctx, regions).await?;
+
+            let region_routes = task
+                .partitions
+                .iter()
+                .enumerate()
+                .map(|(i, partition)| {
+                    let region = Region {
+                        id: RegionId::new(table_id, i as u32),
+                        partition: Some(partition.clone().into()),
+                        ..Default::default()
+                    };
+
+                    let peer = peers[i % peers.len()].clone();
+
+                    RegionRoute {
+                        region,
+                        leader_peer: Some(peer),
+                        ..Default::default()
+                    }
+                })
+                .collect::<Vec<_>>();
+            TableRouteValue::Physical(PhysicalTableRouteValue::new(region_routes))
+        };
+        Ok(table_route)
+    }
+
+    pub async fn create(
+        &self,
+        ctx: &TableMetadataAllocatorContext,
+        task: &CreateTableTask,
+    ) -> Result<TableMetadata> {
+        let table_id = self.allocate_table_id(task).await?;
+        let table_route = self.create_table_route(ctx, table_id, task).await?;
+        let region_wal_options = self.create_wal_options(&table_route)?;
+
+        debug!(
+            "Allocated region wal options {:?} for table {}",
+            region_wal_options, table_id
+        );
+
+        Ok(TableMetadata {
+            table_id,
+            table_route,
+            region_wal_options,
+        })
+    }
+}
+
+pub type PeerAllocatorRef = Arc<dyn PeerAllocator>;
+
+/// [PeerAllocator] allocates [Peer]s for creating regions.
+#[async_trait]
+pub trait PeerAllocator: Send + Sync {
+    /// Allocates `regions` size [Peer]s.
+    async fn alloc(&self, ctx: &TableMetadataAllocatorContext, regions: usize)
+        -> Result<Vec<Peer>>;
+}
+
+struct NoopPeerAllocator;
+
+#[async_trait]
+impl PeerAllocator for NoopPeerAllocator {
+    async fn alloc(
+        &self,
+        _ctx: &TableMetadataAllocatorContext,
+        regions: usize,
+    ) -> Result<Vec<Peer>> {
+        Ok(vec![Peer::default(); regions])
+    }
+}

src/common/meta/src/ddl/truncate_table.rs

@@ -37,6 +37,7 @@ use crate::error::{Result, TableNotFoundSnafu};
 use crate::key::table_info::TableInfoValue;
 use crate::key::table_name::TableNameKey;
 use crate::key::DeserializedValueWithBytes;
+use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
 use crate::metrics;
 use crate::rpc::ddl::TruncateTableTask;
 use crate::rpc::router::{find_leader_regions, find_leaders, RegionRoute};
@@ -75,13 +76,14 @@ impl Procedure for TruncateTableProcedure {
 
     fn lock_key(&self) -> LockKey {
         let table_ref = &self.data.table_ref();
-        let key = common_catalog::format_full_table_name(
-            table_ref.catalog,
-            table_ref.schema,
-            table_ref.table,
-        );
+        let table_id = self.data.table_id();
+        let lock_key = vec![
+            CatalogLock::Read(table_ref.catalog).into(),
+            SchemaLock::read(table_ref.catalog, table_ref.schema).into(),
+            TableLock::Write(table_id).into(),
+        ];
 
-        LockKey::single(key)
+        LockKey::new(lock_key)
     }
 }
 

src/common/meta/src/ddl/utils.rs

@@ -36,7 +36,7 @@ pub fn handle_operate_region_error(datanode: Peer) -> impl FnOnce(crate::error::
 }
 
 pub fn handle_retry_error(e: Error) -> ProcedureError {
-    if matches!(e, error::Error::RetryLater { .. }) {
+    if e.is_retry_later() {
         ProcedureError::retry_later(e)
     } else {
         ProcedureError::external(e)

src/common/meta/src/ddl_manager.rs

@@ -19,17 +19,17 @@ use common_procedure::{watcher, ProcedureId, ProcedureManagerRef, ProcedureWithI
 use common_telemetry::tracing_context::{FutureExt, TracingContext};
 use common_telemetry::{info, tracing};
 use snafu::{OptionExt, ResultExt};
-use store_api::storage::RegionNumber;
+use store_api::storage::{RegionNumber, TableId};
 
 use crate::cache_invalidator::CacheInvalidatorRef;
 use crate::datanode_manager::DatanodeManagerRef;
 use crate::ddl::alter_table::AlterTableProcedure;
 use crate::ddl::create_table::CreateTableProcedure;
 use crate::ddl::drop_table::DropTableProcedure;
+use crate::ddl::table_meta::TableMetadataAllocator;
 use crate::ddl::truncate_table::TruncateTableProcedure;
 use crate::ddl::{
     DdlContext, DdlTaskExecutor, ExecutorContext, TableMetadata, TableMetadataAllocatorContext,
-    TableMetadataAllocatorRef,
 };
 use crate::error::{
     self, RegisterProcedureLoaderSnafu, Result, SubmitProcedureSnafu, TableNotFoundSnafu,
@@ -46,6 +46,8 @@ use crate::rpc::ddl::{
     TruncateTableTask,
 };
 use crate::rpc::router::RegionRoute;
+use crate::table_name::TableName;
+
 pub type DdlManagerRef = Arc<DdlManager>;
 
 /// The [DdlManager] provides the ability to execute Ddl.
@@ -54,7 +56,7 @@ pub struct DdlManager {
     datanode_manager: DatanodeManagerRef,
     cache_invalidator: CacheInvalidatorRef,
     table_metadata_manager: TableMetadataManagerRef,
-    table_metadata_allocator: TableMetadataAllocatorRef,
+    table_metadata_allocator: TableMetadataAllocator,
     memory_region_keeper: MemoryRegionKeeperRef,
 }
 
@@ -65,7 +67,7 @@ impl DdlManager {
         datanode_clients: DatanodeManagerRef,
         cache_invalidator: CacheInvalidatorRef,
         table_metadata_manager: TableMetadataManagerRef,
-        table_metadata_allocator: TableMetadataAllocatorRef,
+        table_metadata_allocator: TableMetadataAllocator,
         memory_region_keeper: MemoryRegionKeeperRef,
     ) -> Result<Self> {
         let manager = Self {
@@ -160,11 +162,17 @@ impl DdlManager {
         cluster_id: u64,
         alter_table_task: AlterTableTask,
         table_info_value: DeserializedValueWithBytes<TableInfoValue>,
+        physical_table_info: Option<(TableId, TableName)>,
     ) -> Result<ProcedureId> {
         let context = self.create_context();
 
-        let procedure =
-            AlterTableProcedure::new(cluster_id, alter_table_task, table_info_value, context)?;
+        let procedure = AlterTableProcedure::new(
+            cluster_id,
+            alter_table_task,
+            table_info_value,
+            physical_table_info,
+            context,
+        )?;
 
         let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
 
@@ -177,7 +185,7 @@ impl DdlManager {
         &self,
         cluster_id: u64,
         create_table_task: CreateTableTask,
-        region_routes: Vec<RegionRoute>,
+        table_route: TableRouteValue,
         region_wal_options: HashMap<RegionNumber, String>,
     ) -> Result<ProcedureId> {
         let context = self.create_context();
@@ -185,7 +193,7 @@ impl DdlManager {
         let procedure = CreateTableProcedure::new(
             cluster_id,
             create_table_task,
-            region_routes,
+            table_route,
             region_wal_options,
             context,
         );
@@ -275,11 +283,10 @@ async fn handle_truncate_table_task(
         table_name: table_ref.to_string(),
     })?;
 
-    let table_route_value = table_route_value.with_context(|| error::TableRouteNotFoundSnafu {
-        table_name: table_ref.to_string(),
-    })?;
+    let table_route_value =
+        table_route_value.context(error::TableRouteNotFoundSnafu { table_id })?;
 
-    let table_route = table_route_value.into_inner().region_routes;
+    let table_route = table_route_value.into_inner().region_routes()?.clone();
 
     let id = ddl_manager
         .submit_truncate_table_task(
@@ -328,8 +335,41 @@ async fn handle_alter_table_task(
             table_name: table_ref.to_string(),
         })?;
 
+    let physical_table_id = ddl_manager
+        .table_metadata_manager()
+        .table_route_manager()
+        .get_physical_table_id(table_id)
+        .await?;
+
+    let physical_table_info = if physical_table_id == table_id {
+        None
+    } else {
+        let physical_table_info = &ddl_manager
+            .table_metadata_manager()
+            .table_info_manager()
+            .get(physical_table_id)
+            .await?
+            .with_context(|| error::TableInfoNotFoundSnafu {
+                table_name: table_ref.to_string(),
+            })?
+            .table_info;
+        Some((
+            physical_table_id,
+            TableName {
+                catalog_name: physical_table_info.catalog_name.clone(),
+                schema_name: physical_table_info.schema_name.clone(),
+                table_name: physical_table_info.name.clone(),
+            },
+        ))
+    };
+
     let id = ddl_manager
-        .submit_alter_table_task(cluster_id, alter_table_task, table_info_value)
+        .submit_alter_table_task(
+            cluster_id,
+            alter_table_task,
+            table_info_value,
+            physical_table_info,
+        )
         .await?;
 
     info!("Table: {table_id} is altered via procedure_id {id:?}");
@@ -349,16 +389,21 @@ async fn handle_drop_table_task(
     let table_metadata_manager = &ddl_manager.table_metadata_manager();
     let table_ref = drop_table_task.table_ref();
 
-    let (table_info_value, table_route_value) =
-        table_metadata_manager.get_full_table_info(table_id).await?;
+    let table_info_value = table_metadata_manager
+        .table_info_manager()
+        .get(table_id)
+        .await?;
+    let (_, table_route_value) = table_metadata_manager
+        .table_route_manager()
+        .get_physical_table_route(table_id)
+        .await?;
 
     let table_info_value = table_info_value.with_context(|| error::TableInfoNotFoundSnafu {
         table_name: table_ref.to_string(),
     })?;
 
-    let table_route_value = table_route_value.with_context(|| error::TableRouteNotFoundSnafu {
-        table_name: table_ref.to_string(),
-    })?;
+    let table_route_value =
+        DeserializedValueWithBytes::from_inner(TableRouteValue::Physical(table_route_value));
 
     let id = ddl_manager
         .submit_drop_table_task(
@@ -392,7 +437,7 @@ async fn handle_create_table_task(
 
     let TableMetadata {
         table_id,
-        region_routes,
+        table_route,
         region_wal_options,
     } = table_meta;
 
@@ -402,7 +447,7 @@ async fn handle_create_table_task(
         .submit_create_table_task(
             cluster_id,
             create_table_task,
-            region_routes,
+            table_route,
             region_wal_options,
         )
         .await?;
@@ -463,15 +508,15 @@ mod tests {
     use crate::ddl::alter_table::AlterTableProcedure;
     use crate::ddl::create_table::CreateTableProcedure;
     use crate::ddl::drop_table::DropTableProcedure;
+    use crate::ddl::table_meta::TableMetadataAllocator;
     use crate::ddl::truncate_table::TruncateTableProcedure;
-    use crate::ddl::{TableMetadata, TableMetadataAllocator, TableMetadataAllocatorContext};
-    use crate::error::Result;
     use crate::key::TableMetadataManager;
     use crate::kv_backend::memory::MemoryKvBackend;
     use crate::peer::Peer;
     use crate::region_keeper::MemoryRegionKeeper;
-    use crate::rpc::ddl::CreateTableTask;
+    use crate::sequence::SequenceBuilder;
     use crate::state_store::KvStateStore;
+    use crate::wal::WalOptionsAllocator;
 
     /// A dummy implemented [DatanodeManager].
     pub struct DummyDatanodeManager;
@@ -483,26 +528,12 @@ mod tests {
         }
     }
 
-    /// A dummy implemented [TableMetadataAllocator].
-    pub struct DummyTableMetadataAllocator;
-
-    #[async_trait::async_trait]
-    impl TableMetadataAllocator for DummyTableMetadataAllocator {
-        async fn create(
-            &self,
-            _ctx: &TableMetadataAllocatorContext,
-            _task: &CreateTableTask,
-        ) -> Result<TableMetadata> {
-            unimplemented!()
-        }
-    }
-
     #[test]
     fn test_try_new() {
         let kv_backend = Arc::new(MemoryKvBackend::new());
         let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
 
-        let state_store = Arc::new(KvStateStore::new(kv_backend));
+        let state_store = Arc::new(KvStateStore::new(kv_backend.clone()));
         let procedure_manager = Arc::new(LocalManager::new(Default::default(), state_store));
 
         let _ = DdlManager::try_new(
@@ -510,7 +541,11 @@ mod tests {
             Arc::new(DummyDatanodeManager),
             Arc::new(DummyCacheInvalidator),
             table_metadata_manager,
-            Arc::new(DummyTableMetadataAllocator),
+            TableMetadataAllocator::new(
+                Arc::new(SequenceBuilder::new("test", kv_backend.clone()).build()),
+                Arc::new(WalOptionsAllocator::default()),
+                Arc::new(TableMetadataManager::new(kv_backend)),
+            ),
             Arc::new(MemoryRegionKeeper::default()),
         );
 

src/common/meta/src/error.rs

@@ -135,9 +135,9 @@ pub enum Error {
         source: table::error::Error,
     },
 
-    #[snafu(display("Table route not found: {}", table_name))]
+    #[snafu(display("Failed to find table route for table id {}", table_id))]
     TableRouteNotFound {
-        table_name: String,
+        table_id: TableId,
         location: Location,
     },
 
@@ -321,6 +321,27 @@ pub enum Error {
         error: rskafka::client::error::Error,
     },
 
+    #[snafu(display(
+        "Failed to build a Kafka partition client, topic: {}, partition: {}",
+        topic,
+        partition
+    ))]
+    BuildKafkaPartitionClient {
+        topic: String,
+        partition: i32,
+        location: Location,
+        #[snafu(source)]
+        error: rskafka::client::error::Error,
+    },
+
+    #[snafu(display("Failed to produce records to Kafka, topic: {}", topic))]
+    ProduceRecord {
+        topic: String,
+        location: Location,
+        #[snafu(source)]
+        error: rskafka::client::error::Error,
+    },
+
     #[snafu(display("Failed to create a Kafka wal topic"))]
     CreateKafkaWalTopic {
         location: Location,
@@ -330,6 +351,9 @@ pub enum Error {
 
     #[snafu(display("The topic pool is empty"))]
     EmptyTopicPool { location: Location },
+
+    #[snafu(display("Unexpected table route type: {}", err_msg))]
+    UnexpectedLogicalRouteTable { location: Location, err_msg: String },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -368,8 +392,11 @@ impl ErrorExt for Error {
             | EncodeWalOptions { .. }
             | BuildKafkaClient { .. }
             | BuildKafkaCtrlClient { .. }
+            | BuildKafkaPartitionClient { .. }
+            | ProduceRecord { .. }
             | CreateKafkaWalTopic { .. }
-            | EmptyTopicPool { .. } => StatusCode::Unexpected,
+            | EmptyTopicPool { .. }
+            | UnexpectedLogicalRouteTable { .. } => StatusCode::Unexpected,
 
             SendMessage { .. }
             | GetKvCache { .. }

src/common/meta/src/instruction.rs

@@ -14,6 +14,7 @@
 
 use std::collections::HashMap;
 use std::fmt::{Display, Formatter};
+use std::time::Duration;
 
 use serde::{Deserialize, Serialize};
 use store_api::storage::{RegionId, RegionNumber};
@@ -91,13 +92,15 @@ impl Display for OpenRegion {
     }
 }
 
+#[serde_with::serde_as]
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct OpenRegion {
     pub region_ident: RegionIdent,
     pub region_storage_path: String,
     pub region_options: HashMap<String, String>,
     #[serde(default)]
-    pub region_wal_options: HashMap<String, String>,
+    #[serde_as(as = "HashMap<serde_with::DisplayFromStr, _>")]
+    pub region_wal_options: HashMap<RegionNumber, String>,
     #[serde(default)]
     pub skip_wal_replay: bool,
 }
@@ -107,7 +110,7 @@ impl OpenRegion {
         region_ident: RegionIdent,
         path: &str,
         region_options: HashMap<String, String>,
-        region_wal_options: HashMap<String, String>,
+        region_wal_options: HashMap<RegionNumber, String>,
         skip_wal_replay: bool,
     ) -> Self {
         Self {
@@ -140,11 +143,12 @@ pub struct UpgradeRegion {
     pub region_id: RegionId,
     /// The `last_entry_id` of old leader region.
     pub last_entry_id: Option<u64>,
-    /// The second of waiting for a wal replay.
+    /// The timeout of waiting for a wal replay.
     ///
     /// `None` stands for no wait,
     /// it's helpful to verify whether the leader region is ready.
-    pub wait_for_replay_secs: Option<u64>,
+    #[serde(with = "humantime_serde")]
+    pub wait_for_replay_timeout: Option<Duration>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Display)]

src/common/meta/src/key.rs

@@ -147,6 +147,14 @@ pub trait TableMetaKey {
     fn as_raw_key(&self) -> Vec<u8>;
 }
 
+pub trait TableMetaValue {
+    fn try_from_raw_value(raw_value: &[u8]) -> Result<Self>
+    where
+        Self: Sized;
+
+    fn try_as_raw_value(&self) -> Result<Vec<u8>>;
+}
+
 pub type TableMetadataManagerRef = Arc<TableMetadataManager>;
 
 pub struct TableMetadataManager {
@@ -221,7 +229,9 @@ impl<T: DeserializeOwned + Serialize> Serialize for DeserializedValueWithBytes<T
     }
 }
 
-impl<'de, T: DeserializeOwned + Serialize> Deserialize<'de> for DeserializedValueWithBytes<T> {
+impl<'de, T: DeserializeOwned + Serialize + TableMetaValue> Deserialize<'de>
+    for DeserializedValueWithBytes<T>
+{
     /// - Deserialize behaviors:
     ///
     /// The `inner` field will be deserialized from the `bytes` field.
@@ -248,11 +258,11 @@ impl<T: Serialize + DeserializeOwned + Clone> Clone for DeserializedValueWithByt
     }
 }
 
-impl<T: Serialize + DeserializeOwned> DeserializedValueWithBytes<T> {
+impl<T: Serialize + DeserializeOwned + TableMetaValue> DeserializedValueWithBytes<T> {
     /// Returns a struct containing a deserialized value and an original `bytes`.
     /// It accepts original bytes of inner.
     pub fn from_inner_bytes(bytes: Bytes) -> Result<Self> {
-        let inner = serde_json::from_slice(&bytes).context(error::SerdeJsonSnafu)?;
+        let inner = T::try_from_raw_value(&bytes)?;
         Ok(Self { bytes, inner })
     }
 
@@ -344,6 +354,7 @@ impl TableMetadataManager {
         &self.kv_backend
     }
 
+    // TODO(ruihang): deprecate this
     pub async fn get_full_table_info(
         &self,
         table_id: TableId,
@@ -373,13 +384,10 @@ impl TableMetadataManager {
     pub async fn create_table_metadata(
         &self,
         mut table_info: RawTableInfo,
-        region_routes: Vec<RegionRoute>,
+        table_route_value: TableRouteValue,
         region_wal_options: HashMap<RegionNumber, String>,
     ) -> Result<()> {
-        let region_numbers = region_routes
-            .iter()
-            .map(|region| region.region.id.region_number())
-            .collect::<Vec<_>>();
+        let region_numbers = table_route_value.region_numbers();
         table_info.meta.region_numbers = region_numbers;
         let table_id = table_info.ident.table_id;
         let engine = table_info.meta.engine.clone();
@@ -403,30 +411,28 @@ impl TableMetadataManager {
             .table_info_manager()
             .build_create_txn(table_id, &table_info_value)?;
 
-        // Creates datanode table key value pairs.
-        let distribution = region_distribution(&region_routes)?;
-        let create_datanode_table_txn = self.datanode_table_manager().build_create_txn(
-            table_id,
-            &engine,
-            &region_storage_path,
-            region_options,
-            region_wal_options,
-            distribution,
-        )?;
-
-        // Creates table route.
-        let table_route_value = TableRouteValue::new(region_routes);
         let (create_table_route_txn, on_create_table_route_failure) = self
             .table_route_manager()
             .build_create_txn(table_id, &table_route_value)?;
 
-        let txn = Txn::merge_all(vec![
+        let mut txn = Txn::merge_all(vec![
             create_table_name_txn,
             create_table_info_txn,
-            create_datanode_table_txn,
             create_table_route_txn,
         ]);
 
+        if let TableRouteValue::Physical(x) = &table_route_value {
+            let create_datanode_table_txn = self.datanode_table_manager().build_create_txn(
+                table_id,
+                &engine,
+                &region_storage_path,
+                region_options,
+                region_wal_options,
+                region_distribution(&x.region_routes),
+            )?;
+            txn = txn.merge(create_datanode_table_txn);
+        }
+
         let r = self.kv_backend.txn(txn).await?;
 
         // Checks whether metadata was already created.
@@ -478,7 +484,7 @@ impl TableMetadataManager {
             .build_delete_txn(table_id, table_info_value)?;
 
         // Deletes datanode table key value pairs.
-        let distribution = region_distribution(&table_route_value.region_routes)?;
+        let distribution = region_distribution(table_route_value.region_routes()?);
         let delete_datanode_txn = self
             .datanode_table_manager()
             .build_delete_txn(table_id, distribution)?;
@@ -599,12 +605,12 @@ impl TableMetadataManager {
         current_table_route_value: &DeserializedValueWithBytes<TableRouteValue>,
         new_region_routes: Vec<RegionRoute>,
         new_region_options: &HashMap<String, String>,
-        new_region_wal_options: &HashMap<String, String>,
+        new_region_wal_options: &HashMap<RegionNumber, String>,
     ) -> Result<()> {
         // Updates the datanode table key value pairs.
         let current_region_distribution =
-            region_distribution(&current_table_route_value.region_routes)?;
-        let new_region_distribution = region_distribution(&new_region_routes)?;
+            region_distribution(current_table_route_value.region_routes()?);
+        let new_region_distribution = region_distribution(&new_region_routes);
 
         let update_datanode_table_txn = self.datanode_table_manager().build_update_txn(
             table_id,
@@ -616,7 +622,7 @@ impl TableMetadataManager {
         )?;
 
         // Updates the table_route.
-        let new_table_route_value = current_table_route_value.update(new_region_routes);
+        let new_table_route_value = current_table_route_value.update(new_region_routes)?;
 
         let (update_table_route_txn, on_update_table_route_failure) = self
             .table_route_manager()
@@ -651,7 +657,7 @@ impl TableMetadataManager {
     where
         F: Fn(&RegionRoute) -> Option<Option<RegionStatus>>,
     {
-        let mut new_region_routes = current_table_route_value.region_routes.clone();
+        let mut new_region_routes = current_table_route_value.region_routes()?.clone();
 
         let mut updated = 0;
         for route in &mut new_region_routes {
@@ -668,7 +674,7 @@ impl TableMetadataManager {
         }
 
         // Updates the table_route.
-        let new_table_route_value = current_table_route_value.update(new_region_routes);
+        let new_table_route_value = current_table_route_value.update(new_region_routes)?;
 
         let (update_table_route_txn, on_update_table_route_failure) = self
             .table_route_manager()
@@ -711,12 +717,12 @@ impl_table_meta_key!(TableNameKey<'_>, TableInfoKey, DatanodeTableKey);
 macro_rules! impl_table_meta_value {
     ($($val_ty: ty), *) => {
         $(
-            impl $val_ty {
-                pub fn try_from_raw_value(raw_value: &[u8]) -> Result<Self> {
+            impl $crate::key::TableMetaValue for $val_ty {
+                fn try_from_raw_value(raw_value: &[u8]) -> Result<Self> {
                     serde_json::from_slice(raw_value).context(SerdeJsonSnafu)
                 }
 
-                pub fn try_as_raw_value(&self) -> Result<Vec<u8>> {
+                fn try_as_raw_value(&self) -> Result<Vec<u8>> {
                     serde_json::to_vec(self).context(SerdeJsonSnafu)
                 }
             }
@@ -744,8 +750,7 @@ macro_rules! impl_optional_meta_value {
 impl_table_meta_value! {
     TableNameValue,
     TableInfoValue,
-    DatanodeTableValue,
-    TableRouteValue
+    DatanodeTableValue
 }
 
 impl_optional_meta_value! {
@@ -765,6 +770,7 @@ mod tests {
     use super::datanode_table::DatanodeTableKey;
     use super::test_utils;
     use crate::ddl::utils::region_storage_path;
+    use crate::error::Result;
     use crate::key::datanode_table::RegionInfo;
     use crate::key::table_info::TableInfoValue;
     use crate::key::table_name::TableNameKey;
@@ -780,14 +786,14 @@ mod tests {
         let region_routes = vec![region_route.clone()];
 
         let expected_region_routes =
-            TableRouteValue::new(vec![region_route.clone(), region_route.clone()]);
+            TableRouteValue::physical(vec![region_route.clone(), region_route.clone()]);
         let expected = serde_json::to_vec(&expected_region_routes).unwrap();
 
         // Serialize behaviors:
         // The inner field will be ignored.
         let value = DeserializedValueWithBytes {
             // ignored
-            inner: TableRouteValue::new(region_routes.clone()),
+            inner: TableRouteValue::physical(region_routes.clone()),
             bytes: Bytes::from(expected.clone()),
         };
 
@@ -831,43 +837,56 @@ mod tests {
         test_utils::new_test_table_info(10, region_numbers)
     }
 
+    async fn create_physical_table_metadata(
+        table_metadata_manager: &TableMetadataManager,
+        table_info: RawTableInfo,
+        region_routes: Vec<RegionRoute>,
+    ) -> Result<()> {
+        table_metadata_manager
+            .create_table_metadata(
+                table_info,
+                TableRouteValue::physical(region_routes),
+                HashMap::default(),
+            )
+            .await
+    }
+
     #[tokio::test]
     async fn test_create_table_metadata() {
         let mem_kv = Arc::new(MemoryKvBackend::default());
         let table_metadata_manager = TableMetadataManager::new(mem_kv);
         let region_route = new_test_region_route();
-        let region_routes = vec![region_route.clone()];
+        let region_routes = &vec![region_route.clone()];
         let table_info: RawTableInfo =
             new_test_table_info(region_routes.iter().map(|r| r.region.id.region_number())).into();
         // creates metadata.
-        table_metadata_manager
-            .create_table_metadata(
-                table_info.clone(),
-                region_routes.clone(),
-                HashMap::default(),
-            )
-            .await
-            .unwrap();
+        create_physical_table_metadata(
+            &table_metadata_manager,
+            table_info.clone(),
+            region_routes.clone(),
+        )
+        .await
+        .unwrap();
+
         // if metadata was already created, it should be ok.
-        table_metadata_manager
-            .create_table_metadata(
-                table_info.clone(),
-                region_routes.clone(),
-                HashMap::default(),
-            )
-            .await
-            .unwrap();
+        assert!(create_physical_table_metadata(
+            &table_metadata_manager,
+            table_info.clone(),
+            region_routes.clone(),
+        )
+        .await
+        .is_ok());
+
         let mut modified_region_routes = region_routes.clone();
         modified_region_routes.push(region_route.clone());
         // if remote metadata was exists, it should return an error.
-        assert!(table_metadata_manager
-            .create_table_metadata(
-                table_info.clone(),
-                modified_region_routes,
-                HashMap::default()
-            )
-            .await
-            .is_err());
+        assert!(create_physical_table_metadata(
+            &table_metadata_manager,
+            table_info.clone(),
+            modified_region_routes
+        )
+        .await
+        .is_err());
 
         let (remote_table_info, remote_table_route) = table_metadata_manager
             .get_full_table_info(10)
@@ -879,7 +898,11 @@ mod tests {
             table_info
         );
         assert_eq!(
-            remote_table_route.unwrap().into_inner().region_routes,
+            remote_table_route
+                .unwrap()
+                .into_inner()
+                .region_routes()
+                .unwrap(),
             region_routes
         );
     }
@@ -889,23 +912,23 @@ mod tests {
         let mem_kv = Arc::new(MemoryKvBackend::default());
         let table_metadata_manager = TableMetadataManager::new(mem_kv);
         let region_route = new_test_region_route();
-        let region_routes = vec![region_route.clone()];
+        let region_routes = &vec![region_route.clone()];
         let table_info: RawTableInfo =
             new_test_table_info(region_routes.iter().map(|r| r.region.id.region_number())).into();
         let table_id = table_info.ident.table_id;
         let datanode_id = 2;
-        let table_route_value =
-            DeserializedValueWithBytes::from_inner(TableRouteValue::new(region_routes.clone()));
+        let table_route_value = DeserializedValueWithBytes::from_inner(TableRouteValue::physical(
+            region_routes.clone(),
+        ));
 
         // creates metadata.
-        table_metadata_manager
-            .create_table_metadata(
-                table_info.clone(),
-                region_routes.clone(),
-                HashMap::default(),
-            )
-            .await
-            .unwrap();
+        create_physical_table_metadata(
+            &table_metadata_manager,
+            table_info.clone(),
+            region_routes.clone(),
+        )
+        .await
+        .unwrap();
 
         let table_info_value =
             DeserializedValueWithBytes::from_inner(TableInfoValue::new(table_info.clone()));
@@ -960,7 +983,7 @@ mod tests {
             .unwrap()
             .unwrap()
             .into_inner();
-        assert_eq!(removed_table_route.region_routes, region_routes);
+        assert_eq!(removed_table_route.region_routes().unwrap(), region_routes);
     }
 
     #[tokio::test]
@@ -973,14 +996,14 @@ mod tests {
             new_test_table_info(region_routes.iter().map(|r| r.region.id.region_number())).into();
         let table_id = table_info.ident.table_id;
         // creates metadata.
-        table_metadata_manager
-            .create_table_metadata(
-                table_info.clone(),
-                region_routes.clone(),
-                HashMap::default(),
-            )
-            .await
-            .unwrap();
+        create_physical_table_metadata(
+            &table_metadata_manager,
+            table_info.clone(),
+            region_routes.clone(),
+        )
+        .await
+        .unwrap();
+
         let new_table_name = "another_name".to_string();
         let table_info_value =
             DeserializedValueWithBytes::from_inner(TableInfoValue::new(table_info.clone()));
@@ -1045,14 +1068,14 @@ mod tests {
             new_test_table_info(region_routes.iter().map(|r| r.region.id.region_number())).into();
         let table_id = table_info.ident.table_id;
         // creates metadata.
-        table_metadata_manager
-            .create_table_metadata(
-                table_info.clone(),
-                region_routes.clone(),
-                HashMap::default(),
-            )
-            .await
-            .unwrap();
+        create_physical_table_metadata(
+            &table_metadata_manager,
+            table_info.clone(),
+            region_routes.clone(),
+        )
+        .await
+        .unwrap();
+
         let mut new_table_info = table_info.clone();
         new_table_info.name = "hi".to_string();
         let current_table_info_value =
@@ -1123,17 +1146,18 @@ mod tests {
         let table_info: RawTableInfo =
             new_test_table_info(region_routes.iter().map(|r| r.region.id.region_number())).into();
         let table_id = table_info.ident.table_id;
-        let current_table_route_value =
-            DeserializedValueWithBytes::from_inner(TableRouteValue::new(region_routes.clone()));
+        let current_table_route_value = DeserializedValueWithBytes::from_inner(
+            TableRouteValue::physical(region_routes.clone()),
+        );
+
         // creates metadata.
-        table_metadata_manager
-            .create_table_metadata(
-                table_info.clone(),
-                region_routes.clone(),
-                HashMap::default(),
-            )
-            .await
-            .unwrap();
+        create_physical_table_metadata(
+            &table_metadata_manager,
+            table_info.clone(),
+            region_routes.clone(),
+        )
+        .await
+        .unwrap();
 
         table_metadata_manager
             .update_leader_region_status(table_id, &current_table_route_value, |region_route| {
@@ -1154,11 +1178,11 @@ mod tests {
             .unwrap();
 
         assert_eq!(
-            updated_route_value.region_routes[0].leader_status,
+            updated_route_value.region_routes().unwrap()[0].leader_status,
             Some(RegionStatus::Downgraded)
         );
         assert_eq!(
-            updated_route_value.region_routes[1].leader_status,
+            updated_route_value.region_routes().unwrap()[1].leader_status,
             Some(RegionStatus::Downgraded)
         );
     }
@@ -1168,7 +1192,7 @@ mod tests {
         table_id: u32,
         region_routes: &[RegionRoute],
     ) {
-        let region_distribution = region_distribution(region_routes).unwrap();
+        let region_distribution = region_distribution(region_routes);
         for (datanode, regions) in region_distribution {
             let got = table_metadata_manager
                 .datanode_table_manager()
@@ -1193,17 +1217,19 @@ mod tests {
         let engine = table_info.meta.engine.as_str();
         let region_storage_path =
             region_storage_path(&table_info.catalog_name, &table_info.schema_name);
-        let current_table_route_value =
-            DeserializedValueWithBytes::from_inner(TableRouteValue::new(region_routes.clone()));
+        let current_table_route_value = DeserializedValueWithBytes::from_inner(
+            TableRouteValue::physical(region_routes.clone()),
+        );
+
         // creates metadata.
-        table_metadata_manager
-            .create_table_metadata(
-                table_info.clone(),
-                region_routes.clone(),
-                HashMap::default(),
-            )
-            .await
-            .unwrap();
+        create_physical_table_metadata(
+            &table_metadata_manager,
+            table_info.clone(),
+            region_routes.clone(),
+        )
+        .await
+        .unwrap();
+
         assert_datanode_table(&table_metadata_manager, table_id, &region_routes).await;
         let new_region_routes = vec![
             new_region_route(1, 1),
@@ -1250,7 +1276,8 @@ mod tests {
         let current_table_route_value = DeserializedValueWithBytes::from_inner(
             current_table_route_value
                 .inner
-                .update(new_region_routes.clone()),
+                .update(new_region_routes.clone())
+                .unwrap(),
         );
         let new_region_routes = vec![new_region_route(2, 4), new_region_route(5, 5)];
         // it should be ok.
@@ -1274,13 +1301,16 @@ mod tests {
 
         // if the current_table_route_value is wrong, it should return an error.
         // The ABA problem.
-        let wrong_table_route_value =
-            DeserializedValueWithBytes::from_inner(current_table_route_value.update(vec![
-                new_region_route(1, 1),
-                new_region_route(2, 2),
-                new_region_route(3, 3),
-                new_region_route(4, 4),
-            ]));
+        let wrong_table_route_value = DeserializedValueWithBytes::from_inner(
+            current_table_route_value
+                .update(vec![
+                    new_region_route(1, 1),
+                    new_region_route(2, 2),
+                    new_region_route(3, 3),
+                    new_region_route(4, 4),
+                ])
+                .unwrap(),
+        );
         assert!(table_metadata_manager
             .update_table_route(
                 table_id,

src/common/meta/src/key/datanode_table.rs

@@ -24,7 +24,8 @@ use table::metadata::TableId;
 
 use crate::error::{InvalidTableMetadataSnafu, Result};
 use crate::key::{
-    RegionDistribution, TableMetaKey, DATANODE_TABLE_KEY_PATTERN, DATANODE_TABLE_KEY_PREFIX,
+    RegionDistribution, TableMetaKey, TableMetaValue, DATANODE_TABLE_KEY_PATTERN,
+    DATANODE_TABLE_KEY_PREFIX,
 };
 use crate::kv_backend::txn::{Txn, TxnOp};
 use crate::kv_backend::KvBackendRef;
@@ -33,6 +34,7 @@ use crate::rpc::store::RangeRequest;
 use crate::rpc::KeyValue;
 use crate::DatanodeId;
 
+#[serde_with::serde_as]
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
 /// RegionInfo
 /// For compatible reason, DON'T modify the field name.
@@ -47,14 +49,15 @@ pub struct RegionInfo {
     #[serde(default)]
     pub region_options: HashMap<String, String>,
     /// The per-region wal options.
-    /// Key: region number (in string representation). Value: the encoded wal options of the region.
+    /// Key: region number. Value: the encoded wal options of the region.
     #[serde(default)]
-    pub region_wal_options: HashMap<String, String>,
+    #[serde_as(as = "HashMap<serde_with::DisplayFromStr, _>")]
+    pub region_wal_options: HashMap<RegionNumber, String>,
 }
 
 pub struct DatanodeTableKey {
-    datanode_id: DatanodeId,
-    table_id: TableId,
+    pub datanode_id: DatanodeId,
+    pub table_id: TableId,
 }
 
 impl DatanodeTableKey {
@@ -175,15 +178,6 @@ impl DatanodeTableManager {
         let txns = distribution
             .into_iter()
             .map(|(datanode_id, regions)| {
-                let filtered_region_wal_options = regions
-                    .iter()
-                    .filter_map(|region_number| {
-                        region_wal_options
-                            .get(region_number)
-                            .map(|wal_options| (region_number.to_string(), wal_options.clone()))
-                    })
-                    .collect();
-
                 let key = DatanodeTableKey::new(datanode_id, table_id);
                 let val = DatanodeTableValue::new(
                     table_id,
@@ -192,7 +186,9 @@ impl DatanodeTableManager {
                         engine: engine.to_string(),
                         region_storage_path: region_storage_path.to_string(),
                         region_options: region_options.clone(),
-                        region_wal_options: filtered_region_wal_options,
+                        // FIXME(weny): Before we store all region wal options into table metadata or somewhere,
+                        // We must store all region wal options.
+                        region_wal_options: region_wal_options.clone(),
                     },
                 );
 
@@ -213,7 +209,7 @@ impl DatanodeTableManager {
         current_region_distribution: RegionDistribution,
         new_region_distribution: RegionDistribution,
         new_region_options: &HashMap<String, String>,
-        new_region_wal_options: &HashMap<String, String>,
+        new_region_wal_options: &HashMap<RegionNumber, String>,
     ) -> Result<Txn> {
         let mut opts = Vec::new();
 
@@ -240,7 +236,15 @@ impl DatanodeTableManager {
             if need_update {
                 let key = DatanodeTableKey::new(datanode, table_id);
                 let raw_key = key.as_raw_key();
-                let val = DatanodeTableValue::new(table_id, regions, region_info.clone())
+                // FIXME(weny): add unit tests.
+                let mut new_region_info = region_info.clone();
+                if need_update_options {
+                    new_region_info.region_options = new_region_options.clone();
+                }
+                if need_update_wal_options {
+                    new_region_info.region_wal_options = new_region_wal_options.clone();
+                }
+                let val = DatanodeTableValue::new(table_id, regions, new_region_info)
                     .try_as_raw_value()?;
                 opts.push(TxnOp::Put(raw_key, val));
             }
@@ -305,6 +309,61 @@ mod tests {
         assert!(parsed.is_ok());
     }
 
+    #[derive(Debug, Serialize, Deserialize, PartialEq)]
+    struct StringHashMap {
+        inner: HashMap<String, String>,
+    }
+
+    #[serde_with::serde_as]
+    #[derive(Debug, Serialize, Deserialize, PartialEq)]
+    struct IntegerHashMap {
+        #[serde_as(as = "HashMap<serde_with::DisplayFromStr, _>")]
+        inner: HashMap<u32, String>,
+    }
+
+    #[test]
+    fn test_serde_with_integer_hash_map() {
+        let map = StringHashMap {
+            inner: HashMap::from([
+                ("1".to_string(), "aaa".to_string()),
+                ("2".to_string(), "bbb".to_string()),
+                ("3".to_string(), "ccc".to_string()),
+            ]),
+        };
+        let encoded = serde_json::to_string(&map).unwrap();
+        let decoded: IntegerHashMap = serde_json::from_str(&encoded).unwrap();
+        assert_eq!(
+            IntegerHashMap {
+                inner: HashMap::from([
+                    (1, "aaa".to_string()),
+                    (2, "bbb".to_string()),
+                    (3, "ccc".to_string()),
+                ]),
+            },
+            decoded
+        );
+
+        let map = IntegerHashMap {
+            inner: HashMap::from([
+                (1, "aaa".to_string()),
+                (2, "bbb".to_string()),
+                (3, "ccc".to_string()),
+            ]),
+        };
+        let encoded = serde_json::to_string(&map).unwrap();
+        let decoded: StringHashMap = serde_json::from_str(&encoded).unwrap();
+        assert_eq!(
+            StringHashMap {
+                inner: HashMap::from([
+                    ("1".to_string(), "aaa".to_string()),
+                    ("2".to_string(), "bbb".to_string()),
+                    ("3".to_string(), "ccc".to_string()),
+                ]),
+            },
+            decoded
+        );
+    }
+
     // This test intends to ensure both the `serde_json::to_string` + `serde_json::from_str`
     // and `serde_json::to_vec` + `serde_json::from_slice` work for `DatanodeTableValue`.
     // Warning: if the key of `region_wal_options` is of type non-String, this test would fail.
@@ -319,9 +378,9 @@ mod tests {
                 ("c".to_string(), "cc".to_string()),
             ]),
             region_wal_options: HashMap::from([
-                ("1".to_string(), "aaa".to_string()),
-                ("2".to_string(), "bbb".to_string()),
-                ("3".to_string(), "ccc".to_string()),
+                (1, "aaa".to_string()),
+                (2, "bbb".to_string()),
+                (3, "ccc".to_string()),
             ]),
         };
         let table_value = DatanodeTableValue {

src/common/meta/src/key/table_info.rs

@@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
 use table::engine::TableReference;
 use table::metadata::{RawTableInfo, TableId};
 
-use super::{DeserializedValueWithBytes, TABLE_INFO_KEY_PREFIX};
+use super::{DeserializedValueWithBytes, TableMetaValue, TABLE_INFO_KEY_PREFIX};
 use crate::error::Result;
 use crate::key::{to_removed_key, TableMetaKey};
 use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp, TxnOpResponse};

src/common/meta/src/key/table_name.rs

@@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use table::metadata::TableId;
 
-use super::{TABLE_NAME_KEY_PATTERN, TABLE_NAME_KEY_PREFIX};
+use super::{TableMetaValue, TABLE_NAME_KEY_PATTERN, TABLE_NAME_KEY_PREFIX};
 use crate::error::{Error, InvalidTableMetadataSnafu, Result};
 use crate::key::{to_removed_key, TableMetaKey};
 use crate::kv_backend::memory::MemoryKvBackend;

src/common/meta/src/key/table_region.rs

@@ -71,8 +71,8 @@ impl_table_meta_value! {TableRegionValue}
 
 #[cfg(test)]
 mod tests {
-
     use super::*;
+    use crate::key::TableMetaValue;
 
     #[test]
     fn test_serde() {

src/common/meta/src/key/table_route.rs

@@ -16,11 +16,14 @@ use std::collections::HashMap;
 use std::fmt::Display;
 
 use serde::{Deserialize, Serialize};
-use store_api::storage::RegionId;
+use snafu::{ensure, OptionExt, ResultExt};
+use store_api::storage::{RegionId, RegionNumber};
 use table::metadata::TableId;
 
-use super::DeserializedValueWithBytes;
-use crate::error::Result;
+use super::{DeserializedValueWithBytes, TableMetaValue};
+use crate::error::{
+    Result, SerdeJsonSnafu, TableRouteNotFoundSnafu, UnexpectedLogicalRouteTableSnafu,
+};
 use crate::key::{to_removed_key, RegionDistribution, TableMetaKey, TABLE_ROUTE_PREFIX};
 use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp, TxnOpResponse};
 use crate::kv_backend::KvBackendRef;
@@ -38,41 +41,157 @@ impl TableRouteKey {
 }
 
 #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
-pub struct TableRouteValue {
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum TableRouteValue {
+    Physical(PhysicalTableRouteValue),
+    Logical(LogicalTableRouteValue),
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
+pub struct PhysicalTableRouteValue {
     pub region_routes: Vec<RegionRoute>,
     version: u64,
 }
 
+#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
+pub struct LogicalTableRouteValue {
+    physical_table_id: TableId,
+    region_ids: Vec<RegionId>,
+}
+
 impl TableRouteValue {
+    pub fn physical(region_routes: Vec<RegionRoute>) -> Self {
+        Self::Physical(PhysicalTableRouteValue::new(region_routes))
+    }
+
+    /// Returns a new version [TableRouteValue] with `region_routes`.
+    pub fn update(&self, region_routes: Vec<RegionRoute>) -> Result<Self> {
+        ensure!(
+            self.is_physical(),
+            UnexpectedLogicalRouteTableSnafu {
+                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
+            }
+        );
+        let version = self.physical_table_route().version;
+        Ok(Self::Physical(PhysicalTableRouteValue {
+            region_routes,
+            version: version + 1,
+        }))
+    }
+
+    /// Returns the version.
+    ///
+    /// For test purpose.
+    #[cfg(any(test, feature = "testing"))]
+    pub fn version(&self) -> Result<u64> {
+        ensure!(
+            self.is_physical(),
+            UnexpectedLogicalRouteTableSnafu {
+                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
+            }
+        );
+        Ok(self.physical_table_route().version)
+    }
+
+    /// Returns the corresponding [RegionRoute], returns `None` if it's the specific region is not found.
+    ///
+    /// Note: It throws an error if it's a logical table
+    pub fn region_route(&self, region_id: RegionId) -> Result<Option<RegionRoute>> {
+        ensure!(
+            self.is_physical(),
+            UnexpectedLogicalRouteTableSnafu {
+                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
+            }
+        );
+        Ok(self
+            .physical_table_route()
+            .region_routes
+            .iter()
+            .find(|route| route.region.id == region_id)
+            .cloned())
+    }
+
+    /// Returns true if it's [TableRouteValue::Physical].
+    pub fn is_physical(&self) -> bool {
+        matches!(self, TableRouteValue::Physical(_))
+    }
+
+    /// Gets the [RegionRoute]s of this [TableRouteValue::Physical].
+    pub fn region_routes(&self) -> Result<&Vec<RegionRoute>> {
+        ensure!(
+            self.is_physical(),
+            UnexpectedLogicalRouteTableSnafu {
+                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
+            }
+        );
+        Ok(&self.physical_table_route().region_routes)
+    }
+
+    fn physical_table_route(&self) -> &PhysicalTableRouteValue {
+        match self {
+            TableRouteValue::Physical(x) => x,
+            _ => unreachable!("Mistakenly been treated as a Physical TableRoute: {self:?}"),
+        }
+    }
+
+    pub fn region_numbers(&self) -> Vec<RegionNumber> {
+        match self {
+            TableRouteValue::Physical(x) => x
+                .region_routes
+                .iter()
+                .map(|region_route| region_route.region.id.region_number())
+                .collect::<Vec<_>>(),
+            TableRouteValue::Logical(x) => x
+                .region_ids()
+                .iter()
+                .map(|region_id| region_id.region_number())
+                .collect::<Vec<_>>(),
+        }
+    }
+}
+
+impl TableMetaValue for TableRouteValue {
+    fn try_from_raw_value(raw_value: &[u8]) -> Result<Self> {
+        let r = serde_json::from_slice::<TableRouteValue>(raw_value);
+        match r {
+            // Compatible with old TableRouteValue.
+            Err(e) if e.is_data() => Ok(Self::Physical(
+                serde_json::from_slice::<PhysicalTableRouteValue>(raw_value)
+                    .context(SerdeJsonSnafu)?,
+            )),
+            Ok(x) => Ok(x),
+            Err(e) => Err(e).context(SerdeJsonSnafu),
+        }
+    }
+
+    fn try_as_raw_value(&self) -> Result<Vec<u8>> {
+        serde_json::to_vec(self).context(SerdeJsonSnafu)
+    }
+}
+
+impl PhysicalTableRouteValue {
     pub fn new(region_routes: Vec<RegionRoute>) -> Self {
         Self {
             region_routes,
             version: 0,
         }
     }
+}
 
-    /// Returns a new version [TableRouteValue] with `region_routes`.
-    pub fn update(&self, region_routes: Vec<RegionRoute>) -> Self {
+impl LogicalTableRouteValue {
+    pub fn new(physical_table_id: TableId, region_ids: Vec<RegionId>) -> Self {
         Self {
-            region_routes,
-            version: self.version + 1,
+            physical_table_id,
+            region_ids,
         }
     }
 
-    /// Returns the version.
-    ///
-    /// For test purpose.
-    #[cfg(any(tets, feature = "testing"))]
-    pub fn version(&self) -> u64 {
-        self.version
+    pub fn physical_table_id(&self) -> TableId {
+        self.physical_table_id
     }
 
-    /// Returns the corresponding [RegionRoute].
-    pub fn region_route(&self, region_id: RegionId) -> Option<RegionRoute> {
-        self.region_routes
-            .iter()
-            .find(|route| route.region.id == region_id)
-            .cloned()
+    pub fn region_ids(&self) -> &Vec<RegionId> {
+        &self.region_ids
     }
 }
 
@@ -217,6 +336,54 @@ impl TableRouteManager {
             .transpose()
     }
 
+    pub async fn get_physical_table_id(
+        &self,
+        logical_or_physical_table_id: TableId,
+    ) -> Result<TableId> {
+        let table_route = self
+            .get(logical_or_physical_table_id)
+            .await?
+            .context(TableRouteNotFoundSnafu {
+                table_id: logical_or_physical_table_id,
+            })?
+            .into_inner();
+
+        match table_route {
+            TableRouteValue::Physical(_) => Ok(logical_or_physical_table_id),
+            TableRouteValue::Logical(x) => Ok(x.physical_table_id()),
+        }
+    }
+
+    pub async fn get_physical_table_route(
+        &self,
+        logical_or_physical_table_id: TableId,
+    ) -> Result<(TableId, PhysicalTableRouteValue)> {
+        let table_route = self
+            .get(logical_or_physical_table_id)
+            .await?
+            .context(TableRouteNotFoundSnafu {
+                table_id: logical_or_physical_table_id,
+            })?
+            .into_inner();
+
+        match table_route {
+            TableRouteValue::Physical(x) => Ok((logical_or_physical_table_id, x)),
+            TableRouteValue::Logical(x) => {
+                let physical_table_id = x.physical_table_id();
+                let physical_table_route =
+                    self.get(physical_table_id)
+                        .await?
+                        .context(TableRouteNotFoundSnafu {
+                            table_id: physical_table_id,
+                        })?;
+                Ok((
+                    physical_table_id,
+                    physical_table_route.physical_table_route().clone(),
+                ))
+            }
+        }
+    }
+
     /// It may return a subset of the `table_ids`.
     pub async fn batch_get(
         &self,
@@ -269,7 +436,24 @@ impl TableRouteManager {
     ) -> Result<Option<RegionDistribution>> {
         self.get(table_id)
             .await?
-            .map(|table_route| region_distribution(&table_route.into_inner().region_routes))
+            .map(|table_route| Ok(region_distribution(table_route.region_routes()?)))
             .transpose()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_table_route_compatibility() {
+        let old_raw_v = r#"{"region_routes":[{"region":{"id":1,"name":"r1","partition":null,"attrs":{}},"leader_peer":{"id":2,"addr":"a2"},"follower_peers":[]},{"region":{"id":1,"name":"r1","partition":null,"attrs":{}},"leader_peer":{"id":2,"addr":"a2"},"follower_peers":[]}],"version":0}"#;
+        let v = TableRouteValue::try_from_raw_value(old_raw_v.as_bytes()).unwrap();
+
+        let new_raw_v = format!("{:?}", v);
+        assert_eq!(
+            new_raw_v,
+            r#"Physical(PhysicalTableRouteValue { region_routes: [RegionRoute { region: Region { id: 1(0, 1), name: "r1", partition: None, attrs: {} }, leader_peer: Some(Peer { id: 2, addr: "a2" }), follower_peers: [], leader_status: None }, RegionRoute { region: Region { id: 1(0, 1), name: "r1", partition: None, attrs: {} }, leader_peer: Some(Peer { id: 2, addr: "a2" }), follower_peers: [], leader_status: None }], version: 0 })"#
+        );
+    }
+}

src/common/meta/src/kv_backend.rs

@@ -114,6 +114,7 @@ where
         Ok(!resp.kvs.is_empty())
     }
 
+    /// Returns previous key-value pair if `prev_kv` is `true`.
     async fn delete(&self, key: &[u8], prev_kv: bool) -> Result<Option<KeyValue>, Self::Error> {
         let mut req = DeleteRangeRequest::new().with_key(key.to_vec());
         if prev_kv {

src/common/meta/src/lib.rs

@@ -15,6 +15,7 @@
 #![feature(assert_matches)]
 #![feature(btree_extract_if)]
 #![feature(async_closure)]
+#![feature(let_chains)]
 
 pub mod cache_invalidator;
 pub mod datanode_manager;
@@ -26,6 +27,7 @@ pub mod heartbeat;
 pub mod instruction;
 pub mod key;
 pub mod kv_backend;
+pub mod lock_key;
 pub mod metrics;
 pub mod peer;
 pub mod range_stream;
@@ -35,7 +37,6 @@ pub mod sequence;
 pub mod state_store;
 pub mod table_name;
 pub mod util;
-#[allow(unused)]
 pub mod wal;
 
 pub type ClusterId = u64;

src/common/meta/src/lock_key.rs

@@ -0,0 +1,235 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Display;
+
+use common_catalog::{format_full_table_name, format_schema_name};
+use common_procedure::StringKey;
+use store_api::storage::{RegionId, TableId};
+
+const CATALOG_LOCK_PREFIX: &str = "__catalog_lock";
+const SCHEMA_LOCK_PREFIX: &str = "__schema_lock";
+const TABLE_LOCK_PREFIX: &str = "__table_lock";
+const TABLE_NAME_LOCK_PREFIX: &str = "__table_name_lock";
+const REGION_LOCK_PREFIX: &str = "__region_lock";
+
+/// [CatalogLock] acquires the lock on the tenant level.
+pub enum CatalogLock<'a> {
+    Read(&'a str),
+    Write(&'a str),
+}
+
+impl<'a> Display for CatalogLock<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let key = match self {
+            CatalogLock::Read(s) => s,
+            CatalogLock::Write(s) => s,
+        };
+        write!(f, "{}/{}", CATALOG_LOCK_PREFIX, key)
+    }
+}
+
+impl<'a> From<CatalogLock<'a>> for StringKey {
+    fn from(value: CatalogLock) -> Self {
+        match value {
+            CatalogLock::Write(_) => StringKey::Exclusive(value.to_string()),
+            CatalogLock::Read(_) => StringKey::Share(value.to_string()),
+        }
+    }
+}
+
+/// [SchemaLock] acquires the lock on the database level.
+pub enum SchemaLock {
+    Read(String),
+    Write(String),
+}
+
+impl SchemaLock {
+    pub fn read(catalog: &str, schema: &str) -> Self {
+        Self::Read(format_schema_name(catalog, schema))
+    }
+
+    pub fn write(catalog: &str, schema: &str) -> Self {
+        Self::Write(format_schema_name(catalog, schema))
+    }
+}
+
+impl Display for SchemaLock {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let key = match self {
+            SchemaLock::Read(s) => s,
+            SchemaLock::Write(s) => s,
+        };
+        write!(f, "{}/{}", SCHEMA_LOCK_PREFIX, key)
+    }
+}
+
+impl From<SchemaLock> for StringKey {
+    fn from(value: SchemaLock) -> Self {
+        match value {
+            SchemaLock::Write(_) => StringKey::Exclusive(value.to_string()),
+            SchemaLock::Read(_) => StringKey::Share(value.to_string()),
+        }
+    }
+}
+
+/// [TableNameLock] prevents any procedures trying to create a table named it.
+pub enum TableNameLock {
+    Write(String),
+}
+
+impl TableNameLock {
+    pub fn new(catalog: &str, schema: &str, table: &str) -> Self {
+        Self::Write(format_full_table_name(catalog, schema, table))
+    }
+}
+
+impl Display for TableNameLock {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let TableNameLock::Write(name) = self;
+        write!(f, "{}/{}", TABLE_NAME_LOCK_PREFIX, name)
+    }
+}
+
+impl From<TableNameLock> for StringKey {
+    fn from(value: TableNameLock) -> Self {
+        match value {
+            TableNameLock::Write(_) => StringKey::Exclusive(value.to_string()),
+        }
+    }
+}
+
+/// [TableLock] acquires the lock on the table level.
+///
+/// Note: Allows to read/modify the corresponding table's [TableInfoValue](crate::key::table_info::TableInfoValue),
+/// [TableRouteValue](crate::key::table_route::TableRouteValue), [TableDatanodeValue](crate::key::datanode_table::DatanodeTableValue).
+pub enum TableLock {
+    Read(TableId),
+    Write(TableId),
+}
+
+impl Display for TableLock {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let key = match self {
+            TableLock::Read(s) => s,
+            TableLock::Write(s) => s,
+        };
+        write!(f, "{}/{}", TABLE_LOCK_PREFIX, key)
+    }
+}
+
+impl From<TableLock> for StringKey {
+    fn from(value: TableLock) -> Self {
+        match value {
+            TableLock::Write(_) => StringKey::Exclusive(value.to_string()),
+            TableLock::Read(_) => StringKey::Share(value.to_string()),
+        }
+    }
+}
+
+/// [RegionLock] acquires the lock on the region level.
+///
+/// Note:
+/// - Allows modification the corresponding region's [TableRouteValue](crate::key::table_route::TableRouteValue),
+/// [TableDatanodeValue](crate::key::datanode_table::DatanodeTableValue) even if
+/// it acquires the [RegionLock::Write] only without acquiring the [TableLock::Write].
+///
+/// - Should acquire [TableLock] of the table at same procedure.
+///
+/// TODO(weny): we should consider separating TableRouteValue into finer keys.
+pub enum RegionLock {
+    Read(RegionId),
+    Write(RegionId),
+}
+
+impl Display for RegionLock {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let key = match self {
+            RegionLock::Read(s) => s.as_u64(),
+            RegionLock::Write(s) => s.as_u64(),
+        };
+        write!(f, "{}/{}", REGION_LOCK_PREFIX, key)
+    }
+}
+
+impl From<RegionLock> for StringKey {
+    fn from(value: RegionLock) -> Self {
+        match value {
+            RegionLock::Write(_) => StringKey::Exclusive(value.to_string()),
+            RegionLock::Read(_) => StringKey::Share(value.to_string()),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use common_procedure::StringKey;
+
+    use crate::lock_key::*;
+
+    #[test]
+    fn test_lock_key() {
+        // The catalog lock
+        let string_key: StringKey = CatalogLock::Read("foo").into();
+        assert_eq!(
+            string_key,
+            StringKey::Share(format!("{}/{}", CATALOG_LOCK_PREFIX, "foo"))
+        );
+        let string_key: StringKey = CatalogLock::Write("foo").into();
+        assert_eq!(
+            string_key,
+            StringKey::Exclusive(format!("{}/{}", CATALOG_LOCK_PREFIX, "foo"))
+        );
+        // The schema lock
+        let string_key: StringKey = SchemaLock::read("foo", "bar").into();
+        assert_eq!(
+            string_key,
+            StringKey::Share(format!("{}/{}", SCHEMA_LOCK_PREFIX, "foo.bar"))
+        );
+        let string_key: StringKey = SchemaLock::write("foo", "bar").into();
+        assert_eq!(
+            string_key,
+            StringKey::Exclusive(format!("{}/{}", SCHEMA_LOCK_PREFIX, "foo.bar"))
+        );
+        // The table lock
+        let string_key: StringKey = TableLock::Read(1024).into();
+        assert_eq!(
+            string_key,
+            StringKey::Share(format!("{}/{}", TABLE_LOCK_PREFIX, 1024))
+        );
+        let string_key: StringKey = TableLock::Write(1024).into();
+        assert_eq!(
+            string_key,
+            StringKey::Exclusive(format!("{}/{}", TABLE_LOCK_PREFIX, 1024))
+        );
+        // The table name lock
+        let string_key: StringKey = TableNameLock::new("foo", "bar", "baz").into();
+        assert_eq!(
+            string_key,
+            StringKey::Exclusive(format!("{}/{}", TABLE_NAME_LOCK_PREFIX, "foo.bar.baz"))
+        );
+        // The region lock
+        let region_id = RegionId::new(1024, 1);
+        let string_key: StringKey = RegionLock::Read(region_id).into();
+        assert_eq!(
+            string_key,
+            StringKey::Share(format!("{}/{}", REGION_LOCK_PREFIX, region_id.as_u64()))
+        );
+        let string_key: StringKey = RegionLock::Write(region_id).into();
+        assert_eq!(
+            string_key,
+            StringKey::Exclusive(format!("{}/{}", REGION_LOCK_PREFIX, region_id.as_u64()))
+        );
+    }
+}

src/common/meta/src/metrics.rs

@@ -16,36 +16,43 @@ use lazy_static::lazy_static;
 use prometheus::*;
 
 lazy_static! {
-    pub static ref METRIC_META_TXN_REQUEST: HistogramVec =
-        register_histogram_vec!("meta_txn_request", "meta txn request", &["target", "op"]).unwrap();
+    pub static ref METRIC_META_TXN_REQUEST: HistogramVec = register_histogram_vec!(
+        "greptime_meta_txn_request",
+        "meta txn request",
+        &["target", "op"]
+    )
+    .unwrap();
     pub static ref METRIC_META_CREATE_CATALOG: Histogram =
-        register_histogram!("meta_create_catalog", "meta create catalog").unwrap();
-    pub static ref METRIC_META_CREATE_CATALOG_COUNTER: IntCounter =
-        register_int_counter!("meta_create_catalog_counter", "meta create catalog").unwrap();
+        register_histogram!("greptime_meta_create_catalog", "meta create catalog").unwrap();
+    pub static ref METRIC_META_CREATE_CATALOG_COUNTER: IntCounter = register_int_counter!(
+        "greptime_meta_create_catalog_counter",
+        "meta create catalog"
+    )
+    .unwrap();
     pub static ref METRIC_META_CREATE_SCHEMA: Histogram =
-        register_histogram!("meta_create_schema", "meta create schema").unwrap();
+        register_histogram!("greptime_meta_create_schema", "meta create schema").unwrap();
     pub static ref METRIC_META_CREATE_SCHEMA_COUNTER: IntCounter =
-        register_int_counter!("meta_create_schema_counter", "meta create schema").unwrap();
+        register_int_counter!("greptime_meta_create_schema_counter", "meta create schema").unwrap();
     pub static ref METRIC_META_PROCEDURE_CREATE_TABLE: HistogramVec = register_histogram_vec!(
-        "meta_procedure_create_table",
+        "greptime_meta_procedure_create_table",
         "meta procedure create table",
         &["step"]
     )
     .unwrap();
     pub static ref METRIC_META_PROCEDURE_DROP_TABLE: HistogramVec = register_histogram_vec!(
-        "meta_procedure_drop_table",
+        "greptime_meta_procedure_drop_table",
         "meta procedure drop table",
         &["step"]
     )
     .unwrap();
     pub static ref METRIC_META_PROCEDURE_ALTER_TABLE: HistogramVec = register_histogram_vec!(
-        "meta_procedure_alter_table",
+        "greptime_meta_procedure_alter_table",
         "meta procedure alter table",
         &["step"]
     )
     .unwrap();
     pub static ref METRIC_META_PROCEDURE_TRUNCATE_TABLE: HistogramVec = register_histogram_vec!(
-        "meta_procedure_truncate_table",
+        "greptime_meta_procedure_truncate_table",
         "meta procedure truncate table",
         &["step"]
     )

src/common/meta/src/rpc/router.rs

@@ -30,7 +30,7 @@ use crate::peer::Peer;
 use crate::table_name::TableName;
 use crate::DatanodeId;
 
-pub fn region_distribution(region_routes: &[RegionRoute]) -> Result<RegionDistribution> {
+pub fn region_distribution(region_routes: &[RegionRoute]) -> RegionDistribution {
     let mut regions_id_map = RegionDistribution::new();
     for route in region_routes.iter() {
         if let Some(peer) = route.leader_peer.as_ref() {
@@ -42,7 +42,7 @@ pub fn region_distribution(region_routes: &[RegionRoute]) -> Result<RegionDistri
         // id asc
         regions.sort()
     }
-    Ok(regions_id_map)
+    regions_id_map
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
@@ -123,11 +123,12 @@ pub fn convert_to_region_leader_status_map(
 pub fn find_region_leader(
     region_routes: &[RegionRoute],
     region_number: RegionNumber,
-) -> Option<&Peer> {
+) -> Option<Peer> {
     region_routes
         .iter()
         .find(|x| x.region.id.region_number() == region_number)
         .and_then(|r| r.leader_peer.as_ref())
+        .cloned()
 }
 
 pub fn find_leader_regions(region_routes: &[RegionRoute], datanode: &Peer) -> Vec<RegionNumber> {

src/common/meta/src/wal.rs

@@ -18,12 +18,11 @@ pub mod options_allocator;
 use std::collections::HashMap;
 
 use common_config::wal::StandaloneWalConfig;
+use common_config::WAL_OPTIONS_KEY;
 use serde::{Deserialize, Serialize};
-use serde_with::with_prefix;
+use store_api::storage::{RegionId, RegionNumber};
 
-use crate::error::Result;
 use crate::wal::kafka::KafkaConfig;
-pub use crate::wal::kafka::Topic as KafkaWalTopic;
 pub use crate::wal::options_allocator::{
     allocate_region_wal_options, WalOptionsAllocator, WalOptionsAllocatorRef,
 };
@@ -40,7 +39,7 @@ pub enum WalConfig {
 impl From<StandaloneWalConfig> for WalConfig {
     fn from(value: StandaloneWalConfig) -> Self {
         match value {
-            StandaloneWalConfig::RaftEngine(config) => WalConfig::RaftEngine,
+            StandaloneWalConfig::RaftEngine(_) => WalConfig::RaftEngine,
             StandaloneWalConfig::Kafka(config) => WalConfig::Kafka(KafkaConfig {
                 broker_endpoints: config.base.broker_endpoints,
                 num_topics: config.num_topics,
@@ -55,6 +54,16 @@ impl From<StandaloneWalConfig> for WalConfig {
     }
 }
 
+pub fn prepare_wal_option(
+    options: &mut HashMap<String, String>,
+    region_id: RegionId,
+    region_wal_options: &HashMap<RegionNumber, String>,
+) {
+    if let Some(wal_options) = region_wal_options.get(&region_id.region_number()) {
+        options.insert(WAL_OPTIONS_KEY.to_string(), wal_options.clone());
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::time::Duration;
@@ -88,7 +97,6 @@ mod tests {
             num_topics = 32
             selector_type = "round_robin"
             topic_name_prefix = "greptimedb_wal_topic"
-            num_partitions = 1
             replication_factor = 1
             create_topic_timeout = "30s"
             backoff_init = "500ms"

src/common/meta/src/wal/kafka.rs

@@ -12,21 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod topic;
+#[cfg(any(test, feature = "testing"))]
+pub mod test_util;
 pub mod topic_manager;
 pub mod topic_selector;
 
 use std::time::Duration;
 
 use common_config::wal::kafka::{kafka_backoff, KafkaBackoffConfig, TopicSelectorType};
-use common_config::wal::StandaloneWalConfig;
 use serde::{Deserialize, Serialize};
 
-pub use crate::wal::kafka::topic::Topic;
 pub use crate::wal::kafka::topic_manager::TopicManager;
 
 /// Configurations for kafka wal.
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(default)]
 pub struct KafkaConfig {
     /// The broker endpoints of the Kafka cluster.
     pub broker_endpoints: Vec<String>,
@@ -40,7 +40,7 @@ pub struct KafkaConfig {
     pub num_partitions: i32,
     /// The replication factor of each topic.
     pub replication_factor: i16,
-    /// Above which a topic creation operation will be cancelled.
+    /// The timeout of topic creation.
     #[serde(with = "humantime_serde")]
     pub create_topic_timeout: Duration,
     /// The backoff config.

src/common/meta/src/wal/kafka/test_util.rs

@@ -0,0 +1,33 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_telemetry::warn;
+use futures_util::future::BoxFuture;
+
+pub async fn run_test_with_kafka_wal<F>(test: F)
+where
+    F: FnOnce(Vec<String>) -> BoxFuture<'static, ()>,
+{
+    let Ok(endpoints) = std::env::var("GT_KAFKA_ENDPOINTS") else {
+        warn!("The endpoints is empty, skipping the test");
+        return;
+    };
+
+    let endpoints = endpoints
+        .split(',')
+        .map(|s| s.trim().to_string())
+        .collect::<Vec<_>>();
+
+    test(endpoints).await
+}

src/common/meta/src/wal/kafka/topic_manager.rs

@@ -14,35 +14,39 @@
 
 use std::collections::HashSet;
 use std::sync::Arc;
-use std::time::Duration;
 
 use common_config::wal::kafka::TopicSelectorType;
-use common_telemetry::{debug, error, info};
+use common_telemetry::{error, info};
 use rskafka::client::controller::ControllerClient;
 use rskafka::client::error::Error as RsKafkaError;
 use rskafka::client::error::ProtocolError::TopicAlreadyExists;
-use rskafka::client::ClientBuilder;
+use rskafka::client::partition::{Compression, UnknownTopicHandling};
+use rskafka::client::{Client, ClientBuilder};
+use rskafka::record::Record;
 use rskafka::BackoffConfig;
-use snafu::{ensure, AsErrorSource, ResultExt};
+use snafu::{ensure, ResultExt};
 
 use crate::error::{
-    BuildKafkaClientSnafu, BuildKafkaCtrlClientSnafu, CreateKafkaWalTopicSnafu, DecodeJsonSnafu,
-    EncodeJsonSnafu, InvalidNumTopicsSnafu, Result,
+    BuildKafkaClientSnafu, BuildKafkaCtrlClientSnafu, BuildKafkaPartitionClientSnafu,
+    CreateKafkaWalTopicSnafu, DecodeJsonSnafu, EncodeJsonSnafu, InvalidNumTopicsSnafu,
+    ProduceRecordSnafu, Result,
 };
 use crate::kv_backend::KvBackendRef;
 use crate::rpc::store::PutRequest;
-use crate::wal::kafka::topic::Topic;
 use crate::wal::kafka::topic_selector::{RoundRobinTopicSelector, TopicSelectorRef};
 use crate::wal::kafka::KafkaConfig;
 
 const CREATED_TOPICS_KEY: &str = "__created_wal_topics/kafka/";
 
+// Each topic only has one partition for now.
+// The `DEFAULT_PARTITION` refers to the index of the partition.
+const DEFAULT_PARTITION: i32 = 0;
+
 /// Manages topic initialization and selection.
 pub struct TopicManager {
     config: KafkaConfig,
-    // TODO(niebayes): maybe add a guard to ensure all topics in the topic pool are created.
-    topic_pool: Vec<Topic>,
-    topic_selector: TopicSelectorRef,
+    pub(crate) topic_pool: Vec<String>,
+    pub(crate) topic_selector: TopicSelectorRef,
     kv_backend: KvBackendRef,
 }
 
@@ -81,7 +85,7 @@ impl TopicManager {
         let created_topics = Self::restore_created_topics(&self.kv_backend)
             .await?
             .into_iter()
-            .collect::<HashSet<Topic>>();
+            .collect::<HashSet<String>>();
 
         // Creates missing topics.
         let to_be_created = topics
@@ -103,7 +107,7 @@ impl TopicManager {
     }
 
     /// Tries to create topics specified by indexes in `to_be_created`.
-    async fn try_create_topics(&self, topics: &[Topic], to_be_created: &[usize]) -> Result<()> {
+    async fn try_create_topics(&self, topics: &[String], to_be_created: &[usize]) -> Result<()> {
         // Builds an kafka controller client for creating topics.
         let backoff_config = BackoffConfig {
             init_backoff: self.config.backoff.init,
@@ -117,31 +121,62 @@ impl TopicManager {
             .await
             .with_context(|_| BuildKafkaClientSnafu {
                 broker_endpoints: self.config.broker_endpoints.clone(),
-            })?
+            })?;
+
+        let control_client = client
             .controller_client()
             .context(BuildKafkaCtrlClientSnafu)?;
 
         // Try to create missing topics.
         let tasks = to_be_created
             .iter()
-            .map(|i| self.try_create_topic(&topics[*i], &client))
+            .map(|i| async {
+                self.try_create_topic(&topics[*i], &control_client).await?;
+                self.try_append_noop_record(&topics[*i], &client).await?;
+                Ok(())
+            })
             .collect::<Vec<_>>();
         futures::future::try_join_all(tasks).await.map(|_| ())
     }
 
     /// Selects one topic from the topic pool through the topic selector.
-    pub fn select(&self) -> Result<&Topic> {
+    pub fn select(&self) -> Result<&String> {
         self.topic_selector.select(&self.topic_pool)
     }
 
     /// Selects a batch of topics from the topic pool through the topic selector.
-    pub fn select_batch(&self, num_topics: usize) -> Result<Vec<&Topic>> {
+    pub fn select_batch(&self, num_topics: usize) -> Result<Vec<&String>> {
         (0..num_topics)
             .map(|_| self.topic_selector.select(&self.topic_pool))
             .collect()
     }
 
-    async fn try_create_topic(&self, topic: &Topic, client: &ControllerClient) -> Result<()> {
+    async fn try_append_noop_record(&self, topic: &String, client: &Client) -> Result<()> {
+        let partition_client = client
+            .partition_client(topic, DEFAULT_PARTITION, UnknownTopicHandling::Retry)
+            .await
+            .context(BuildKafkaPartitionClientSnafu {
+                topic,
+                partition: DEFAULT_PARTITION,
+            })?;
+
+        partition_client
+            .produce(
+                vec![Record {
+                    key: None,
+                    value: None,
+                    timestamp: chrono::Utc::now(),
+                    headers: Default::default(),
+                }],
+                Compression::NoCompression,
+            )
+            .await
+            .context(ProduceRecordSnafu { topic })?;
+
+        Ok(())
+    }
+
+    async fn try_create_topic(&self, topic: &String, client: &ControllerClient) -> Result<()> {
         match client
             .create_topic(
                 topic.clone(),
@@ -167,7 +202,7 @@ impl TopicManager {
         }
     }
 
-    async fn restore_created_topics(kv_backend: &KvBackendRef) -> Result<Vec<Topic>> {
+    async fn restore_created_topics(kv_backend: &KvBackendRef) -> Result<Vec<String>> {
         kv_backend
             .get(CREATED_TOPICS_KEY.as_bytes())
             .await?
@@ -177,7 +212,7 @@ impl TopicManager {
             )
     }
 
-    async fn persist_created_topics(topics: &[Topic], kv_backend: &KvBackendRef) -> Result<()> {
+    async fn persist_created_topics(topics: &[String], kv_backend: &KvBackendRef) -> Result<()> {
         let raw_topics = serde_json::to_vec(topics).context(EncodeJsonSnafu)?;
         kv_backend
             .put(PutRequest {
@@ -202,13 +237,9 @@ impl TopicManager {
 
 #[cfg(test)]
 mod tests {
-    use std::env;
-
-    use common_telemetry::info;
-
     use super::*;
     use crate::kv_backend::memory::MemoryKvBackend;
-    use crate::kv_backend::{self};
+    use crate::wal::kafka::test_util::run_test_with_kafka_wal;
 
     // Tests that topics can be successfully persisted into the kv backend and can be successfully restored from the kv backend.
     #[tokio::test]
@@ -235,26 +266,60 @@ mod tests {
         assert_eq!(topics, restored_topics);
     }
 
+    /// Tests that the topic manager could allocate topics correctly.
     #[tokio::test]
-    async fn test_topic_manager() {
-        let endpoints = env::var("GT_KAFKA_ENDPOINTS").unwrap_or_default();
-        common_telemetry::init_default_ut_logging();
+    async fn test_alloc_topics() {
+        run_test_with_kafka_wal(|broker_endpoints| {
+            Box::pin(async {
+                // Constructs topics that should be created.
+                let topics = (0..256)
+                    .map(|i| format!("test_alloc_topics_{}_{}", i, uuid::Uuid::new_v4()))
+                    .collect::<Vec<_>>();
 
-        if endpoints.is_empty() {
-            info!("The endpoints is empty, skipping the test.");
-            return;
-        }
-        // TODO: supports topic prefix
-        let kv_backend = Arc::new(MemoryKvBackend::new());
-        let config = KafkaConfig {
-            replication_factor: 1,
-            broker_endpoints: endpoints
-                .split(',')
-                .map(|s| s.to_string())
-                .collect::<Vec<_>>(),
-            ..Default::default()
-        };
-        let manager = TopicManager::new(config, kv_backend);
-        manager.start().await.unwrap();
+                // Creates a topic manager.
+                let config = KafkaConfig {
+                    replication_factor: broker_endpoints.len() as i16,
+                    broker_endpoints,
+                    ..Default::default()
+                };
+                let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+                let mut manager = TopicManager::new(config.clone(), kv_backend);
+                // Replaces the default topic pool with the constructed topics.
+                manager.topic_pool = topics.clone();
+                // Replaces the default selector with a round-robin selector without shuffled.
+                manager.topic_selector = Arc::new(RoundRobinTopicSelector::default());
+                manager.start().await.unwrap();
+
+                // Selects exactly the number of `num_topics` topics one by one.
+                let got = (0..topics.len())
+                    .map(|_| manager.select().unwrap())
+                    .cloned()
+                    .collect::<Vec<_>>();
+                assert_eq!(got, topics);
+
+                // Selects exactly the number of `num_topics` topics in a batching manner.
+                let got = manager
+                    .select_batch(topics.len())
+                    .unwrap()
+                    .into_iter()
+                    .map(ToString::to_string)
+                    .collect::<Vec<_>>();
+                assert_eq!(got, topics);
+
+                // Selects more than the number of `num_topics` topics.
+                let got = manager
+                    .select_batch(2 * topics.len())
+                    .unwrap()
+                    .into_iter()
+                    .map(ToString::to_string)
+                    .collect::<Vec<_>>();
+                let expected = vec![topics.clone(); 2]
+                    .into_iter()
+                    .flatten()
+                    .collect::<Vec<_>>();
+                assert_eq!(got, expected);
+            })
+        })
+        .await;
     }
 }

src/common/meta/src/wal/kafka/topic_selector.rs

@@ -16,16 +16,14 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
 use rand::Rng;
-use serde::{Deserialize, Serialize};
 use snafu::ensure;
 
 use crate::error::{EmptyTopicPoolSnafu, Result};
-use crate::wal::kafka::topic::Topic;
 
 /// Controls topic selection.
 pub(crate) trait TopicSelector: Send + Sync {
     /// Selects a topic from the topic pool.
-    fn select<'a>(&self, topic_pool: &'a [Topic]) -> Result<&'a Topic>;
+    fn select<'a>(&self, topic_pool: &'a [String]) -> Result<&'a String>;
 }
 
 /// Arc wrapper of TopicSelector.
@@ -49,7 +47,7 @@ impl RoundRobinTopicSelector {
 }
 
 impl TopicSelector for RoundRobinTopicSelector {
-    fn select<'a>(&self, topic_pool: &'a [Topic]) -> Result<&'a Topic> {
+    fn select<'a>(&self, topic_pool: &'a [String]) -> Result<&'a String> {
         ensure!(!topic_pool.is_empty(), EmptyTopicPoolSnafu);
         let which = self.cursor.fetch_add(1, Ordering::Relaxed) % topic_pool.len();
         Ok(&topic_pool[which])
@@ -60,6 +58,14 @@ impl TopicSelector for RoundRobinTopicSelector {
 mod tests {
     use super::*;
 
+    /// Tests that a selector behaves as expected when the given topic pool is empty.
+    #[test]
+    fn test_empty_topic_pool() {
+        let topic_pool = vec![];
+        let selector = RoundRobinTopicSelector::default();
+        assert!(selector.select(&topic_pool).is_err());
+    }
+
     #[test]
     fn test_round_robin_topic_selector() {
         let topic_pool: Vec<_> = [0, 1, 2].into_iter().map(|v| v.to_string()).collect();

src/common/meta/src/wal/options_allocator.rs

@@ -107,14 +107,16 @@ pub fn allocate_region_wal_options(
 mod tests {
     use super::*;
     use crate::kv_backend::memory::MemoryKvBackend;
+    use crate::wal::kafka::test_util::run_test_with_kafka_wal;
+    use crate::wal::kafka::topic_selector::RoundRobinTopicSelector;
+    use crate::wal::kafka::KafkaConfig;
 
     // Tests the wal options allocator could successfully allocate raft-engine wal options.
-    // Note: tests for allocator with kafka are integration tests.
     #[tokio::test]
     async fn test_allocator_with_raft_engine() {
         let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
         let wal_config = WalConfig::RaftEngine;
-        let mut allocator = WalOptionsAllocator::new(wal_config, kv_backend);
+        let allocator = WalOptionsAllocator::new(wal_config, kv_backend);
         allocator.start().await.unwrap();
 
         let num_regions = 32;
@@ -128,4 +130,49 @@ mod tests {
             .collect();
         assert_eq!(got, expected);
     }
+
+    // Tests that the wal options allocator could successfully allocate Kafka wal options.
+    #[tokio::test]
+    async fn test_allocator_with_kafka() {
+        run_test_with_kafka_wal(|broker_endpoints| {
+            Box::pin(async {
+                let topics = (0..256)
+                    .map(|i| format!("test_allocator_with_kafka_{}_{}", i, uuid::Uuid::new_v4()))
+                    .collect::<Vec<_>>();
+
+                // Creates a topic manager.
+                let config = KafkaConfig {
+                    replication_factor: broker_endpoints.len() as i16,
+                    broker_endpoints,
+                    ..Default::default()
+                };
+                let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+                let mut topic_manager = KafkaTopicManager::new(config.clone(), kv_backend);
+                // Replaces the default topic pool with the constructed topics.
+                topic_manager.topic_pool = topics.clone();
+                // Replaces the default selector with a round-robin selector without shuffled.
+                topic_manager.topic_selector = Arc::new(RoundRobinTopicSelector::default());
+
+                // Creates an options allocator.
+                let allocator = WalOptionsAllocator::Kafka(topic_manager);
+                allocator.start().await.unwrap();
+
+                let num_regions = 32;
+                let regions = (0..num_regions).collect::<Vec<_>>();
+                let got = allocate_region_wal_options(regions.clone(), &allocator).unwrap();
+
+                // Check the allocated wal options contain the expected topics.
+                let expected = (0..num_regions)
+                    .map(|i| {
+                        let options = WalOptions::Kafka(KafkaWalOptions {
+                            topic: topics[i as usize].clone(),
+                        });
+                        (i, serde_json::to_string(&options).unwrap())
+                    })
+                    .collect::<HashMap<_, _>>();
+                assert_eq!(got, expected);
+            })
+        })
+        .await;
+    }
 }

src/common/procedure/src/lib.rs

@@ -26,6 +26,6 @@ pub mod watcher;
 pub use crate::error::{Error, Result};
 pub use crate::procedure::{
     BoxedProcedure, Context, ContextProvider, LockKey, Procedure, ProcedureId, ProcedureManager,
-    ProcedureManagerRef, ProcedureState, ProcedureWithId, Status,
+    ProcedureManagerRef, ProcedureState, ProcedureWithId, Status, StringKey,
 };
 pub use crate::watcher::Watcher;

src/common/procedure/src/local.rs

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod lock;
 mod runner;
+mod rwlock;
 
 use std::collections::{HashMap, VecDeque};
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -29,11 +29,11 @@ use snafu::{ensure, ResultExt};
 use tokio::sync::watch::{self, Receiver, Sender};
 use tokio::sync::{Mutex as TokioMutex, Notify};
 
+use self::rwlock::KeyRwLock;
 use crate::error::{
     DuplicateProcedureSnafu, Error, LoaderConflictSnafu, ManagerNotStartSnafu, Result,
     StartRemoveOutdatedMetaTaskSnafu, StopRemoveOutdatedMetaTaskSnafu,
 };
-use crate::local::lock::LockMap;
 use crate::local::runner::Runner;
 use crate::procedure::BoxedProcedureLoader;
 use crate::store::{ProcedureMessage, ProcedureStore, StateStoreRef};
@@ -57,8 +57,6 @@ const META_TTL: Duration = Duration::from_secs(60 * 10);
 pub(crate) struct ProcedureMeta {
     /// Id of this procedure.
     id: ProcedureId,
-    /// Notify to wait for a lock.
-    lock_notify: Notify,
     /// Parent procedure id.
     parent_id: Option<ProcedureId>,
     /// Notify to wait for subprocedures.
@@ -78,7 +76,6 @@ impl ProcedureMeta {
         let (state_sender, state_receiver) = watch::channel(ProcedureState::Running);
         ProcedureMeta {
             id,
-            lock_notify: Notify::new(),
             parent_id,
             child_notify: Notify::new(),
             lock_key,
@@ -131,7 +128,7 @@ struct LoadedProcedure {
 pub(crate) struct ManagerContext {
     /// Procedure loaders. The key is the type name of the procedure which the loader returns.
     loaders: Mutex<HashMap<String, BoxedProcedureLoader>>,
-    lock_map: LockMap,
+    key_lock: KeyRwLock<String>,
     procedures: RwLock<HashMap<ProcedureId, ProcedureMetaRef>>,
     /// Messages loaded from the procedure store.
     messages: Mutex<HashMap<ProcedureId, ProcedureMessage>>,
@@ -152,8 +149,8 @@ impl ManagerContext {
     /// Returns a new [ManagerContext].
     fn new() -> ManagerContext {
         ManagerContext {
+            key_lock: KeyRwLock::new(),
             loaders: Mutex::new(HashMap::new()),
-            lock_map: LockMap::new(),
             procedures: RwLock::new(HashMap::new()),
             messages: Mutex::new(HashMap::new()),
             finished_procedures: Mutex::new(VecDeque::new()),
@@ -850,7 +847,7 @@ mod tests {
         assert!(manager.procedure_watcher(procedure_id).is_none());
 
         let mut procedure = ProcedureToLoad::new("submit");
-        procedure.lock_key = LockKey::single("test.submit");
+        procedure.lock_key = LockKey::single_exclusive("test.submit");
         assert!(manager
             .submit(ProcedureWithId {
                 id: procedure_id,
@@ -918,7 +915,7 @@ mod tests {
             }
 
             fn lock_key(&self) -> LockKey {
-                LockKey::single("test.submit")
+                LockKey::single_exclusive("test.submit")
             }
         }
 
@@ -955,7 +952,7 @@ mod tests {
         let manager = LocalManager::new(config, state_store);
 
         let mut procedure = ProcedureToLoad::new("submit");
-        procedure.lock_key = LockKey::single("test.submit");
+        procedure.lock_key = LockKey::single_exclusive("test.submit");
         let procedure_id = ProcedureId::random();
         assert_matches!(
             manager
@@ -986,7 +983,7 @@ mod tests {
         manager.start().await.unwrap();
 
         let mut procedure = ProcedureToLoad::new("submit");
-        procedure.lock_key = LockKey::single("test.submit");
+        procedure.lock_key = LockKey::single_exclusive("test.submit");
         let procedure_id = ProcedureId::random();
         assert!(manager
             .submit(ProcedureWithId {
@@ -1018,7 +1015,7 @@ mod tests {
         manager.manager_ctx.set_running();
 
         let mut procedure = ProcedureToLoad::new("submit");
-        procedure.lock_key = LockKey::single("test.submit");
+        procedure.lock_key = LockKey::single_exclusive("test.submit");
         let procedure_id = ProcedureId::random();
         assert!(manager
             .submit(ProcedureWithId {
@@ -1041,7 +1038,7 @@ mod tests {
         // The remove_outdated_meta method has been stopped, so any procedure meta-data will not be automatically removed.
         manager.stop().await.unwrap();
         let mut procedure = ProcedureToLoad::new("submit");
-        procedure.lock_key = LockKey::single("test.submit");
+        procedure.lock_key = LockKey::single_exclusive("test.submit");
         let procedure_id = ProcedureId::random();
 
         manager.manager_ctx.set_running();
@@ -1063,7 +1060,7 @@ mod tests {
 
         // After restart
         let mut procedure = ProcedureToLoad::new("submit");
-        procedure.lock_key = LockKey::single("test.submit");
+        procedure.lock_key = LockKey::single_exclusive("test.submit");
         let procedure_id = ProcedureId::random();
         assert!(manager
             .submit(ProcedureWithId {

src/common/procedure/src/local/lock.rs

@@ -1,214 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::{HashMap, VecDeque};
-use std::sync::RwLock;
-
-use crate::local::ProcedureMetaRef;
-use crate::ProcedureId;
-
-/// A lock entry.
-#[derive(Debug)]
-struct Lock {
-    /// Current lock owner.
-    owner: ProcedureMetaRef,
-    /// Waiter procedures.
-    waiters: VecDeque<ProcedureMetaRef>,
-}
-
-impl Lock {
-    /// Returns a [Lock] with specific `owner` procedure.
-    fn from_owner(owner: ProcedureMetaRef) -> Lock {
-        Lock {
-            owner,
-            waiters: VecDeque::new(),
-        }
-    }
-
-    /// Try to pop a waiter from the waiter list, set it as owner
-    /// and wake up the new owner.
-    ///
-    /// Returns false if there is no waiter in the waiter list.
-    fn switch_owner(&mut self) -> bool {
-        if let Some(waiter) = self.waiters.pop_front() {
-            // Update owner.
-            self.owner = waiter.clone();
-            // We need to use notify_one() since the waiter may have not called `notified()` yet.
-            waiter.lock_notify.notify_one();
-            true
-        } else {
-            false
-        }
-    }
-}
-
-/// Manages lock entries for procedures.
-pub(crate) struct LockMap {
-    locks: RwLock<HashMap<String, Lock>>,
-}
-
-impl LockMap {
-    /// Returns a new [LockMap].
-    pub(crate) fn new() -> LockMap {
-        LockMap {
-            locks: RwLock::new(HashMap::new()),
-        }
-    }
-
-    /// Acquire lock by `key` for procedure with specific `meta`.
-    ///
-    /// Though `meta` is cloneable, callers must ensure that only one `meta`
-    /// is acquiring and holding the lock at the same time.
-    ///
-    /// # Panics
-    /// Panics if the procedure acquires the lock recursively.
-    pub(crate) async fn acquire_lock(&self, key: &str, meta: ProcedureMetaRef) {
-        assert!(!self.hold_lock(key, meta.id));
-
-        {
-            let mut locks = self.locks.write().unwrap();
-            if let Some(lock) = locks.get_mut(key) {
-                // Lock already exists, but we don't expect that a procedure acquires
-                // the same lock again.
-                assert_ne!(lock.owner.id, meta.id);
-
-                // Add this procedure to the waiter list. Here we don't check
-                // whether the procedure is already in the waiter list as we
-                // expect that a procedure should not wait for two lock simultaneously.
-                lock.waiters.push_back(meta.clone());
-            } else {
-                let _ = locks.insert(key.to_string(), Lock::from_owner(meta));
-
-                return;
-            }
-        }
-
-        // Wait for notify.
-        meta.lock_notify.notified().await;
-
-        assert!(self.hold_lock(key, meta.id));
-    }
-
-    /// Release lock by `key`.
-    pub(crate) fn release_lock(&self, key: &str, procedure_id: ProcedureId) {
-        let mut locks = self.locks.write().unwrap();
-        if let Some(lock) = locks.get_mut(key) {
-            if lock.owner.id != procedure_id {
-                // This is not the lock owner.
-                return;
-            }
-
-            if !lock.switch_owner() {
-                // No body waits for this lock, we can remove the lock entry.
-                let _ = locks.remove(key);
-            }
-        }
-    }
-
-    /// Returns true if the procedure with specific `procedure_id` holds the
-    /// lock of `key`.
-    fn hold_lock(&self, key: &str, procedure_id: ProcedureId) -> bool {
-        let locks = self.locks.read().unwrap();
-        locks
-            .get(key)
-            .map(|lock| lock.owner.id == procedure_id)
-            .unwrap_or(false)
-    }
-
-    /// Returns true if the procedure is waiting for the lock `key`.
-    #[cfg(test)]
-    fn waiting_lock(&self, key: &str, procedure_id: ProcedureId) -> bool {
-        let locks = self.locks.read().unwrap();
-        locks
-            .get(key)
-            .map(|lock| lock.waiters.iter().any(|meta| meta.id == procedure_id))
-            .unwrap_or(false)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use super::*;
-    use crate::local::test_util;
-
-    #[test]
-    fn test_lock_no_waiter() {
-        let meta = Arc::new(test_util::procedure_meta_for_test());
-        let mut lock = Lock::from_owner(meta);
-
-        assert!(!lock.switch_owner());
-    }
-
-    #[tokio::test]
-    async fn test_lock_with_waiter() {
-        let owner = Arc::new(test_util::procedure_meta_for_test());
-        let mut lock = Lock::from_owner(owner);
-
-        let waiter = Arc::new(test_util::procedure_meta_for_test());
-        lock.waiters.push_back(waiter.clone());
-
-        assert!(lock.switch_owner());
-        assert!(lock.waiters.is_empty());
-
-        waiter.lock_notify.notified().await;
-        assert_eq!(lock.owner.id, waiter.id);
-    }
-
-    #[tokio::test]
-    async fn test_lock_map() {
-        let key = "hello";
-
-        let owner = Arc::new(test_util::procedure_meta_for_test());
-        let lock_map = Arc::new(LockMap::new());
-        lock_map.acquire_lock(key, owner.clone()).await;
-
-        let waiter = Arc::new(test_util::procedure_meta_for_test());
-        let waiter_id = waiter.id;
-
-        // Waiter release the lock, this should not take effect.
-        lock_map.release_lock(key, waiter_id);
-
-        let lock_map2 = lock_map.clone();
-        let owner_id = owner.id;
-        let handle = tokio::spawn(async move {
-            assert!(lock_map2.hold_lock(key, owner_id));
-            assert!(!lock_map2.hold_lock(key, waiter_id));
-
-            // Waiter wait for lock.
-            lock_map2.acquire_lock(key, waiter.clone()).await;
-
-            assert!(lock_map2.hold_lock(key, waiter_id));
-        });
-
-        // Owner still holds the lock.
-        assert!(lock_map.hold_lock(key, owner_id));
-
-        // Wait until the waiter acquired the lock
-        while !lock_map.waiting_lock(key, waiter_id) {
-            tokio::time::sleep(std::time::Duration::from_millis(5)).await;
-        }
-        // Release lock
-        lock_map.release_lock(key, owner_id);
-        assert!(!lock_map.hold_lock(key, owner_id));
-
-        // Wait for task.
-        handle.await.unwrap();
-        // The waiter should hold the lock now.
-        assert!(lock_map.hold_lock(key, waiter_id));
-
-        lock_map.release_lock(key, waiter_id);
-    }
-}

src/common/procedure/src/local/runner.rs

@@ -19,8 +19,10 @@ use backon::{BackoffBuilder, ExponentialBuilder};
 use common_telemetry::logging;
 use tokio::time;
 
+use super::rwlock::OwnedKeyRwLockGuard;
 use crate::error::{self, ProcedurePanicSnafu, Result};
 use crate::local::{ManagerContext, ProcedureMeta, ProcedureMetaRef};
+use crate::procedure::StringKey;
 use crate::store::ProcedureStore;
 use crate::ProcedureState::Retrying;
 use crate::{BoxedProcedure, Context, Error, ProcedureId, ProcedureState, ProcedureWithId, Status};
@@ -56,6 +58,7 @@ impl ExecResult {
 struct ProcedureGuard {
     meta: ProcedureMetaRef,
     manager_ctx: Arc<ManagerContext>,
+    key_guards: Vec<OwnedKeyRwLockGuard>,
     finish: bool,
 }
 
@@ -65,6 +68,7 @@ impl ProcedureGuard {
         ProcedureGuard {
             meta,
             manager_ctx,
+            key_guards: vec![],
             finish: false,
         }
     }
@@ -95,10 +99,15 @@ impl Drop for ProcedureGuard {
             self.manager_ctx.notify_by_subprocedure(parent_id);
         }
 
-        // Release lock in reverse order.
-        for key in self.meta.lock_key.keys_to_unlock() {
-            self.manager_ctx.lock_map.release_lock(key, self.meta.id);
+        // Drops the key guards in the reverse order.
+        while !self.key_guards.is_empty() {
+            self.key_guards.pop();
         }
+
+        // Clean the staled locks.
+        self.manager_ctx
+            .key_lock
+            .clean_keys(self.meta.lock_key.keys_to_lock().map(|k| k.as_string()));
     }
 }
 
@@ -121,7 +130,7 @@ impl Runner {
     /// Run the procedure.
     pub(crate) async fn run(mut self) {
         // Ensure we can update the procedure state.
-        let guard = ProcedureGuard::new(self.meta.clone(), self.manager_ctx.clone());
+        let mut guard = ProcedureGuard::new(self.meta.clone(), self.manager_ctx.clone());
 
         logging::info!(
             "Runner {}-{} starts",
@@ -133,10 +142,14 @@ impl Runner {
         // recursive locking by adding a root procedure id to the meta.
         for key in self.meta.lock_key.keys_to_lock() {
             // Acquire lock for each key.
-            self.manager_ctx
-                .lock_map
-                .acquire_lock(key, self.meta.clone())
-                .await;
+            let key_guard = match key {
+                StringKey::Share(key) => self.manager_ctx.key_lock.read(key.clone()).await.into(),
+                StringKey::Exclusive(key) => {
+                    self.manager_ctx.key_lock.write(key.clone()).await.into()
+                }
+            };
+
+            guard.key_guards.push(key_guard);
         }
 
         // Execute the procedure. We need to release the lock whenever the the execution
@@ -604,7 +617,7 @@ mod tests {
         };
         let normal = ProcedureAdapter {
             data: "normal".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -665,7 +678,7 @@ mod tests {
         };
         let suspend = ProcedureAdapter {
             data: "suspend".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -697,7 +710,7 @@ mod tests {
         };
         let child = ProcedureAdapter {
             data: "child".to_string(),
-            lock_key: LockKey::new(keys.iter().map(|k| k.to_string())),
+            lock_key: LockKey::new_exclusive(keys.iter().map(|k| k.to_string())),
             exec_fn,
         };
 
@@ -765,7 +778,7 @@ mod tests {
         };
         let parent = ProcedureAdapter {
             data: "parent".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -784,6 +797,7 @@ mod tests {
         runner.manager_ctx = manager_ctx.clone();
 
         runner.run().await;
+        assert!(manager_ctx.key_lock.is_empty());
 
         // Check child procedures.
         for child_id in children_ids {
@@ -810,7 +824,7 @@ mod tests {
         let exec_fn = move |_| async move { Ok(Status::Executing { persist: true }) }.boxed();
         let normal = ProcedureAdapter {
             data: "normal".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -851,7 +865,7 @@ mod tests {
             |_| async { Err(Error::external(MockError::new(StatusCode::Unexpected))) }.boxed();
         let normal = ProcedureAdapter {
             data: "fail".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -875,7 +889,7 @@ mod tests {
             |_| async { Err(Error::external(MockError::new(StatusCode::Unexpected))) }.boxed();
         let fail = ProcedureAdapter {
             data: "fail".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -917,7 +931,7 @@ mod tests {
 
         let retry_later = ProcedureAdapter {
             data: "retry_later".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -952,7 +966,7 @@ mod tests {
 
         let exceed_max_retry_later = ProcedureAdapter {
             data: "exceed_max_retry_later".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -993,7 +1007,7 @@ mod tests {
                     };
                     let fail = ProcedureAdapter {
                         data: "fail".to_string(),
-                        lock_key: LockKey::single("catalog.schema.table.region-0"),
+                        lock_key: LockKey::single_exclusive("catalog.schema.table.region-0"),
                         exec_fn,
                     };
 
@@ -1027,7 +1041,7 @@ mod tests {
         };
         let parent = ProcedureAdapter {
             data: "parent".to_string(),
-            lock_key: LockKey::single("catalog.schema.table"),
+            lock_key: LockKey::single_exclusive("catalog.schema.table"),
             exec_fn,
         };
 
@@ -1042,10 +1056,11 @@ mod tests {
         // Manually add this procedure to the manager ctx.
         assert!(manager_ctx.try_insert_procedure(meta.clone()));
         // Replace the manager ctx.
-        runner.manager_ctx = manager_ctx;
+        runner.manager_ctx = manager_ctx.clone();
 
         // Run the runner and execute the procedure.
         runner.run().await;
+        assert!(manager_ctx.key_lock.is_empty());
         let err = meta.state().error().unwrap().output_msg();
         assert!(err.contains("subprocedure failed"), "{err}");
     }

src/common/procedure/src/local/rwlock.rs

@@ -0,0 +1,247 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::sync::{Arc, Mutex};
+
+use tokio::sync::{OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock};
+
+pub enum OwnedKeyRwLockGuard {
+    Read(OwnedRwLockReadGuard<()>),
+    Write(OwnedRwLockWriteGuard<()>),
+}
+
+impl From<OwnedRwLockReadGuard<()>> for OwnedKeyRwLockGuard {
+    fn from(guard: OwnedRwLockReadGuard<()>) -> Self {
+        OwnedKeyRwLockGuard::Read(guard)
+    }
+}
+
+impl From<OwnedRwLockWriteGuard<()>> for OwnedKeyRwLockGuard {
+    fn from(guard: OwnedRwLockWriteGuard<()>) -> Self {
+        OwnedKeyRwLockGuard::Write(guard)
+    }
+}
+
+/// Locks based on a key, allowing other keys to lock independently.
+#[derive(Debug)]
+pub struct KeyRwLock<K> {
+    /// The inner map of locks for specific keys.
+    inner: Mutex<HashMap<K, Arc<RwLock<()>>>>,
+}
+
+impl<K> KeyRwLock<K>
+where
+    K: Eq + Hash + Clone,
+{
+    pub fn new() -> Self {
+        KeyRwLock {
+            inner: Default::default(),
+        }
+    }
+
+    /// Locks the key with shared read access, returning a guard.
+    pub async fn read(&self, key: K) -> OwnedRwLockReadGuard<()> {
+        let lock = {
+            let mut locks = self.inner.lock().unwrap();
+            locks.entry(key).or_default().clone()
+        };
+
+        lock.read_owned().await
+    }
+
+    /// Locks the key with exclusive write access, returning a guard.
+    pub async fn write(&self, key: K) -> OwnedRwLockWriteGuard<()> {
+        let lock = {
+            let mut locks = self.inner.lock().unwrap();
+            locks.entry(key).or_default().clone()
+        };
+
+        lock.write_owned().await
+    }
+
+    /// Clean up stale locks.
+    ///
+    /// Note: It only cleans a lock if
+    /// - Its strong ref count equals one.
+    /// - Able to acquire the write lock.
+    pub fn clean_keys<'a>(&'a self, iter: impl IntoIterator<Item = &'a K>) {
+        let mut locks = self.inner.lock().unwrap();
+        let mut keys = Vec::new();
+        for key in iter {
+            if let Some(lock) = locks.get(key) {
+                if lock.try_write().is_ok() {
+                    debug_assert_eq!(Arc::weak_count(lock), 0);
+                    // Ensures nobody keeps this ref.
+                    if Arc::strong_count(lock) == 1 {
+                        keys.push(key);
+                    }
+                }
+            }
+        }
+
+        for key in keys {
+            locks.remove(key);
+        }
+    }
+}
+
+#[cfg(test)]
+impl<K> KeyRwLock<K>
+where
+    K: Eq + Hash + Clone,
+{
+    /// Tries to lock the key with shared read access, returning immediately.
+    pub fn try_read(&self, key: K) -> Result<OwnedRwLockReadGuard<()>, tokio::sync::TryLockError> {
+        let lock = {
+            let mut locks = self.inner.lock().unwrap();
+            locks.entry(key).or_default().clone()
+        };
+
+        lock.try_read_owned()
+    }
+
+    /// Tries lock this key with exclusive write access, returning immediately.
+    pub fn try_write(
+        &self,
+        key: K,
+    ) -> Result<OwnedRwLockWriteGuard<()>, tokio::sync::TryLockError> {
+        let lock = {
+            let mut locks = self.inner.lock().unwrap();
+            locks.entry(key).or_default().clone()
+        };
+
+        lock.try_write_owned()
+    }
+
+    /// Returns number of keys.
+    pub fn len(&self) -> usize {
+        self.inner.lock().unwrap().len()
+    }
+
+    /// Returns true the inner map is empty.
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_naive() {
+        let lock_key = KeyRwLock::new();
+
+        {
+            let _guard = lock_key.read("test1").await;
+            assert_eq!(lock_key.len(), 1);
+            assert!(lock_key.try_read("test1").is_ok());
+            assert!(lock_key.try_write("test1").is_err());
+        }
+
+        {
+            let _guard0 = lock_key.write("test2").await;
+            let _guard = lock_key.write("test1").await;
+            assert_eq!(lock_key.len(), 2);
+            assert!(lock_key.try_read("test1").is_err());
+            assert!(lock_key.try_write("test1").is_err());
+        }
+
+        assert_eq!(lock_key.len(), 2);
+
+        lock_key.clean_keys(&vec!["test1", "test2"]);
+        assert!(lock_key.is_empty());
+
+        let mut guards = Vec::new();
+        for key in ["test1", "test2"] {
+            guards.push(lock_key.read(key).await);
+        }
+        while !guards.is_empty() {
+            guards.pop();
+        }
+        lock_key.clean_keys(vec![&"test1", &"test2"]);
+        assert_eq!(lock_key.len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_clean_keys() {
+        let lock_key = KeyRwLock::<&str>::new();
+        {
+            let rwlock = {
+                lock_key
+                    .inner
+                    .lock()
+                    .unwrap()
+                    .entry("test")
+                    .or_default()
+                    .clone()
+            };
+            assert_eq!(Arc::strong_count(&rwlock), 2);
+            let _guard = rwlock.read_owned().await;
+
+            {
+                let inner = lock_key.inner.lock().unwrap();
+                let rwlock = inner.get("test").unwrap();
+                assert_eq!(Arc::strong_count(rwlock), 2);
+            }
+        }
+
+        {
+            let rwlock = {
+                lock_key
+                    .inner
+                    .lock()
+                    .unwrap()
+                    .entry("test")
+                    .or_default()
+                    .clone()
+            };
+            assert_eq!(Arc::strong_count(&rwlock), 2);
+            let _guard = rwlock.write_owned().await;
+
+            {
+                let inner = lock_key.inner.lock().unwrap();
+                let rwlock = inner.get("test").unwrap();
+                assert_eq!(Arc::strong_count(rwlock), 2);
+            }
+        }
+
+        {
+            let inner = lock_key.inner.lock().unwrap();
+            let rwlock = inner.get("test").unwrap();
+            assert_eq!(Arc::strong_count(rwlock), 1);
+        }
+
+        // Someone has the ref of the rwlock, but it waits to be granted the lock.
+        let rwlock = {
+            lock_key
+                .inner
+                .lock()
+                .unwrap()
+                .entry("test")
+                .or_default()
+                .clone()
+        };
+        assert_eq!(Arc::strong_count(&rwlock), 2);
+        // However, One thread trying to remove the "test" key should have no effect.
+        lock_key.clean_keys(vec![&"test"]);
+        // Should get the rwlock.
+        {
+            let inner = lock_key.inner.lock().unwrap();
+            inner.get("test").unwrap();
+        }
+    }
+}

src/common/procedure/src/procedure.rs

@@ -116,22 +116,49 @@ impl<T: Procedure + ?Sized> Procedure for Box<T> {
     }
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum StringKey {
+    Share(String),
+    Exclusive(String),
+}
+
 /// Keys to identify required locks.
 ///
 /// [LockKey] always sorts keys lexicographically so that they can be acquired
 /// in the same order.
-// Most procedures should only acquire 1 ~ 2 locks so we use smallvec to hold keys.
+/// Most procedures should only acquire 1 ~ 2 locks so we use smallvec to hold keys.
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct LockKey(SmallVec<[String; 2]>);
+pub struct LockKey(SmallVec<[StringKey; 2]>);
+
+impl StringKey {
+    pub fn into_string(self) -> String {
+        match self {
+            StringKey::Share(s) => s,
+            StringKey::Exclusive(s) => s,
+        }
+    }
+
+    pub fn as_string(&self) -> &String {
+        match self {
+            StringKey::Share(s) => s,
+            StringKey::Exclusive(s) => s,
+        }
+    }
+}
 
 impl LockKey {
     /// Returns a new [LockKey] with only one key.
-    pub fn single(key: impl Into<String>) -> LockKey {
+    pub fn single(key: impl Into<StringKey>) -> LockKey {
         LockKey(smallvec![key.into()])
     }
 
+    /// Returns a new [LockKey] with only one key.
+    pub fn single_exclusive(key: impl Into<String>) -> LockKey {
+        LockKey(smallvec![StringKey::Exclusive(key.into())])
+    }
+
     /// Returns a new [LockKey] with keys from specific `iter`.
-    pub fn new(iter: impl IntoIterator<Item = String>) -> LockKey {
+    pub fn new(iter: impl IntoIterator<Item = StringKey>) -> LockKey {
         let mut vec: SmallVec<_> = iter.into_iter().collect();
         vec.sort();
         // Dedup keys to avoid acquiring the same key multiple times.
@@ -139,14 +166,14 @@ impl LockKey {
         LockKey(vec)
     }
 
-    /// Returns the keys to lock.
-    pub fn keys_to_lock(&self) -> impl Iterator<Item = &String> {
-        self.0.iter()
+    /// Returns a new [LockKey] with keys from specific `iter`.
+    pub fn new_exclusive(iter: impl IntoIterator<Item = String>) -> LockKey {
+        Self::new(iter.into_iter().map(StringKey::Exclusive))
     }
 
-    /// Returns the keys to unlock.
-    pub fn keys_to_unlock(&self) -> impl Iterator<Item = &String> {
-        self.0.iter().rev()
+    /// Returns the keys to lock.
+    pub fn keys_to_lock(&self) -> impl Iterator<Item = &StringKey> {
+        self.0.iter()
     }
 }
 
@@ -340,20 +367,25 @@ mod tests {
     #[test]
     fn test_lock_key() {
         let entity = "catalog.schema.my_table";
-        let key = LockKey::single(entity);
-        assert_eq!(vec![entity], key.keys_to_lock().collect::<Vec<_>>());
-        assert_eq!(vec![entity], key.keys_to_unlock().collect::<Vec<_>>());
+        let key = LockKey::single_exclusive(entity);
+        assert_eq!(
+            vec![&StringKey::Exclusive(entity.to_string())],
+            key.keys_to_lock().collect::<Vec<_>>()
+        );
 
-        let key = LockKey::new([
+        let key = LockKey::new_exclusive([
             "b".to_string(),
             "c".to_string(),
             "a".to_string(),
             "c".to_string(),
         ]);
-        assert_eq!(vec!["a", "b", "c"], key.keys_to_lock().collect::<Vec<_>>());
         assert_eq!(
-            vec!["c", "b", "a"],
-            key.keys_to_unlock().collect::<Vec<_>>()
+            vec![
+                &StringKey::Exclusive("a".to_string()),
+                &StringKey::Exclusive("b".to_string()),
+                &StringKey::Exclusive("c".to_string())
+            ],
+            key.keys_to_lock().collect::<Vec<_>>()
         );
     }
 

src/common/procedure/src/store/state_store.rs

@@ -87,7 +87,7 @@ impl StateStore for ObjectStateStore {
         let mut lister = self
             .store
             .lister_with(path)
-            .delimiter("")
+            .recursive(true)
             .await
             .map_err(|e| {
                 BoxedError::new(PlainError::new(

src/common/procedure/src/watcher.rs

@@ -98,7 +98,7 @@ mod tests {
             }
 
             fn lock_key(&self) -> LockKey {
-                LockKey::single("test.submit")
+                LockKey::single_exclusive("test.submit")
             }
         }
 

src/common/query/src/prelude.rs

@@ -18,3 +18,10 @@ pub use crate::columnar_value::ColumnarValue;
 pub use crate::function::*;
 pub use crate::logical_plan::{create_udf, AggregateFunction, Expr, ScalarUdf};
 pub use crate::signature::{Signature, TypeSignature, Volatility};
+
+/// Default timestamp column name for Prometheus metrics.
+pub const GREPTIME_TIMESTAMP: &str = "greptime_timestamp";
+/// Default value column name for Prometheus metrics.
+pub const GREPTIME_VALUE: &str = "greptime_value";
+/// Default counter column name for OTLP metrics.
+pub const GREPTIME_COUNT: &str = "greptime_count";

src/common/runtime/src/metrics.rs

@@ -20,13 +20,13 @@ pub const THREAD_NAME_LABEL: &str = "thread_name";
 
 lazy_static! {
     pub static ref METRIC_RUNTIME_THREADS_ALIVE: IntGaugeVec = register_int_gauge_vec!(
-        "runtime_threads_alive",
+        "greptime_runtime_threads_alive",
         "runtime threads alive",
         &[THREAD_NAME_LABEL]
     )
     .unwrap();
     pub static ref METRIC_RUNTIME_THREADS_IDLE: IntGaugeVec = register_int_gauge_vec!(
-        "runtime_threads_idle",
+        "greptime_runtime_threads_idle",
         "runtime threads idle",
         &[THREAD_NAME_LABEL]
     )

src/common/telemetry/src/panic_hook.rs

@@ -22,7 +22,7 @@ use prometheus::*;
 
 lazy_static! {
     pub static ref PANIC_COUNTER: IntCounter =
-        register_int_counter!("panic_counter", "panic_counter").unwrap();
+        register_int_counter!("greptime_panic_counter", "panic_counter").unwrap();
 }
 
 pub fn set_panic_hook() {

src/common/test-util/Cargo.toml

@@ -5,6 +5,9 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
+client.workspace = true
+common-query.workspace = true
+common-recordbatch.workspace = true
 once_cell.workspace = true
 rand.workspace = true
 tempfile.workspace = true

src/common/test-util/src/lib.rs

@@ -19,6 +19,7 @@ use std::process::Command;
 use std::sync::LazyLock;
 
 pub mod ports;
+pub mod recordbatch;
 pub mod temp_dir;
 
 // Rust is working on an env possibly named `CARGO_WORKSPACE_DIR` to find the root path to the

src/common/test-util/src/recordbatch.rs

@@ -0,0 +1,46 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use client::Database;
+use common_query::Output;
+use common_recordbatch::util;
+
+pub enum ExpectedOutput<'a> {
+    AffectedRows(usize),
+    QueryResult(&'a str),
+}
+
+pub async fn execute_and_check_output(db: &Database, sql: &str, expected: ExpectedOutput<'_>) {
+    let output = db.sql(sql).await.unwrap();
+    match (&output, expected) {
+        (Output::AffectedRows(x), ExpectedOutput::AffectedRows(y)) => {
+            assert_eq!(*x, y, "actual: \n{}", x)
+        }
+        (Output::RecordBatches(_), ExpectedOutput::QueryResult(x))
+        | (Output::Stream(_), ExpectedOutput::QueryResult(x)) => {
+            check_output_stream(output, x).await
+        }
+        _ => panic!(),
+    }
+}
+
+pub async fn check_output_stream(output: Output, expected: &str) {
+    let recordbatches = match output {
+        Output::Stream(stream) => util::collect_batches(stream).await.unwrap(),
+        Output::RecordBatches(recordbatches) => recordbatches,
+        _ => unreachable!(),
+    };
+    let pretty_print = recordbatches.pretty_print().unwrap();
+    assert_eq!(pretty_print, expected, "actual: \n{}", pretty_print);
+}

src/common/time/Cargo.toml

@@ -10,6 +10,7 @@ chrono-tz = "0.8"
 chrono.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
+once_cell.workspace = true
 serde = { version = "1.0", features = ["derive"] }
 serde_json.workspace = true
 snafu.workspace = true

src/common/time/src/datetime.rs

@@ -20,7 +20,7 @@ use chrono::{Days, LocalResult, Months, NaiveDateTime, TimeZone as ChronoTimeZon
 use serde::{Deserialize, Serialize};
 
 use crate::error::{Error, InvalidDateStrSnafu, Result};
-use crate::timezone::TimeZone;
+use crate::timezone::Timezone;
 use crate::util::{format_utc_datetime, local_datetime_to_utc};
 use crate::{Date, Interval};
 
@@ -110,11 +110,11 @@ impl DateTime {
         NaiveDateTime::from_timestamp_millis(self.0)
     }
 
-    pub fn to_chrono_datetime_with_timezone(&self, tz: Option<TimeZone>) -> Option<NaiveDateTime> {
+    pub fn to_chrono_datetime_with_timezone(&self, tz: Option<Timezone>) -> Option<NaiveDateTime> {
         let datetime = self.to_chrono_datetime();
         datetime.map(|v| match tz {
-            Some(TimeZone::Offset(offset)) => offset.from_utc_datetime(&v).naive_local(),
-            Some(TimeZone::Named(tz)) => tz.from_utc_datetime(&v).naive_local(),
+            Some(Timezone::Offset(offset)) => offset.from_utc_datetime(&v).naive_local(),
+            Some(Timezone::Named(tz)) => tz.from_utc_datetime(&v).naive_local(),
             None => Utc.from_utc_datetime(&v).naive_local(),
         })
     }
@@ -155,10 +155,11 @@ impl DateTime {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::timezone::set_default_timezone;
 
     #[test]
     pub fn test_new_date_time() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("Asia/Shanghai")).unwrap();
         assert_eq!("1970-01-01 08:00:00+0800", DateTime::new(0).to_string());
         assert_eq!("1970-01-01 08:00:01+0800", DateTime::new(1000).to_string());
         assert_eq!("1970-01-01 07:59:59+0800", DateTime::new(-1000).to_string());
@@ -166,7 +167,7 @@ mod tests {
 
     #[test]
     pub fn test_parse_from_string() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("Asia/Shanghai")).unwrap();
         let time = "1970-01-01 00:00:00+0800";
         let dt = DateTime::from_str(time).unwrap();
         assert_eq!(time, &dt.to_string());
@@ -194,7 +195,7 @@ mod tests {
 
     #[test]
     fn test_parse_local_date_time() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("Asia/Shanghai")).unwrap();
         assert_eq!(
             -28800000,
             DateTime::from_str("1970-01-01 00:00:00").unwrap().val()

src/common/time/src/error.rs

@@ -51,8 +51,8 @@ pub enum Error {
     #[snafu(display("Timestamp arithmetic overflow, msg: {}", msg))]
     ArithmeticOverflow { msg: String, location: Location },
 
-    #[snafu(display("Invalid time zone offset: {hours}:{minutes}"))]
-    InvalidTimeZoneOffset {
+    #[snafu(display("Invalid timezone offset: {hours}:{minutes}"))]
+    InvalidTimezoneOffset {
         hours: i32,
         minutes: u32,
         location: Location,
@@ -66,8 +66,8 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Invalid time zone string {raw}"))]
-    ParseTimeZoneName { raw: String, location: Location },
+    #[snafu(display("Invalid timezone string {raw}"))]
+    ParseTimezoneName { raw: String, location: Location },
 }
 
 impl ErrorExt for Error {
@@ -75,9 +75,9 @@ impl ErrorExt for Error {
         match self {
             Error::ParseDateStr { .. }
             | Error::ParseTimestamp { .. }
-            | Error::InvalidTimeZoneOffset { .. }
+            | Error::InvalidTimezoneOffset { .. }
             | Error::ParseOffsetStr { .. }
-            | Error::ParseTimeZoneName { .. } => StatusCode::InvalidArguments,
+            | Error::ParseTimezoneName { .. } => StatusCode::InvalidArguments,
             Error::TimestampOverflow { .. } => StatusCode::Internal,
             Error::InvalidDateStr { .. } | Error::ArithmeticOverflow { .. } => {
                 StatusCode::InvalidArguments
@@ -96,9 +96,9 @@ impl ErrorExt for Error {
             | Error::TimestampOverflow { location, .. }
             | Error::ArithmeticOverflow { location, .. } => Some(*location),
             Error::ParseDateStr { .. }
-            | Error::InvalidTimeZoneOffset { .. }
+            | Error::InvalidTimezoneOffset { .. }
             | Error::ParseOffsetStr { .. }
-            | Error::ParseTimeZoneName { .. } => None,
+            | Error::ParseTimezoneName { .. } => None,
             Error::InvalidDateStr { location, .. } => Some(*location),
             Error::ParseInterval { location, .. } => Some(*location),
         }

src/common/time/src/lib.rs

@@ -31,4 +31,4 @@ pub use interval::Interval;
 pub use range::RangeMillis;
 pub use timestamp::Timestamp;
 pub use timestamp_millis::TimestampMillis;
-pub use timezone::TimeZone;
+pub use timezone::Timezone;

src/common/time/src/time.rs

@@ -19,8 +19,7 @@ use chrono::{NaiveDateTime, NaiveTime, TimeZone as ChronoTimeZone, Utc};
 use serde::{Deserialize, Serialize};
 
 use crate::timestamp::TimeUnit;
-use crate::timezone::TimeZone;
-use crate::util::format_utc_datetime;
+use crate::timezone::{get_timezone, Timezone};
 
 /// Time value, represents the elapsed time since midnight in the unit of `TimeUnit`.
 #[derive(Debug, Clone, Default, Copy, Serialize, Deserialize)]
@@ -109,30 +108,28 @@ impl Time {
         self.as_formatted_string("%H:%M:%S%.f%z", None)
     }
 
-    /// Format Time for local timezone.
-    pub fn to_local_string(&self) -> String {
+    /// Format Time for system timeszone.
+    pub fn to_system_tz_string(&self) -> String {
         self.as_formatted_string("%H:%M:%S%.f", None)
     }
 
     /// Format Time for given timezone.
-    /// When timezone is None, using local time by default.
-    pub fn to_timezone_aware_string(&self, tz: Option<TimeZone>) -> String {
+    /// When timezone is None, using system timezone by default.
+    pub fn to_timezone_aware_string(&self, tz: Option<Timezone>) -> String {
         self.as_formatted_string("%H:%M:%S%.f", tz)
     }
 
-    fn as_formatted_string(self, pattern: &str, timezone: Option<TimeZone>) -> String {
+    fn as_formatted_string(self, pattern: &str, timezone: Option<Timezone>) -> String {
         if let Some(time) = self.to_chrono_time() {
             let date = Utc::now().date_naive();
             let datetime = NaiveDateTime::new(date, time);
-
-            match timezone {
-                Some(TimeZone::Offset(offset)) => {
+            match get_timezone(timezone) {
+                Timezone::Offset(offset) => {
                     format!("{}", offset.from_utc_datetime(&datetime).format(pattern))
                 }
-                Some(TimeZone::Named(tz)) => {
+                Timezone::Named(tz) => {
                     format!("{}", tz.from_utc_datetime(&datetime).format(pattern))
                 }
-                None => format_utc_datetime(&datetime, pattern),
             }
         } else {
             format!("[Time{}: {}]", self.unit, self.value)
@@ -223,6 +220,7 @@ mod tests {
     use serde_json::Value;
 
     use super::*;
+    use crate::timezone::set_default_timezone;
 
     #[test]
     fn test_time() {
@@ -312,33 +310,33 @@ mod tests {
 
     #[test]
     fn test_to_iso8601_string() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("+10:00")).unwrap();
         let time_millis = 1000001;
         let ts = Time::new_millisecond(time_millis);
-        assert_eq!("08:16:40.001+0800", ts.to_iso8601_string());
+        assert_eq!("10:16:40.001+1000", ts.to_iso8601_string());
 
         let time_millis = 1000;
         let ts = Time::new_millisecond(time_millis);
-        assert_eq!("08:00:01+0800", ts.to_iso8601_string());
+        assert_eq!("10:00:01+1000", ts.to_iso8601_string());
 
         let time_millis = 1;
         let ts = Time::new_millisecond(time_millis);
-        assert_eq!("08:00:00.001+0800", ts.to_iso8601_string());
+        assert_eq!("10:00:00.001+1000", ts.to_iso8601_string());
 
         let time_seconds = 9 * 3600;
         let ts = Time::new_second(time_seconds);
-        assert_eq!("17:00:00+0800", ts.to_iso8601_string());
+        assert_eq!("19:00:00+1000", ts.to_iso8601_string());
 
         let time_seconds = 23 * 3600;
         let ts = Time::new_second(time_seconds);
-        assert_eq!("07:00:00+0800", ts.to_iso8601_string());
+        assert_eq!("09:00:00+1000", ts.to_iso8601_string());
     }
 
     #[test]
     fn test_serialize_to_json_value() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("+10:00")).unwrap();
         assert_eq!(
-            "08:00:01+0800",
+            "10:00:01+1000",
             match serde_json::Value::from(Time::new(1, TimeUnit::Second)) {
                 Value::String(s) => s,
                 _ => unreachable!(),
@@ -346,7 +344,7 @@ mod tests {
         );
 
         assert_eq!(
-            "08:00:00.001+0800",
+            "10:00:00.001+1000",
             match serde_json::Value::from(Time::new(1, TimeUnit::Millisecond)) {
                 Value::String(s) => s,
                 _ => unreachable!(),
@@ -354,7 +352,7 @@ mod tests {
         );
 
         assert_eq!(
-            "08:00:00.000001+0800",
+            "10:00:00.000001+1000",
             match serde_json::Value::from(Time::new(1, TimeUnit::Microsecond)) {
                 Value::String(s) => s,
                 _ => unreachable!(),
@@ -362,7 +360,7 @@ mod tests {
         );
 
         assert_eq!(
-            "08:00:00.000000001+0800",
+            "10:00:00.000000001+1000",
             match serde_json::Value::from(Time::new(1, TimeUnit::Nanosecond)) {
                 Value::String(s) => s,
                 _ => unreachable!(),
@@ -372,46 +370,47 @@ mod tests {
 
     #[test]
     fn test_to_timezone_aware_string() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("+10:00")).unwrap();
 
         assert_eq!(
-            "08:00:00.001",
+            "10:00:00.001",
             Time::new(1, TimeUnit::Millisecond).to_timezone_aware_string(None)
         );
+        std::env::set_var("TZ", "Asia/Shanghai");
         assert_eq!(
             "08:00:00.001",
             Time::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("SYSTEM").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("SYSTEM").unwrap()))
         );
         assert_eq!(
             "08:00:00.001",
             Time::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("+08:00").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("+08:00").unwrap()))
         );
         assert_eq!(
             "07:00:00.001",
             Time::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("+07:00").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("+07:00").unwrap()))
         );
         assert_eq!(
             "23:00:00.001",
             Time::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("-01:00").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("-01:00").unwrap()))
         );
         assert_eq!(
             "08:00:00.001",
             Time::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("Asia/Shanghai").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("Asia/Shanghai").unwrap()))
         );
         assert_eq!(
             "00:00:00.001",
             Time::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("UTC").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("UTC").unwrap()))
         );
         assert_eq!(
             "03:00:00.001",
             Time::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("Europe/Moscow").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("Europe/Moscow").unwrap()))
         );
     }
 }

src/common/time/src/timestamp.rs

@@ -27,12 +27,12 @@ use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 
 use crate::error::{ArithmeticOverflowSnafu, Error, ParseTimestampSnafu, TimestampOverflowSnafu};
-use crate::timezone::TimeZone;
-use crate::util::{div_ceil, format_utc_datetime};
+use crate::timezone::{get_timezone, Timezone};
+use crate::util::div_ceil;
 use crate::{error, Interval};
 
 /// Timestamp represents the value of units(seconds/milliseconds/microseconds/nanoseconds) elapsed
-/// since UNIX epoch. The valid value range of [Timestamp] depends on it's unit (all in UTC time zone):
+/// since UNIX epoch. The valid value range of [Timestamp] depends on it's unit (all in UTC timezone):
 /// - for [TimeUnit::Second]: [-262144-01-01 00:00:00, +262143-12-31 23:59:59]
 /// - for [TimeUnit::Millisecond]: [-262144-01-01 00:00:00.000, +262143-12-31 23:59:59.999]
 /// - for [TimeUnit::Microsecond]: [-262144-01-01 00:00:00.000000, +262143-12-31 23:59:59.999999]
@@ -293,26 +293,26 @@ impl Timestamp {
         self.as_formatted_string("%Y-%m-%d %H:%M:%S%.f%z", None)
     }
 
+    /// Format timestamp use **system timezone**.
     pub fn to_local_string(&self) -> String {
         self.as_formatted_string("%Y-%m-%d %H:%M:%S%.f", None)
     }
 
     /// Format timestamp for given timezone.
-    /// When timezone is None, using local time by default.
-    pub fn to_timezone_aware_string(&self, tz: Option<TimeZone>) -> String {
+    /// If `tz==None`, the server default timezone will used.
+    pub fn to_timezone_aware_string(&self, tz: Option<Timezone>) -> String {
         self.as_formatted_string("%Y-%m-%d %H:%M:%S%.f", tz)
     }
 
-    fn as_formatted_string(self, pattern: &str, timezone: Option<TimeZone>) -> String {
+    fn as_formatted_string(self, pattern: &str, timezone: Option<Timezone>) -> String {
         if let Some(v) = self.to_chrono_datetime() {
-            match timezone {
-                Some(TimeZone::Offset(offset)) => {
+            match get_timezone(timezone) {
+                Timezone::Offset(offset) => {
                     format!("{}", offset.from_utc_datetime(&v).format(pattern))
                 }
-                Some(TimeZone::Named(tz)) => {
+                Timezone::Named(tz) => {
                     format!("{}", tz.from_utc_datetime(&v).format(pattern))
                 }
-                None => format_utc_datetime(&v, pattern),
             }
         } else {
             format!("[Timestamp{}: {}]", self.unit, self.value)
@@ -324,11 +324,11 @@ impl Timestamp {
         NaiveDateTime::from_timestamp_opt(sec, nsec)
     }
 
-    pub fn to_chrono_datetime_with_timezone(&self, tz: Option<TimeZone>) -> Option<NaiveDateTime> {
+    pub fn to_chrono_datetime_with_timezone(&self, tz: Option<Timezone>) -> Option<NaiveDateTime> {
         let datetime = self.to_chrono_datetime();
         datetime.map(|v| match tz {
-            Some(TimeZone::Offset(offset)) => offset.from_utc_datetime(&v).naive_local(),
-            Some(TimeZone::Named(tz)) => tz.from_utc_datetime(&v).naive_local(),
+            Some(Timezone::Offset(offset)) => offset.from_utc_datetime(&v).naive_local(),
+            Some(Timezone::Named(tz)) => tz.from_utc_datetime(&v).naive_local(),
             None => Utc.from_utc_datetime(&v).naive_local(),
         })
     }
@@ -560,6 +560,7 @@ mod tests {
     use serde_json::Value;
 
     use super::*;
+    use crate::timezone::set_default_timezone;
 
     #[test]
     pub fn test_time_unit() {
@@ -789,7 +790,7 @@ mod tests {
 
     #[test]
     fn test_to_iso8601_string() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("Asia/Shanghai")).unwrap();
         let datetime_str = "2020-09-08 13:42:29.042+0000";
         let ts = Timestamp::from_str(datetime_str).unwrap();
         assert_eq!("2020-09-08 21:42:29.042+0800", ts.to_iso8601_string());
@@ -813,7 +814,7 @@ mod tests {
 
     #[test]
     fn test_serialize_to_json_value() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("Asia/Shanghai")).unwrap();
         assert_eq!(
             "1970-01-01 08:00:01+0800",
             match serde_json::Value::from(Timestamp::new(1, TimeUnit::Second)) {
@@ -1054,7 +1055,7 @@ mod tests {
 
     // $TZ doesn't take effort.
     #[test]
-    fn test_parse_in_time_zone() {
+    fn test_parse_in_timezone() {
         std::env::set_var("TZ", "Asia/Shanghai");
         assert_eq!(
             Timestamp::new(28800, TimeUnit::Second),
@@ -1074,7 +1075,7 @@ mod tests {
 
     #[test]
     fn test_to_local_string() {
-        std::env::set_var("TZ", "Asia/Shanghai");
+        set_default_timezone(Some("Asia/Shanghai")).unwrap();
 
         assert_eq!(
             "1970-01-01 08:00:00.000000001",
@@ -1107,51 +1108,52 @@ mod tests {
 
     #[test]
     fn test_to_timezone_aware_string() {
+        set_default_timezone(Some("Asia/Shanghai")).unwrap();
         std::env::set_var("TZ", "Asia/Shanghai");
-
         assert_eq!(
             "1970-01-01 08:00:00.001",
-            Timestamp::new(1, TimeUnit::Millisecond).to_timezone_aware_string(None)
+            Timestamp::new(1, TimeUnit::Millisecond)
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("SYSTEM").unwrap()))
         );
         assert_eq!(
             "1970-01-01 08:00:00.001",
             Timestamp::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("SYSTEM").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("SYSTEM").unwrap()))
         );
         assert_eq!(
             "1970-01-01 08:00:00.001",
             Timestamp::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("+08:00").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("+08:00").unwrap()))
         );
         assert_eq!(
             "1970-01-01 07:00:00.001",
             Timestamp::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("+07:00").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("+07:00").unwrap()))
         );
         assert_eq!(
             "1969-12-31 23:00:00.001",
             Timestamp::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("-01:00").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("-01:00").unwrap()))
         );
         assert_eq!(
             "1970-01-01 08:00:00.001",
             Timestamp::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("Asia/Shanghai").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("Asia/Shanghai").unwrap()))
         );
         assert_eq!(
             "1970-01-01 00:00:00.001",
             Timestamp::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("UTC").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("UTC").unwrap()))
         );
         assert_eq!(
             "1970-01-01 01:00:00.001",
             Timestamp::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("Europe/Berlin").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("Europe/Berlin").unwrap()))
         );
         assert_eq!(
             "1970-01-01 03:00:00.001",
             Timestamp::new(1, TimeUnit::Millisecond)
-                .to_timezone_aware_string(TimeZone::from_tz_string("Europe/Moscow").unwrap())
+                .to_timezone_aware_string(Some(Timezone::from_tz_string("Europe/Moscow").unwrap()))
         );
     }
 

src/common/time/src/timezone.rs

@@ -15,24 +15,52 @@
 use std::fmt::Display;
 use std::str::FromStr;
 
-use chrono::{FixedOffset, Local, Offset};
+use chrono::FixedOffset;
 use chrono_tz::Tz;
+use once_cell::sync::OnceCell;
 use snafu::{OptionExt, ResultExt};
 
 use crate::error::{
-    InvalidTimeZoneOffsetSnafu, ParseOffsetStrSnafu, ParseTimeZoneNameSnafu, Result,
+    InvalidTimezoneOffsetSnafu, ParseOffsetStrSnafu, ParseTimezoneNameSnafu, Result,
 };
 use crate::util::find_tz_from_env;
 
+/// System timezone in `frontend`/`standalone`,
+/// config by option `default_timezone` in toml,
+/// default value is `UTC` when `default_timezone` is not set.
+static DEFAULT_TIMEZONE: OnceCell<Timezone> = OnceCell::new();
+
+// Set the System timezone by `tz_str`
+pub fn set_default_timezone(tz_str: Option<&str>) -> Result<()> {
+    let tz = match tz_str {
+        None | Some("") => Timezone::Named(Tz::UTC),
+        Some(tz) => Timezone::from_tz_string(tz)?,
+    };
+    DEFAULT_TIMEZONE.get_or_init(|| tz);
+    Ok(())
+}
+
+#[inline(always)]
+/// If the `tz=Some(timezone)`, return `timezone` directly,
+/// or return current system timezone.
+pub fn get_timezone(tz: Option<Timezone>) -> Timezone {
+    tz.unwrap_or_else(|| {
+        DEFAULT_TIMEZONE
+            .get()
+            .cloned()
+            .unwrap_or(Timezone::Named(Tz::UTC))
+    })
+}
+
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub enum TimeZone {
+pub enum Timezone {
     Offset(FixedOffset),
     Named(Tz),
 }
 
-impl TimeZone {
+impl Timezone {
     /// Compute timezone from given offset hours and minutes
-    /// Return `None` if given offset exceeds scope
+    /// Return `Err` if given offset exceeds scope
     pub fn hours_mins_opt(offset_hours: i32, offset_mins: u32) -> Result<Self> {
         let offset_secs = if offset_hours > 0 {
             offset_hours * 3600 + offset_mins as i32 * 60
@@ -42,7 +70,7 @@ impl TimeZone {
 
         FixedOffset::east_opt(offset_secs)
             .map(Self::Offset)
-            .context(InvalidTimeZoneOffsetSnafu {
+            .context(InvalidTimezoneOffsetSnafu {
                 hours: offset_hours,
                 minutes: offset_mins,
             })
@@ -57,10 +85,10 @@ impl TimeZone {
     /// - `SYSTEM`
     /// - Offset to UTC: `+08:00` , `-11:30`
     /// - Named zones: `Asia/Shanghai`, `Europe/Berlin`
-    pub fn from_tz_string(tz_string: &str) -> Result<Option<Self>> {
+    pub fn from_tz_string(tz_string: &str) -> Result<Self> {
         // Use system timezone
         if tz_string.eq_ignore_ascii_case("SYSTEM") {
-            Ok(None)
+            Ok(Timezone::Named(find_tz_from_env().unwrap_or(Tz::UTC)))
         } else if let Some((hrs, mins)) = tz_string.split_once(':') {
             let hrs = hrs
                 .parse::<i32>()
@@ -68,16 +96,16 @@ impl TimeZone {
             let mins = mins
                 .parse::<u32>()
                 .context(ParseOffsetStrSnafu { raw: tz_string })?;
-            Self::hours_mins_opt(hrs, mins).map(Some)
+            Self::hours_mins_opt(hrs, mins)
         } else if let Ok(tz) = Tz::from_str(tz_string) {
-            Ok(Some(Self::Named(tz)))
+            Ok(Self::Named(tz))
         } else {
-            ParseTimeZoneNameSnafu { raw: tz_string }.fail()
+            ParseTimezoneNameSnafu { raw: tz_string }.fail()
         }
     }
 }
 
-impl Display for TimeZone {
+impl Display for Timezone {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::Named(tz) => write!(f, "{}", tz.name()),
@@ -87,12 +115,9 @@ impl Display for TimeZone {
 }
 
 #[inline]
-pub fn system_time_zone_name() -> String {
-    if let Some(tz) = find_tz_from_env() {
-        Local::now().with_timezone(&tz).offset().fix().to_string()
-    } else {
-        Local::now().offset().to_string()
-    }
+/// Return current system config timezone, default config is UTC
+pub fn system_timezone_name() -> String {
+    format!("{}", get_timezone(None))
 }
 
 #[cfg(test)]
@@ -101,61 +126,56 @@ mod tests {
 
     #[test]
     fn test_from_tz_string() {
-        assert_eq!(None, TimeZone::from_tz_string("SYSTEM").unwrap());
-
-        let utc_plus_8 = Some(TimeZone::Offset(FixedOffset::east_opt(3600 * 8).unwrap()));
-        assert_eq!(utc_plus_8, TimeZone::from_tz_string("+8:00").unwrap());
-        assert_eq!(utc_plus_8, TimeZone::from_tz_string("+08:00").unwrap());
-        assert_eq!(utc_plus_8, TimeZone::from_tz_string("08:00").unwrap());
-
-        let utc_minus_8 = Some(TimeZone::Offset(FixedOffset::west_opt(3600 * 8).unwrap()));
-        assert_eq!(utc_minus_8, TimeZone::from_tz_string("-08:00").unwrap());
-        assert_eq!(utc_minus_8, TimeZone::from_tz_string("-8:00").unwrap());
-
-        let utc_minus_8_5 = Some(TimeZone::Offset(
-            FixedOffset::west_opt(3600 * 8 + 60 * 30).unwrap(),
-        ));
-        assert_eq!(utc_minus_8_5, TimeZone::from_tz_string("-8:30").unwrap());
-
-        let utc_plus_max = Some(TimeZone::Offset(FixedOffset::east_opt(3600 * 14).unwrap()));
-        assert_eq!(utc_plus_max, TimeZone::from_tz_string("14:00").unwrap());
-
-        let utc_minus_max = Some(TimeZone::Offset(
-            FixedOffset::west_opt(3600 * 13 + 60 * 59).unwrap(),
-        ));
-        assert_eq!(utc_minus_max, TimeZone::from_tz_string("-13:59").unwrap());
-
         assert_eq!(
-            Some(TimeZone::Named(Tz::Asia__Shanghai)),
-            TimeZone::from_tz_string("Asia/Shanghai").unwrap()
-        );
-        assert_eq!(
-            Some(TimeZone::Named(Tz::UTC)),
-            TimeZone::from_tz_string("UTC").unwrap()
+            Timezone::Named(Tz::UTC),
+            Timezone::from_tz_string("SYSTEM").unwrap()
         );
 
-        assert!(TimeZone::from_tz_string("WORLD_PEACE").is_err());
-        assert!(TimeZone::from_tz_string("A0:01").is_err());
-        assert!(TimeZone::from_tz_string("20:0A").is_err());
-        assert!(TimeZone::from_tz_string(":::::").is_err());
-        assert!(TimeZone::from_tz_string("Asia/London").is_err());
-        assert!(TimeZone::from_tz_string("Unknown").is_err());
+        let utc_plus_8 = Timezone::Offset(FixedOffset::east_opt(3600 * 8).unwrap());
+        assert_eq!(utc_plus_8, Timezone::from_tz_string("+8:00").unwrap());
+        assert_eq!(utc_plus_8, Timezone::from_tz_string("+08:00").unwrap());
+        assert_eq!(utc_plus_8, Timezone::from_tz_string("08:00").unwrap());
+
+        let utc_minus_8 = Timezone::Offset(FixedOffset::west_opt(3600 * 8).unwrap());
+        assert_eq!(utc_minus_8, Timezone::from_tz_string("-08:00").unwrap());
+        assert_eq!(utc_minus_8, Timezone::from_tz_string("-8:00").unwrap());
+
+        let utc_minus_8_5 = Timezone::Offset(FixedOffset::west_opt(3600 * 8 + 60 * 30).unwrap());
+        assert_eq!(utc_minus_8_5, Timezone::from_tz_string("-8:30").unwrap());
+
+        let utc_plus_max = Timezone::Offset(FixedOffset::east_opt(3600 * 14).unwrap());
+        assert_eq!(utc_plus_max, Timezone::from_tz_string("14:00").unwrap());
+
+        let utc_minus_max = Timezone::Offset(FixedOffset::west_opt(3600 * 13 + 60 * 59).unwrap());
+        assert_eq!(utc_minus_max, Timezone::from_tz_string("-13:59").unwrap());
+
+        assert_eq!(
+            Timezone::Named(Tz::Asia__Shanghai),
+            Timezone::from_tz_string("Asia/Shanghai").unwrap()
+        );
+        assert_eq!(
+            Timezone::Named(Tz::UTC),
+            Timezone::from_tz_string("UTC").unwrap()
+        );
+
+        assert!(Timezone::from_tz_string("WORLD_PEACE").is_err());
+        assert!(Timezone::from_tz_string("A0:01").is_err());
+        assert!(Timezone::from_tz_string("20:0A").is_err());
+        assert!(Timezone::from_tz_string(":::::").is_err());
+        assert!(Timezone::from_tz_string("Asia/London").is_err());
+        assert!(Timezone::from_tz_string("Unknown").is_err());
     }
 
     #[test]
     fn test_timezone_to_string() {
-        assert_eq!("UTC", TimeZone::Named(Tz::UTC).to_string());
+        assert_eq!("UTC", Timezone::Named(Tz::UTC).to_string());
         assert_eq!(
             "+01:00",
-            TimeZone::from_tz_string("01:00")
-                .unwrap()
-                .unwrap()
-                .to_string()
+            Timezone::from_tz_string("01:00").unwrap().to_string()
         );
         assert_eq!(
             "Asia/Shanghai",
-            TimeZone::from_tz_string("Asia/Shanghai")
-                .unwrap()
+            Timezone::from_tz_string("Asia/Shanghai")
                 .unwrap()
                 .to_string()
         );