chore: pick #6415 to release/v0.15 (#6686)

ci: add check-version script to check whether push the latast image (#6415)

Signed-off-by: liyang <daviderli614@gmail.com>
Co-authored-by: liyang <daviderli614@gmail.com>

.cargo/config.toml

@@ -12,3 +12,6 @@ fetch = true
 checkout = true
 list_files = true
 internal_use_git2 = false
+
+[env]
+CARGO_WORKSPACE_DIR = { value = "", relative = true }

.github/scripts/check-version.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Get current version
+CURRENT_VERSION=$1
+if [ -z "$CURRENT_VERSION" ]; then
+  echo "Error: Failed to get current version"
+  exit 1
+fi
+
+# Get the latest version from GitHub Releases
+API_RESPONSE=$(curl -s "https://api.github.com/repos/GreptimeTeam/greptimedb/releases/latest")
+
+if [ -z "$API_RESPONSE" ] || [ "$(echo "$API_RESPONSE" | jq -r '.message')" = "Not Found" ]; then
+  echo "Error: Failed to fetch latest version from GitHub"
+  exit 1
+fi
+
+# Get the latest version
+LATEST_VERSION=$(echo "$API_RESPONSE" | jq -r '.tag_name')
+
+if [ -z "$LATEST_VERSION" ] || [ "$LATEST_VERSION" = "null" ]; then
+  echo "Error: No valid version found in GitHub releases"
+  exit 1
+fi
+
+# Cleaned up version number format (removed possible 'v' prefix and -nightly suffix)
+CLEAN_CURRENT=$(echo "$CURRENT_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//')
+CLEAN_LATEST=$(echo "$LATEST_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//')
+
+echo "Current version: $CLEAN_CURRENT"
+echo "Latest release version: $CLEAN_LATEST"
+
+# Use sort -V to compare versions
+HIGHER_VERSION=$(printf "%s\n%s" "$CLEAN_CURRENT" "$CLEAN_LATEST" | sort -V | tail -n1)
+
+if [ "$HIGHER_VERSION" = "$CLEAN_CURRENT" ]; then
+  echo "Current version ($CLEAN_CURRENT) is NEWER than or EQUAL to latest ($CLEAN_LATEST)"
+  echo "should-push-latest-tag=true" >> $GITHUB_OUTPUT
+else
+  echo "Current version ($CLEAN_CURRENT) is OLDER than latest ($CLEAN_LATEST)"
+  echo "should-push-latest-tag=false" >> $GITHUB_OUTPUT
+fi

.github/workflows/release.yml

@@ -110,6 +110,8 @@ jobs:
 
       # The 'version' use as the global tag name of the release workflow.
       version: ${{ steps.create-version.outputs.version }}
+
+      should-push-latest-tag: ${{ steps.check-version.outputs.should-push-latest-tag }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -135,6 +137,11 @@ jobs:
           GITHUB_REF_NAME: ${{ github.ref_name }}
           NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
 
+      - name: Check version
+        id: check-version
+        run: |
+          ./.github/scripts/check-version.sh "${{ steps.create-version.outputs.version }}"
+
       - name: Allocate linux-amd64 runner
         if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
         uses: ./.github/actions/start-runner
@@ -314,7 +321,7 @@ jobs:
           image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          push-latest-tag: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
+          push-latest-tag: ${{ needs.allocate-runners.outputs.should-push-latest-tag == 'true' && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
 
       - name: Set build image result
         id: set-build-image-result
@@ -361,7 +368,7 @@ jobs:
           dev-mode: false
           upload-to-s3: true
           update-version-info: true
-          push-latest-tag: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
+          push-latest-tag: ${{ needs.allocate-runners.outputs.should-push-latest-tag == 'true' && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
 
   publish-github-release:
     name: Create GitHub release and upload artifacts

Cargo.lock

@@ -211,7 +211,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"
 
 [[package]]
 name = "api"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "common-base",
  "common-decimal",
@@ -944,7 +944,7 @@ dependencies = [
 
 [[package]]
 name = "auth"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "async-trait",
@@ -1586,7 +1586,7 @@ dependencies = [
 
 [[package]]
 name = "cache"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "catalog",
  "common-error",
@@ -1602,6 +1602,17 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564"
 
+[[package]]
+name = "cargo-manifest"
+version = "0.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d8af896b707212cd0e99c112a78c9497dd32994192a463ed2f7419d29bd8c6"
+dependencies = [
+ "serde",
+ "thiserror 2.0.12",
+ "toml 0.8.19",
+]
+
 [[package]]
 name = "cast"
 version = "0.3.0"
@@ -1610,7 +1621,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "catalog"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "arrow 54.2.1",
@@ -1648,6 +1659,8 @@ dependencies = [
  "partition",
  "paste",
  "prometheus",
+ "promql-parser",
+ "rand 0.9.0",
  "rustc-hash 2.0.0",
  "serde_json",
  "session",
@@ -1948,7 +1961,7 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
 
 [[package]]
 name = "cli"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -1993,7 +2006,7 @@ dependencies = [
  "session",
  "snafu 0.8.5",
  "store-api",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "table",
  "tempfile",
  "tokio",
@@ -2002,7 +2015,7 @@ dependencies = [
 
 [[package]]
 name = "client"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -2032,7 +2045,7 @@ dependencies = [
  "rand 0.9.0",
  "serde_json",
  "snafu 0.8.5",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "substrait 0.37.3",
  "tokio",
  "tokio-stream",
@@ -2073,7 +2086,7 @@ dependencies = [
 
 [[package]]
 name = "cmd"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-trait",
  "auth",
@@ -2134,7 +2147,7 @@ dependencies = [
  "snafu 0.8.5",
  "stat",
  "store-api",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "table",
  "temp-env",
  "tempfile",
@@ -2181,7 +2194,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"
 
 [[package]]
 name = "common-base"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "anymap2",
  "async-trait",
@@ -2203,11 +2216,11 @@ dependencies = [
 
 [[package]]
 name = "common-catalog"
-version = "0.15.0"
+version = "0.15.5"
 
 [[package]]
 name = "common-config"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "common-base",
  "common-error",
@@ -2232,7 +2245,7 @@ dependencies = [
 
 [[package]]
 name = "common-datasource"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "arrow 54.2.1",
  "arrow-schema 54.3.1",
@@ -2269,7 +2282,7 @@ dependencies = [
 
 [[package]]
 name = "common-decimal"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "bigdecimal 0.4.8",
  "common-error",
@@ -2282,7 +2295,7 @@ dependencies = [
 
 [[package]]
 name = "common-error"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "common-macro",
  "http 1.1.0",
@@ -2293,7 +2306,7 @@ dependencies = [
 
 [[package]]
 name = "common-frontend"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-trait",
  "common-error",
@@ -2302,6 +2315,7 @@ dependencies = [
  "common-meta",
  "greptime-proto",
  "meta-client",
+ "session",
  "snafu 0.8.5",
  "tokio",
  "tonic 0.12.3",
@@ -2309,7 +2323,7 @@ dependencies = [
 
 [[package]]
 name = "common-function"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -2362,7 +2376,7 @@ dependencies = [
 
 [[package]]
 name = "common-greptimedb-telemetry"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-trait",
  "common-runtime",
@@ -2379,7 +2393,7 @@ dependencies = [
 
 [[package]]
 name = "common-grpc"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -2411,7 +2425,7 @@ dependencies = [
 
 [[package]]
 name = "common-grpc-expr"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "common-base",
@@ -2430,7 +2444,7 @@ dependencies = [
 
 [[package]]
 name = "common-macro"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "arc-swap",
  "common-query",
@@ -2444,7 +2458,7 @@ dependencies = [
 
 [[package]]
 name = "common-mem-prof"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "anyhow",
  "common-error",
@@ -2460,7 +2474,7 @@ dependencies = [
 
 [[package]]
 name = "common-meta"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "anymap2",
  "api",
@@ -2480,6 +2494,7 @@ dependencies = [
  "common-procedure-test",
  "common-query",
  "common-recordbatch",
+ "common-runtime",
  "common-telemetry",
  "common-test-util",
  "common-time",
@@ -2525,7 +2540,7 @@ dependencies = [
 
 [[package]]
 name = "common-options"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "common-grpc",
  "humantime-serde",
@@ -2534,11 +2549,11 @@ dependencies = [
 
 [[package]]
 name = "common-plugins"
-version = "0.15.0"
+version = "0.15.5"
 
 [[package]]
 name = "common-pprof"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "common-error",
  "common-macro",
@@ -2550,7 +2565,7 @@ dependencies = [
 
 [[package]]
 name = "common-procedure"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -2577,16 +2592,17 @@ dependencies = [
 
 [[package]]
 name = "common-procedure-test"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-trait",
  "common-procedure",
  "snafu 0.8.5",
+ "tokio",
 ]
 
 [[package]]
 name = "common-query"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "async-trait",
@@ -2612,7 +2628,7 @@ dependencies = [
 
 [[package]]
 name = "common-recordbatch"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "arc-swap",
  "common-error",
@@ -2632,7 +2648,7 @@ dependencies = [
 
 [[package]]
 name = "common-runtime"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-trait",
  "clap 4.5.19",
@@ -2662,17 +2678,18 @@ dependencies = [
 
 [[package]]
 name = "common-session"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "strum 0.27.1",
 ]
 
 [[package]]
 name = "common-telemetry"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "backtrace",
  "common-error",
+ "common-version",
  "console-subscriber",
  "greptime-proto",
  "humantime-serde",
@@ -2696,7 +2713,7 @@ dependencies = [
 
 [[package]]
 name = "common-test-util"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "client",
  "common-grpc",
@@ -2709,7 +2726,7 @@ dependencies = [
 
 [[package]]
 name = "common-time"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "arrow 54.2.1",
  "chrono",
@@ -2727,9 +2744,10 @@ dependencies = [
 
 [[package]]
 name = "common-version"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "build-data",
+ "cargo-manifest",
  "const_format",
  "serde",
  "shadow-rs",
@@ -2737,7 +2755,7 @@ dependencies = [
 
 [[package]]
 name = "common-wal"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "common-base",
  "common-error",
@@ -2760,7 +2778,7 @@ dependencies = [
 
 [[package]]
 name = "common-workload"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "common-telemetry",
@@ -3716,7 +3734,7 @@ dependencies = [
 
 [[package]]
 name = "datanode"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -3769,7 +3787,7 @@ dependencies = [
  "session",
  "snafu 0.8.5",
  "store-api",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "table",
  "tokio",
  "toml 0.8.19",
@@ -3778,7 +3796,7 @@ dependencies = [
 
 [[package]]
 name = "datatypes"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "arrow 54.2.1",
  "arrow-array 54.2.1",
@@ -4438,7 +4456,7 @@ checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
 
 [[package]]
 name = "file-engine"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "async-trait",
@@ -4575,7 +4593,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8"
 
 [[package]]
 name = "flow"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "arrow 54.2.1",
@@ -4640,7 +4658,7 @@ dependencies = [
  "sql",
  "store-api",
  "strum 0.27.1",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "table",
  "tokio",
  "tonic 0.12.3",
@@ -4695,10 +4713,11 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
 
 [[package]]
 name = "frontend"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "arc-swap",
+ "async-stream",
  "async-trait",
  "auth",
  "bytes",
@@ -4754,7 +4773,7 @@ dependencies = [
  "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)",
  "store-api",
  "strfmt",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "table",
  "tokio",
  "tokio-util",
@@ -5144,7 +5163,7 @@ dependencies = [
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=82fe5c6282f623c185b86f03e898ee8952e50cf9#82fe5c6282f623c185b86f03e898ee8952e50cf9"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=f3103a8c9b8ce162457d0a3e3ca00d53d1a8bd06#f3103a8c9b8ce162457d0a3e3ca00d53d1a8bd06"
 dependencies = [
  "prost 0.13.5",
  "serde",
@@ -5915,7 +5934,7 @@ dependencies = [
 
 [[package]]
 name = "index"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-trait",
  "asynchronous-codec",
@@ -6800,7 +6819,7 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
 [[package]]
 name = "log-query"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "chrono",
  "common-error",
@@ -6812,7 +6831,7 @@ dependencies = [
 
 [[package]]
 name = "log-store"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -7110,7 +7129,7 @@ dependencies = [
 
 [[package]]
 name = "meta-client"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "async-trait",
@@ -7138,10 +7157,13 @@ dependencies = [
 
 [[package]]
 name = "meta-srv"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "async-trait",
+ "axum 0.8.1",
+ "axum-extra",
+ "axum-macros",
  "bytes",
  "chrono",
  "clap 4.5.19",
@@ -7174,6 +7196,7 @@ dependencies = [
  "http-body-util",
  "humantime",
  "humantime-serde",
+ "hyper 0.14.30",
  "hyper-util",
  "itertools 0.14.0",
  "lazy_static",
@@ -7201,6 +7224,7 @@ dependencies = [
  "toml 0.8.19",
  "tonic 0.12.3",
  "tower 0.5.2",
+ "tower-http 0.6.2",
  "tracing",
  "tracing-subscriber",
  "typetag",
@@ -7229,7 +7253,7 @@ dependencies = [
 
 [[package]]
 name = "metric-engine"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -7239,6 +7263,7 @@ dependencies = [
  "common-base",
  "common-error",
  "common-macro",
+ "common-meta",
  "common-query",
  "common-recordbatch",
  "common-runtime",
@@ -7253,6 +7278,7 @@ dependencies = [
  "lazy_static",
  "mito-codec",
  "mito2",
+ "moka",
  "mur3",
  "object-store",
  "prometheus",
@@ -7319,7 +7345,7 @@ dependencies = [
 
 [[package]]
 name = "mito-codec"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "bytes",
@@ -7342,7 +7368,7 @@ dependencies = [
 
 [[package]]
 name = "mito2"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -8092,7 +8118,7 @@ dependencies = [
 
 [[package]]
 name = "object-store"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "anyhow",
  "bytes",
@@ -8406,7 +8432,7 @@ dependencies = [
 
 [[package]]
 name = "operator"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -8461,7 +8487,7 @@ dependencies = [
  "sql",
  "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)",
  "store-api",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "table",
  "tokio",
  "tokio-util",
@@ -8728,7 +8754,7 @@ dependencies = [
 
 [[package]]
 name = "partition"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "async-trait",
@@ -9016,7 +9042,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pipeline"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -9159,7 +9185,7 @@ dependencies = [
 
 [[package]]
 name = "plugins"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "auth",
  "clap 4.5.19",
@@ -9472,7 +9498,7 @@ dependencies = [
 
 [[package]]
 name = "promql"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "ahash 0.8.11",
  "async-trait",
@@ -9568,7 +9594,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
 dependencies = [
  "heck 0.5.0",
- "itertools 0.14.0",
+ "itertools 0.11.0",
  "log",
  "multimap",
  "once_cell",
@@ -9614,7 +9640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
 dependencies = [
  "anyhow",
- "itertools 0.14.0",
+ "itertools 0.11.0",
  "proc-macro2",
  "quote",
  "syn 2.0.100",
@@ -9754,7 +9780,7 @@ dependencies = [
 
 [[package]]
 name = "puffin"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-compression 0.4.13",
  "async-trait",
@@ -9796,7 +9822,7 @@ dependencies = [
 
 [[package]]
 name = "query"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -9862,7 +9888,7 @@ dependencies = [
  "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)",
  "statrs",
  "store-api",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "table",
  "tokio",
  "tokio-stream",
@@ -11148,7 +11174,7 @@ dependencies = [
 
 [[package]]
 name = "servers"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -11269,7 +11295,7 @@ dependencies = [
 
 [[package]]
 name = "session"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -11608,7 +11634,7 @@ dependencies = [
 
 [[package]]
 name = "sql"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "chrono",
@@ -11663,7 +11689,7 @@ dependencies = [
 
 [[package]]
 name = "sqlness-runner"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-trait",
  "clap 4.5.19",
@@ -11963,7 +11989,7 @@ dependencies = [
 
 [[package]]
 name = "stat"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "nix 0.30.1",
 ]
@@ -11989,7 +12015,7 @@ dependencies = [
 
 [[package]]
 name = "store-api"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -12150,7 +12176,7 @@ dependencies = [
 
 [[package]]
 name = "substrait"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "async-trait",
  "bytes",
@@ -12330,7 +12356,7 @@ dependencies = [
 
 [[package]]
 name = "table"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "async-trait",
@@ -12591,7 +12617,7 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
 
 [[package]]
 name = "tests-fuzz"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "arbitrary",
  "async-trait",
@@ -12635,7 +12661,7 @@ dependencies = [
 
 [[package]]
 name = "tests-integration"
-version = "0.15.0"
+version = "0.15.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -12702,7 +12728,7 @@ dependencies = [
  "sql",
  "sqlx",
  "store-api",
- "substrait 0.15.0",
+ "substrait 0.15.5",
  "table",
  "tempfile",
  "time",
@@ -12712,6 +12738,7 @@ dependencies = [
  "tonic 0.12.3",
  "tower 0.5.2",
  "url",
+ "urlencoding",
  "uuid",
  "yaml-rust",
  "zstd 0.13.2",
@@ -13072,6 +13099,7 @@ version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
 dependencies = [
+ "indexmap 2.9.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -14183,7 +14211,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]

Cargo.toml

@@ -71,7 +71,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.15.0"
+version = "0.15.5"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -134,7 +134,7 @@ etcd-client = "0.14"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "82fe5c6282f623c185b86f03e898ee8952e50cf9" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "f3103a8c9b8ce162457d0a3e3ca00d53d1a8bd06" }
 hex = "0.4"
 http = "1"
 humantime = "2.1"
@@ -220,6 +220,8 @@ tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
 tonic = { version = "0.12", features = ["tls", "gzip", "zstd"] }
 tower = "0.5"
+tower-http = "0.6"
+tracing = "0.1"
 tracing-appender = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
 typetag = "0.2"

config/config.md

@@ -147,6 +147,7 @@
 | `region_engine.mito.write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
+| `region_engine.mito.max_concurrent_scan_files` | Integer | `128` | Maximum number of SST files to scan concurrently. |
 | `region_engine.mito.allow_stale_entries` | Bool | `false` | Whether to allow stale WAL entries read during replay. |
 | `region_engine.mito.min_compaction_interval` | String | `0m` | Minimum time interval between two compactions.<br/>To align with the old behavior, the default value is 0 (no restrictions). |
 | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. |
@@ -496,6 +497,7 @@
 | `region_engine.mito.write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
+| `region_engine.mito.max_concurrent_scan_files` | Integer | `128` | Maximum number of SST files to scan concurrently. |
 | `region_engine.mito.allow_stale_entries` | Bool | `false` | Whether to allow stale WAL entries read during replay. |
 | `region_engine.mito.min_compaction_interval` | String | `0m` | Minimum time interval between two compactions.<br/>To align with the old behavior, the default value is 0 (no restrictions). |
 | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. |

config/datanode.example.toml

@@ -474,6 +474,9 @@ sst_write_buffer_size = "8MB"
 ## Capacity of the channel to send data from parallel scan tasks to the main task.
 parallel_scan_channel_size = 32
 
+## Maximum number of SST files to scan concurrently.
+max_concurrent_scan_files = 128
+
 ## Whether to allow stale WAL entries read during replay.
 allow_stale_entries = false
 

config/standalone.example.toml

@@ -565,6 +565,9 @@ sst_write_buffer_size = "8MB"
 ## Capacity of the channel to send data from parallel scan tasks to the main task.
 parallel_scan_channel_size = 32
 
+## Maximum number of SST files to scan concurrently.
+max_concurrent_scan_files = 128
+
 ## Whether to allow stale WAL entries read during replay.
 allow_stale_entries = false
 

src/api/src/region.rs

@@ -22,6 +22,7 @@ use greptime_proto::v1::region::RegionResponse as RegionResponseV1;
 pub struct RegionResponse {
     pub affected_rows: AffectedRows,
     pub extensions: HashMap<String, Vec<u8>>,
+    pub metadata: Vec<u8>,
 }
 
 impl RegionResponse {
@@ -29,6 +30,7 @@ impl RegionResponse {
         Self {
             affected_rows: region_response.affected_rows as _,
             extensions: region_response.extensions,
+            metadata: region_response.metadata,
         }
     }
 
@@ -37,6 +39,16 @@ impl RegionResponse {
         Self {
             affected_rows,
             extensions: Default::default(),
+            metadata: Vec::new(),
+        }
+    }
+
+    /// Creates one response with metadata.
+    pub fn from_metadata(metadata: Vec<u8>) -> Self {
+        Self {
+            affected_rows: 0,
+            extensions: Default::default(),
+            metadata,
         }
     }
 }

src/api/src/v1/column_def.rs

@@ -24,7 +24,7 @@ use greptime_proto::v1::{
 };
 use snafu::ResultExt;
 
-use crate::error::{self, Result};
+use crate::error::{self, ConvertColumnDefaultConstraintSnafu, Result};
 use crate::helper::ColumnDataTypeWrapper;
 use crate::v1::{ColumnDef, ColumnOptions, SemanticType};
 
@@ -77,6 +77,48 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
         })
 }
 
+/// Tries to construct a `ColumnDef` from the given `ColumnSchema`.
+///
+/// TODO(weny): Add tests for this function.
+pub fn try_as_column_def(column_schema: &ColumnSchema, is_primary_key: bool) -> Result<ColumnDef> {
+    let column_datatype =
+        ColumnDataTypeWrapper::try_from(column_schema.data_type.clone()).map(|w| w.to_parts())?;
+
+    let semantic_type = if column_schema.is_time_index() {
+        SemanticType::Timestamp
+    } else if is_primary_key {
+        SemanticType::Tag
+    } else {
+        SemanticType::Field
+    } as i32;
+    let comment = column_schema
+        .metadata()
+        .get(COMMENT_KEY)
+        .cloned()
+        .unwrap_or_default();
+
+    let default_constraint = match column_schema.default_constraint() {
+        None => vec![],
+        Some(v) => v
+            .clone()
+            .try_into()
+            .context(ConvertColumnDefaultConstraintSnafu {
+                column: &column_schema.name,
+            })?,
+    };
+    let options = options_from_column_schema(column_schema);
+    Ok(ColumnDef {
+        name: column_schema.name.clone(),
+        data_type: column_datatype.0 as i32,
+        is_nullable: column_schema.is_nullable(),
+        default_constraint,
+        semantic_type,
+        comment,
+        datatype_extension: column_datatype.1,
+        options,
+    })
+}
+
 /// Constructs a `ColumnOptions` from the given `ColumnSchema`.
 pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option<ColumnOptions> {
     let mut options = ColumnOptions::default();
@@ -226,18 +268,20 @@ mod tests {
         assert!(options.is_none());
 
         let mut schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true)
-            .with_fulltext_options(FulltextOptions {
+            .with_fulltext_options(FulltextOptions::new_unchecked(
-                enable: true,
+                true,
-                analyzer: FulltextAnalyzer::English,
+                FulltextAnalyzer::English,
-                case_sensitive: false,
+                false,
-                backend: FulltextBackend::Bloom,
+                FulltextBackend::Bloom,
-            })
+                10240,
+                0.01,
+            ))
             .unwrap();
         schema.set_inverted_index(true);
         let options = options_from_column_schema(&schema).unwrap();
         assert_eq!(
             options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
-            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}"
+            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
         );
         assert_eq!(
             options.options.get(INVERTED_INDEX_GRPC_KEY).unwrap(),
@@ -247,16 +291,18 @@ mod tests {
 
     #[test]
     fn test_options_with_fulltext() {
-        let fulltext = FulltextOptions {
+        let fulltext = FulltextOptions::new_unchecked(
-            enable: true,
+            true,
-            analyzer: FulltextAnalyzer::English,
+            FulltextAnalyzer::English,
-            case_sensitive: false,
+            false,
-            backend: FulltextBackend::Bloom,
+            FulltextBackend::Bloom,
-        };
+            10240,
+            0.01,
+        );
         let options = options_from_fulltext(&fulltext).unwrap().unwrap();
         assert_eq!(
             options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
-            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}"
+            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
         );
     }
 

src/catalog/Cargo.toml

@@ -43,6 +43,8 @@ moka = { workspace = true, features = ["future", "sync"] }
 partition.workspace = true
 paste.workspace = true
 prometheus.workspace = true
+promql-parser.workspace = true
+rand.workspace = true
 rustc-hash.workspace = true
 serde_json.workspace = true
 session.workspace = true

src/catalog/src/information_extension.rs

@@ -16,8 +16,8 @@ use api::v1::meta::ProcedureStatus;
 use common_error::ext::BoxedError;
 use common_meta::cluster::{ClusterInfo, NodeInfo};
 use common_meta::datanode::RegionStat;
-use common_meta::ddl::{ExecutorContext, ProcedureExecutor};
 use common_meta::key::flow::flow_state::FlowStat;
+use common_meta::procedure_executor::{ExecutorContext, ProcedureExecutor};
 use common_meta::rpc::procedure;
 use common_procedure::{ProcedureInfo, ProcedureState};
 use meta_client::MetaClientRef;

src/catalog/src/kvbackend/manager.rs

@@ -28,7 +28,7 @@ use common_meta::cache::{
 use common_meta::key::catalog_name::CatalogNameKey;
 use common_meta::key::flow::FlowMetadataManager;
 use common_meta::key::schema_name::SchemaNameKey;
-use common_meta::key::table_info::TableInfoValue;
+use common_meta::key::table_info::{TableInfoManager, TableInfoValue};
 use common_meta::key::table_name::TableNameKey;
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
@@ -39,6 +39,7 @@ use moka::sync::Cache;
 use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
 use session::context::{Channel, QueryContext};
 use snafu::prelude::*;
+use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
 use table::dist_table::DistTable;
 use table::metadata::TableId;
 use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
@@ -142,6 +143,61 @@ impl KvBackendCatalogManager {
     pub fn procedure_manager(&self) -> Option<ProcedureManagerRef> {
         self.procedure_manager.clone()
     }
+
+    // Override logical table's partition key indices with physical table's.
+    async fn override_logical_table_partition_key_indices(
+        table_route_cache: &TableRouteCacheRef,
+        table_info_manager: &TableInfoManager,
+        table: TableRef,
+    ) -> Result<TableRef> {
+        // If the table is not a metric table, return the table directly.
+        if table.table_info().meta.engine != METRIC_ENGINE_NAME {
+            return Ok(table);
+        }
+
+        if let Some(table_route_value) = table_route_cache
+            .get(table.table_info().table_id())
+            .await
+            .context(TableMetadataManagerSnafu)?
+            && let TableRoute::Logical(logical_route) = &*table_route_value
+            && let Some(physical_table_info_value) = table_info_manager
+                .get(logical_route.physical_table_id())
+                .await
+                .context(TableMetadataManagerSnafu)?
+        {
+            let mut new_table_info = (*table.table_info()).clone();
+
+            // Remap partition key indices from physical table to logical table
+            new_table_info.meta.partition_key_indices = physical_table_info_value
+                .table_info
+                .meta
+                .partition_key_indices
+                .iter()
+                .filter_map(|&physical_index| {
+                    // Get the column name from the physical table using the physical index
+                    physical_table_info_value
+                        .table_info
+                        .meta
+                        .schema
+                        .column_schemas
+                        .get(physical_index)
+                        .and_then(|physical_column| {
+                            // Find the corresponding index in the logical table schema
+                            new_table_info
+                                .meta
+                                .schema
+                                .column_index_by_name(physical_column.name.as_str())
+                        })
+                })
+                .collect();
+
+            let new_table = DistTable::table(Arc::new(new_table_info));
+
+            return Ok(new_table);
+        }
+
+        Ok(table)
+    }
 }
 
 #[async_trait::async_trait]
@@ -268,10 +324,7 @@ impl CatalogManager for KvBackendCatalogManager {
         let table_cache: TableCacheRef = self.cache_registry.get().context(CacheNotFoundSnafu {
             name: "table_cache",
         })?;
-        let table_route_cache: TableRouteCacheRef =
+
-            self.cache_registry.get().context(CacheNotFoundSnafu {
-                name: "table_route_cache",
-            })?;
         let table = table_cache
             .get_by_ref(&TableName {
                 catalog_name: catalog_name.to_string(),
@@ -281,55 +334,18 @@ impl CatalogManager for KvBackendCatalogManager {
             .await
             .context(GetTableCacheSnafu)?;
 
-        // Override logical table's partition key indices with physical table's.
+        if let Some(table) = table {
-        if let Some(table) = &table
+            let table_route_cache: TableRouteCacheRef =
-            && let Some(table_route_value) = table_route_cache
+                self.cache_registry.get().context(CacheNotFoundSnafu {
-                .get(table.table_info().table_id())
+                    name: "table_route_cache",
+                })?;
+            return Self::override_logical_table_partition_key_indices(
+                &table_route_cache,
+                self.table_metadata_manager.table_info_manager(),
+                table,
+            )
             .await
-                .context(TableMetadataManagerSnafu)?
+            .map(Some);
-            && let TableRoute::Logical(logical_route) = &*table_route_value
-            && let Some(physical_table_info_value) = self
-                .table_metadata_manager
-                .table_info_manager()
-                .get(logical_route.physical_table_id())
-                .await
-                .context(TableMetadataManagerSnafu)?
-        {
-            let mut new_table_info = (*table.table_info()).clone();
-            // Gather all column names from the logical table
-            let logical_column_names: std::collections::HashSet<_> = new_table_info
-                .meta
-                .schema
-                .column_schemas()
-                .iter()
-                .map(|col| &col.name)
-                .collect();
-
-            // Only preserve partition key indices where the corresponding columns exist in logical table
-            new_table_info.meta.partition_key_indices = physical_table_info_value
-                .table_info
-                .meta
-                .partition_key_indices
-                .iter()
-                .filter(|&&index| {
-                    if let Some(physical_column) = physical_table_info_value
-                        .table_info
-                        .meta
-                        .schema
-                        .column_schemas
-                        .get(index)
-                    {
-                        logical_column_names.contains(&physical_column.name)
-                    } else {
-                        false
-                    }
-                })
-                .cloned()
-                .collect();
-
-            let new_table = DistTable::table(Arc::new(new_table_info));
-
-            return Ok(Some(new_table));
         }
 
         if channel == Channel::Postgres {
@@ -342,7 +358,7 @@ impl CatalogManager for KvBackendCatalogManager {
             }
         }
 
-        Ok(table)
+        Ok(None)
     }
 
     async fn tables_by_ids(
@@ -394,8 +410,20 @@ impl CatalogManager for KvBackendCatalogManager {
         let catalog = catalog.to_string();
         let schema = schema.to_string();
         let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
+        let table_route_cache: Result<TableRouteCacheRef> =
+            self.cache_registry.get().context(CacheNotFoundSnafu {
+                name: "table_route_cache",
+            });
 
         common_runtime::spawn_global(async move {
+            let table_route_cache = match table_route_cache {
+                Ok(table_route_cache) => table_route_cache,
+                Err(e) => {
+                    let _ = tx.send(Err(e)).await;
+                    return;
+                }
+            };
+
             let table_id_stream = metadata_manager
                 .table_name_manager()
                 .tables(&catalog, &schema)
@@ -422,6 +450,7 @@ impl CatalogManager for KvBackendCatalogManager {
                 let metadata_manager = metadata_manager.clone();
                 let tx = tx.clone();
                 let semaphore = semaphore.clone();
+                let table_route_cache = table_route_cache.clone();
                 common_runtime::spawn_global(async move {
                     // we don't explicitly close the semaphore so just ignore the potential error.
                     let _ = semaphore.acquire().await;
@@ -439,6 +468,16 @@ impl CatalogManager for KvBackendCatalogManager {
                     };
 
                     for table in table_info_values.into_values().map(build_table) {
+                        let table = if let Ok(table) = table {
+                            Self::override_logical_table_partition_key_indices(
+                                &table_route_cache,
+                                metadata_manager.table_info_manager(),
+                                table,
+                            )
+                            .await
+                        } else {
+                            table
+                        };
                         if tx.send(table).await.is_err() {
                             return;
                         }

src/catalog/src/process_manager.rs

@@ -14,17 +14,24 @@
 
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
-use std::fmt::{Debug, Formatter};
+use std::fmt::{Debug, Display, Formatter};
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::{Arc, RwLock};
+use std::time::{Duration, Instant, UNIX_EPOCH};
 
 use api::v1::frontend::{KillProcessRequest, ListProcessRequest, ProcessInfo};
 use common_base::cancellation::CancellationHandle;
 use common_frontend::selector::{FrontendSelector, MetaClientSelector};
-use common_telemetry::{debug, info};
+use common_frontend::slow_query_event::SlowQueryEvent;
+use common_telemetry::{debug, error, info, warn};
 use common_time::util::current_time_millis;
 use meta_client::MetaClientRef;
+use promql_parser::parser::EvalStmt;
+use rand::random;
+use session::context::QueryContextRef;
 use snafu::{ensure, OptionExt, ResultExt};
+use sql::statements::statement::Statement;
+use tokio::sync::mpsc::Sender;
 
 use crate::error;
 use crate::metrics::{PROCESS_KILL_COUNT, PROCESS_LIST_COUNT};
@@ -44,6 +51,23 @@ pub struct ProcessManager {
     frontend_selector: Option<MetaClientSelector>,
 }
 
+/// Represents a parsed query statement, functionally equivalent to [query::parser::QueryStatement].
+/// This enum is defined here to avoid cyclic dependencies with the query parser module.
+#[derive(Debug, Clone)]
+pub enum QueryStatement {
+    Sql(Statement),
+    Promql(EvalStmt),
+}
+
+impl Display for QueryStatement {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            QueryStatement::Sql(stmt) => write!(f, "{}", stmt),
+            QueryStatement::Promql(eval_stmt) => write!(f, "{}", eval_stmt),
+        }
+    }
+}
+
 impl ProcessManager {
     /// Create a [ProcessManager] instance with server address and kv client.
     pub fn new(server_addr: String, meta_client: Option<MetaClientRef>) -> Self {
@@ -67,6 +91,7 @@ impl ProcessManager {
         query: String,
         client: String,
         query_id: Option<ProcessId>,
+        _slow_query_timer: Option<SlowQueryTimer>,
     ) -> Ticket {
         let id = query_id.unwrap_or_else(|| self.next_id.fetch_add(1, Ordering::Relaxed));
         let process = ProcessInfo {
@@ -93,6 +118,7 @@ impl ProcessManager {
             manager: self.clone(),
             id,
             cancellation_handle,
+            _slow_query_timer,
         }
     }
 
@@ -141,14 +167,20 @@ impl ProcessManager {
                 .await
                 .context(error::InvokeFrontendSnafu)?;
             for mut f in frontends {
-                processes.extend(
+                let result = f
-                    f.list_process(ListProcessRequest {
+                    .list_process(ListProcessRequest {
                         catalog: catalog.unwrap_or_default().to_string(),
                     })
                     .await
-                    .context(error::InvokeFrontendSnafu)?
+                    .context(error::InvokeFrontendSnafu);
-                    .processes,
+                match result {
-                );
+                    Ok(resp) => {
+                        processes.extend(resp.processes);
+                    }
+                    Err(e) => {
+                        warn!(e; "Skipping failing node: {:?}", f)
+                    }
+                }
             }
         }
         processes.extend(self.local_processes(catalog)?);
@@ -217,6 +249,7 @@ pub struct Ticket {
     pub(crate) manager: ProcessManagerRef,
     pub(crate) id: ProcessId,
     pub cancellation_handle: Arc<CancellationHandle>,
+    _slow_query_timer: Option<SlowQueryTimer>,
 }
 
 impl Drop for Ticket {
@@ -257,6 +290,107 @@ impl Debug for CancellableProcess {
     }
 }
 
+/// SlowQueryTimer is used to log slow query when it's dropped.
+/// In drop(), it will check if the query is slow and send the slow query event to the handler.
+pub struct SlowQueryTimer {
+    start: Instant,
+    stmt: QueryStatement,
+    query_ctx: QueryContextRef,
+    threshold: Option<Duration>,
+    sample_ratio: Option<f64>,
+    tx: Sender<SlowQueryEvent>,
+}
+
+impl SlowQueryTimer {
+    pub fn new(
+        stmt: QueryStatement,
+        query_ctx: QueryContextRef,
+        threshold: Option<Duration>,
+        sample_ratio: Option<f64>,
+        tx: Sender<SlowQueryEvent>,
+    ) -> Self {
+        Self {
+            start: Instant::now(),
+            stmt,
+            query_ctx,
+            threshold,
+            sample_ratio,
+            tx,
+        }
+    }
+}
+
+impl SlowQueryTimer {
+    fn send_slow_query_event(&self, elapsed: Duration, threshold: Duration) {
+        let mut slow_query_event = SlowQueryEvent {
+            cost: elapsed.as_millis() as u64,
+            threshold: threshold.as_millis() as u64,
+            query: "".to_string(),
+            query_ctx: self.query_ctx.clone(),
+
+            // The following fields are only used for PromQL queries.
+            is_promql: false,
+            promql_range: None,
+            promql_step: None,
+            promql_start: None,
+            promql_end: None,
+        };
+
+        match &self.stmt {
+            QueryStatement::Promql(stmt) => {
+                slow_query_event.is_promql = true;
+                slow_query_event.query = stmt.expr.to_string();
+                slow_query_event.promql_step = Some(stmt.interval.as_millis() as u64);
+
+                let start = stmt
+                    .start
+                    .duration_since(UNIX_EPOCH)
+                    .unwrap_or_default()
+                    .as_millis() as i64;
+
+                let end = stmt
+                    .end
+                    .duration_since(UNIX_EPOCH)
+                    .unwrap_or_default()
+                    .as_millis() as i64;
+
+                slow_query_event.promql_range = Some((end - start) as u64);
+                slow_query_event.promql_start = Some(start);
+                slow_query_event.promql_end = Some(end);
+            }
+            QueryStatement::Sql(stmt) => {
+                slow_query_event.query = stmt.to_string();
+            }
+        }
+
+        // Send SlowQueryEvent to the handler.
+        if let Err(e) = self.tx.try_send(slow_query_event) {
+            error!(e; "Failed to send slow query event");
+        }
+    }
+}
+
+impl Drop for SlowQueryTimer {
+    fn drop(&mut self) {
+        if let Some(threshold) = self.threshold {
+            // Calculate the elaspsed duration since the timer is created.
+            let elapsed = self.start.elapsed();
+            if elapsed > threshold {
+                if let Some(ratio) = self.sample_ratio {
+                    // Only capture a portion of slow queries based on sample_ratio.
+                    // Generate a random number in [0, 1) and compare it with sample_ratio.
+                    if ratio >= 1.0 || random::<f64>() <= ratio {
+                        self.send_slow_query_event(elapsed, threshold);
+                    }
+                } else {
+                    // Captures all slow queries if sample_ratio is not set.
+                    self.send_slow_query_event(elapsed, threshold);
+                }
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
@@ -272,6 +406,7 @@ mod tests {
             "SELECT * FROM table".to_string(),
             "".to_string(),
             None,
+            None,
         );
 
         let running_processes = process_manager.local_processes(None).unwrap();
@@ -295,6 +430,7 @@ mod tests {
             "SELECT * FROM table".to_string(),
             "client1".to_string(),
             Some(custom_id),
+            None,
         );
 
         assert_eq!(ticket.id, custom_id);
@@ -315,6 +451,7 @@ mod tests {
             "SELECT * FROM table1".to_string(),
             "client1".to_string(),
             None,
+            None,
         );
 
         let ticket2 = process_manager.clone().register_query(
@@ -323,6 +460,7 @@ mod tests {
             "SELECT * FROM table2".to_string(),
             "client2".to_string(),
             None,
+            None,
         );
 
         let running_processes = process_manager.local_processes(Some("public")).unwrap();
@@ -344,6 +482,7 @@ mod tests {
             "SELECT * FROM table1".to_string(),
             "client1".to_string(),
             None,
+            None,
         );
 
         let _ticket2 = process_manager.clone().register_query(
@@ -352,6 +491,7 @@ mod tests {
             "SELECT * FROM table2".to_string(),
             "client2".to_string(),
             None,
+            None,
         );
 
         // Test listing processes for specific catalog
@@ -378,6 +518,7 @@ mod tests {
             "SELECT * FROM table".to_string(),
             "client1".to_string(),
             None,
+            None,
         );
         assert_eq!(process_manager.local_processes(None).unwrap().len(), 1);
         process_manager.deregister_query("public".to_string(), ticket.id);
@@ -394,6 +535,7 @@ mod tests {
             "SELECT * FROM table".to_string(),
             "client1".to_string(),
             None,
+            None,
         );
 
         assert!(!ticket.cancellation_handle.is_cancelled());
@@ -411,6 +553,7 @@ mod tests {
             "SELECT * FROM table".to_string(),
             "client1".to_string(),
             None,
+            None,
         );
         assert!(!ticket.cancellation_handle.is_cancelled());
         let killed = process_manager
@@ -456,6 +599,7 @@ mod tests {
             "SELECT COUNT(*) FROM users WHERE age > 18".to_string(),
             "test_client".to_string(),
             Some(42),
+            None,
         );
 
         let processes = process_manager.local_processes(None).unwrap();
@@ -482,6 +626,7 @@ mod tests {
                 "SELECT * FROM table".to_string(),
                 "client1".to_string(),
                 None,
+                None,
             );
 
             // Process should be registered

src/cli/src/bench.rs

@@ -160,6 +160,7 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
         options: Default::default(),
         region_numbers: (1..=100).collect(),
         partition_key_indices: vec![],
+        column_ids: vec![],
     };
 
     RawTableInfo {

src/cli/src/metadata/repair.rs

@@ -241,7 +241,6 @@ impl RepairTool {
             let alter_table_request = alter_table::make_alter_region_request_for_peer(
                 logical_table_id,
                 &alter_table_expr,
-                full_table_metadata.table_info.ident.version,
                 peer,
                 physical_region_routes,
             )?;

src/cli/src/metadata/repair/alter_table.rs

@@ -66,7 +66,6 @@ pub fn generate_alter_table_expr_for_all_columns(
 pub fn make_alter_region_request_for_peer(
     logical_table_id: TableId,
     alter_table_expr: &AlterTableExpr,
-    schema_version: u64,
     peer: &Peer,
     region_routes: &[RegionRoute],
 ) -> Result<RegionRequest> {
@@ -74,7 +73,7 @@ pub fn make_alter_region_request_for_peer(
     let mut requests = Vec::with_capacity(regions_on_this_peer.len());
     for region_number in &regions_on_this_peer {
         let region_id = RegionId::new(logical_table_id, *region_number);
-        let request = make_alter_region_request(region_id, alter_table_expr, schema_version);
+        let request = make_alter_region_request(region_id, alter_table_expr);
         requests.push(request);
     }
 

src/client/src/database.rs

@@ -211,12 +211,18 @@ impl Database {
                         retries += 1;
                         warn!("Retrying {} times with error = {:?}", retries, err);
                         continue;
+                    } else {
+                        error!(
+                            err; "Failed to send request to grpc handle, retries = {}, not retryable error, aborting",
+                            retries
+                        );
+                        return Err(err.into());
                     }
                 }
                 (Err(err), false) => {
                     error!(
-                        "Failed to send request to grpc handle after {} retries, error = {:?}",
+                        err; "Failed to send request to grpc handle after {} retries",
-                        retries, err
+                        retries,
                     );
                     return Err(err.into());
                 }

src/client/src/region.rs

@@ -163,19 +163,70 @@ impl RegionRequester {
             let _span = tracing_context.attach(common_telemetry::tracing::info_span!(
                 "poll_flight_data_stream"
             ));
-            while let Some(flight_message) = flight_message_stream.next().await {
+
-                let flight_message = flight_message
+            let mut buffered_message: Option<FlightMessage> = None;
-                    .map_err(BoxedError::new)
+            let mut stream_ended = false;
-                    .context(ExternalSnafu)?;
+
+            while !stream_ended {
+                // get the next message from the buffered message or read from the flight message stream
+                let flight_message_item = if let Some(msg) = buffered_message.take() {
+                    Some(Ok(msg))
+                } else {
+                    flight_message_stream.next().await
+                };
+
+                let flight_message = match flight_message_item {
+                    Some(Ok(message)) => message,
+                    Some(Err(e)) => {
+                        yield Err(BoxedError::new(e)).context(ExternalSnafu);
+                        break;
+                    }
+                    None => break,
+                };
 
                 match flight_message {
                     FlightMessage::RecordBatch(record_batch) => {
-                        yield RecordBatch::try_from_df_record_batch(
+                        let result_to_yield = RecordBatch::try_from_df_record_batch(
                             schema_cloned.clone(),
                             record_batch,
-                        )
+                        );
+
+                        // get the next message from the stream. normally it should be a metrics message.
+                        if let Some(next_flight_message_result) = flight_message_stream.next().await
+                        {
+                            match next_flight_message_result {
+                                Ok(FlightMessage::Metrics(s)) => {
+                                    let m = serde_json::from_str(&s).ok().map(Arc::new);
+                                    metrics_ref.swap(m);
+                                }
+                                Ok(FlightMessage::RecordBatch(rb)) => {
+                                    // for some reason it's not a metrics message, so we need to buffer this record batch
+                                    // and yield it in the next iteration.
+                                    buffered_message = Some(FlightMessage::RecordBatch(rb));
+                                }
+                                Ok(_) => {
+                                    yield IllegalFlightMessagesSnafu {
+                                        reason: "A RecordBatch message can only be succeeded by a Metrics message or another RecordBatch message"
+                                    }
+                                    .fail()
+                                    .map_err(BoxedError::new)
+                                    .context(ExternalSnafu);
+                                    break;
+                                }
+                                Err(e) => {
+                                    yield Err(BoxedError::new(e)).context(ExternalSnafu);
+                                    break;
+                                }
+                            }
+                        } else {
+                            // the stream has ended
+                            stream_ended = true;
+                        }
+
+                        yield result_to_yield;
                     }
                     FlightMessage::Metrics(s) => {
+                        // just a branch in case of some metrics message comes after other things.
                         let m = serde_json::from_str(&s).ok().map(Arc::new);
                         metrics_ref.swap(m);
                         break;

src/cmd/src/bin/greptime.rs

@@ -20,11 +20,11 @@ use cmd::error::{InitTlsProviderSnafu, Result};
 use cmd::options::GlobalOptions;
 use cmd::{cli, datanode, flownode, frontend, metasrv, standalone, App};
 use common_base::Plugins;
-use common_version::version;
+use common_version::{verbose_version, version};
 use servers::install_ring_crypto_provider;
 
 #[derive(Parser)]
-#[command(name = "greptime", author, version, long_version = version(), about)]
+#[command(name = "greptime", author, version, long_version = verbose_version(), about)]
 #[command(propagate_version = true)]
 pub(crate) struct Command {
     #[clap(subcommand)]
@@ -143,10 +143,8 @@ async fn start(cli: Command) -> Result<()> {
 }
 
 fn setup_human_panic() {
-    human_panic::setup_panic!(
+    human_panic::setup_panic!(human_panic::Metadata::new("GreptimeDB", version())
-        human_panic::Metadata::new("GreptimeDB", env!("CARGO_PKG_VERSION"))
+        .homepage("https://github.com/GreptimeTeam/greptimedb/discussions"));
-            .homepage("https://github.com/GreptimeTeam/greptimedb/discussions")
-    );
 
     common_telemetry::set_panic_hook();
 }

src/cmd/src/datanode/builder.rs

@@ -19,7 +19,7 @@ use catalog::kvbackend::MetaKvBackend;
 use common_base::Plugins;
 use common_meta::cache::LayeredCacheRegistryBuilder;
 use common_telemetry::info;
-use common_version::{short_version, version};
+use common_version::{short_version, verbose_version};
 use datanode::datanode::DatanodeBuilder;
 use datanode::service::DatanodeServiceBuilder;
 use meta_client::MetaClientType;
@@ -67,7 +67,7 @@ impl InstanceBuilder {
             None,
         );
 
-        log_versions(version(), short_version(), APP_NAME);
+        log_versions(verbose_version(), short_version(), APP_NAME);
         create_resource_limit_metrics(APP_NAME);
 
         plugins::setup_datanode_plugins(plugins, &opts.plugins, dn_opts)

src/cmd/src/flownode.rs

@@ -32,7 +32,7 @@ use common_meta::key::flow::FlowMetadataManager;
 use common_meta::key::TableMetadataManager;
 use common_telemetry::info;
 use common_telemetry::logging::{TracingOptions, DEFAULT_LOGGING_DIR};
-use common_version::{short_version, version};
+use common_version::{short_version, verbose_version};
 use flow::{
     get_flow_auth_options, FlownodeBuilder, FlownodeInstance, FlownodeServiceBuilder,
     FrontendClient, FrontendInvoker,
@@ -279,7 +279,7 @@ impl StartCommand {
             None,
         );
 
-        log_versions(version(), short_version(), APP_NAME);
+        log_versions(verbose_version(), short_version(), APP_NAME);
         create_resource_limit_metrics(APP_NAME);
 
         info!("Flownode start command: {:#?}", self);

src/cmd/src/frontend.rs

@@ -33,7 +33,7 @@ use common_meta::heartbeat::handler::HandlerGroupExecutor;
 use common_telemetry::info;
 use common_telemetry::logging::{TracingOptions, DEFAULT_LOGGING_DIR};
 use common_time::timezone::set_default_timezone;
-use common_version::{short_version, version};
+use common_version::{short_version, verbose_version};
 use frontend::frontend::Frontend;
 use frontend::heartbeat::HeartbeatTask;
 use frontend::instance::builder::FrontendBuilder;
@@ -282,7 +282,7 @@ impl StartCommand {
             opts.component.slow_query.as_ref(),
         );
 
-        log_versions(version(), short_version(), APP_NAME);
+        log_versions(verbose_version(), short_version(), APP_NAME);
         create_resource_limit_metrics(APP_NAME);
 
         info!("Frontend start command: {:#?}", self);

src/cmd/src/lib.rs

@@ -112,7 +112,7 @@ pub trait App: Send {
 pub fn log_versions(version: &str, short_version: &str, app: &str) {
     // Report app version as gauge.
     APP_VERSION
-        .with_label_values(&[env!("CARGO_PKG_VERSION"), short_version, app])
+        .with_label_values(&[common_version::version(), short_version, app])
         .inc();
 
     // Log version and argument flags.

src/cmd/src/metasrv.rs

@@ -22,7 +22,7 @@ use common_base::Plugins;
 use common_config::Configurable;
 use common_telemetry::info;
 use common_telemetry::logging::{TracingOptions, DEFAULT_LOGGING_DIR};
-use common_version::{short_version, version};
+use common_version::{short_version, verbose_version};
 use meta_srv::bootstrap::MetasrvInstance;
 use meta_srv::metasrv::BackendImpl;
 use snafu::ResultExt;
@@ -320,7 +320,7 @@ impl StartCommand {
             None,
         );
 
-        log_versions(version(), short_version(), APP_NAME);
+        log_versions(verbose_version(), short_version(), APP_NAME);
         create_resource_limit_metrics(APP_NAME);
 
         info!("Metasrv start command: {:#?}", self);
@@ -341,7 +341,7 @@ impl StartCommand {
             .context(error::BuildMetaServerSnafu)?;
         let metasrv = builder.build().await.context(error::BuildMetaServerSnafu)?;
 
-        let instance = MetasrvInstance::new(opts, plugins, metasrv)
+        let instance = MetasrvInstance::new(metasrv)
             .await
             .context(error::BuildMetaServerSnafu)?;
 

src/cmd/src/standalone.rs

@@ -30,21 +30,18 @@ use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
 use common_config::{metadata_store_dir, Configurable, KvBackendConfig};
 use common_error::ext::BoxedError;
 use common_meta::cache::LayeredCacheRegistryBuilder;
-use common_meta::cache_invalidator::CacheInvalidatorRef;
 use common_meta::cluster::{NodeInfo, NodeStatus};
 use common_meta::datanode::RegionStat;
-use common_meta::ddl::flow_meta::{FlowMetadataAllocator, FlowMetadataAllocatorRef};
+use common_meta::ddl::flow_meta::FlowMetadataAllocator;
-use common_meta::ddl::table_meta::{TableMetadataAllocator, TableMetadataAllocatorRef};
+use common_meta::ddl::table_meta::TableMetadataAllocator;
-use common_meta::ddl::{DdlContext, NoopRegionFailureDetectorControl, ProcedureExecutorRef};
+use common_meta::ddl::{DdlContext, NoopRegionFailureDetectorControl};
 use common_meta::ddl_manager::DdlManager;
-#[cfg(feature = "enterprise")]
-use common_meta::ddl_manager::TriggerDdlManagerRef;
 use common_meta::key::flow::flow_state::FlowStat;
-use common_meta::key::flow::{FlowMetadataManager, FlowMetadataManagerRef};
+use common_meta::key::flow::FlowMetadataManager;
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
-use common_meta::node_manager::NodeManagerRef;
 use common_meta::peer::Peer;
+use common_meta::procedure_executor::LocalProcedureExecutor;
 use common_meta::region_keeper::MemoryRegionKeeper;
 use common_meta::region_registry::LeaderRegionRegistry;
 use common_meta::sequence::SequenceBuilder;
@@ -55,7 +52,7 @@ use common_telemetry::logging::{
     LoggingOptions, SlowQueryOptions, TracingOptions, DEFAULT_LOGGING_DIR,
 };
 use common_time::timezone::set_default_timezone;
-use common_version::{short_version, version};
+use common_version::{short_version, verbose_version};
 use common_wal::config::DatanodeWalConfig;
 use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
@@ -470,7 +467,7 @@ impl StartCommand {
             opts.component.slow_query.as_ref(),
         );
 
-        log_versions(version(), short_version(), APP_NAME);
+        log_versions(verbose_version(), short_version(), APP_NAME);
         create_resource_limit_metrics(APP_NAME);
 
         info!("Standalone start command: {:#?}", self);
@@ -594,28 +591,39 @@ impl StartCommand {
             .await
             .context(error::BuildWalOptionsAllocatorSnafu)?;
         let wal_options_allocator = Arc::new(wal_options_allocator);
-        let table_meta_allocator = Arc::new(TableMetadataAllocator::new(
+        let table_metadata_allocator = Arc::new(TableMetadataAllocator::new(
             table_id_sequence,
             wal_options_allocator.clone(),
         ));
-        let flow_meta_allocator = Arc::new(FlowMetadataAllocator::with_noop_peer_allocator(
+        let flow_metadata_allocator = Arc::new(FlowMetadataAllocator::with_noop_peer_allocator(
             flow_id_sequence,
         ));
 
+        let ddl_context = DdlContext {
+            node_manager: node_manager.clone(),
+            cache_invalidator: layered_cache_registry.clone(),
+            memory_region_keeper: Arc::new(MemoryRegionKeeper::default()),
+            leader_region_registry: Arc::new(LeaderRegionRegistry::default()),
+            table_metadata_manager: table_metadata_manager.clone(),
+            table_metadata_allocator: table_metadata_allocator.clone(),
+            flow_metadata_manager: flow_metadata_manager.clone(),
+            flow_metadata_allocator: flow_metadata_allocator.clone(),
+            region_failure_detector_controller: Arc::new(NoopRegionFailureDetectorControl),
+        };
+
+        let ddl_manager = DdlManager::try_new(ddl_context, procedure_manager.clone(), true)
+            .context(error::InitDdlManagerSnafu)?;
         #[cfg(feature = "enterprise")]
-        let trigger_ddl_manager: Option<TriggerDdlManagerRef> = plugins.get();
+        let ddl_manager = {
-        let ddl_task_executor = Self::create_ddl_task_executor(
+            let trigger_ddl_manager: Option<common_meta::ddl_manager::TriggerDdlManagerRef> =
+                plugins.get();
+            ddl_manager.with_trigger_ddl_manager(trigger_ddl_manager)
+        };
+
+        let procedure_executor = Arc::new(LocalProcedureExecutor::new(
+            Arc::new(ddl_manager),
             procedure_manager.clone(),
-            node_manager.clone(),
+        ));
-            layered_cache_registry.clone(),
-            table_metadata_manager,
-            table_meta_allocator,
-            flow_metadata_manager,
-            flow_meta_allocator,
-            #[cfg(feature = "enterprise")]
-            trigger_ddl_manager,
-        )
-        .await?;
 
         let fe_instance = FrontendBuilder::new(
             fe_opts.clone(),
@@ -623,7 +631,7 @@ impl StartCommand {
             layered_cache_registry.clone(),
             catalog_manager.clone(),
             node_manager.clone(),
-            ddl_task_executor.clone(),
+            procedure_executor.clone(),
             process_manager,
         )
         .with_plugin(plugins.clone())
@@ -648,7 +656,7 @@ impl StartCommand {
             catalog_manager.clone(),
             kv_backend.clone(),
             layered_cache_registry.clone(),
-            ddl_task_executor.clone(),
+            procedure_executor,
             node_manager,
         )
         .await
@@ -679,41 +687,6 @@ impl StartCommand {
         })
     }
 
-    #[allow(clippy::too_many_arguments)]
-    pub async fn create_ddl_task_executor(
-        procedure_manager: ProcedureManagerRef,
-        node_manager: NodeManagerRef,
-        cache_invalidator: CacheInvalidatorRef,
-        table_metadata_manager: TableMetadataManagerRef,
-        table_metadata_allocator: TableMetadataAllocatorRef,
-        flow_metadata_manager: FlowMetadataManagerRef,
-        flow_metadata_allocator: FlowMetadataAllocatorRef,
-        #[cfg(feature = "enterprise")] trigger_ddl_manager: Option<TriggerDdlManagerRef>,
-    ) -> Result<ProcedureExecutorRef> {
-        let procedure_executor: ProcedureExecutorRef = Arc::new(
-            DdlManager::try_new(
-                DdlContext {
-                    node_manager,
-                    cache_invalidator,
-                    memory_region_keeper: Arc::new(MemoryRegionKeeper::default()),
-                    leader_region_registry: Arc::new(LeaderRegionRegistry::default()),
-                    table_metadata_manager,
-                    table_metadata_allocator,
-                    flow_metadata_manager,
-                    flow_metadata_allocator,
-                    region_failure_detector_controller: Arc::new(NoopRegionFailureDetectorControl),
-                },
-                procedure_manager,
-                true,
-                #[cfg(feature = "enterprise")]
-                trigger_ddl_manager,
-            )
-            .context(error::InitDdlManagerSnafu)?,
-        );
-
-        Ok(procedure_executor)
-    }
-
     pub async fn create_table_metadata_manager(
         kv_backend: KvBackendRef,
     ) -> Result<TableMetadataManagerRef> {

src/common/frontend/Cargo.toml

@@ -12,6 +12,7 @@ common-macro.workspace = true
 common-meta.workspace = true
 greptime-proto.workspace = true
 meta-client.workspace = true
+session.workspace = true
 snafu.workspace = true
 tonic.workspace = true
 

src/common/frontend/src/lib.rs

@@ -19,6 +19,7 @@ use snafu::OptionExt;
 
 pub mod error;
 pub mod selector;
+pub mod slow_query_event;
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub struct DisplayProcessId {

src/common/frontend/src/selector.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::Debug;
 use std::time::Duration;
 
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
@@ -30,7 +31,7 @@ use crate::error::{MetaSnafu, Result};
 pub type FrontendClientPtr = Box<dyn FrontendClient>;
 
 #[async_trait::async_trait]
-pub trait FrontendClient: Send {
+pub trait FrontendClient: Send + Debug {
     async fn list_process(&mut self, req: ListProcessRequest) -> Result<ListProcessResponse>;
 
     async fn kill_process(&mut self, req: KillProcessRequest) -> Result<KillProcessResponse>;

src/common/frontend/src/slow_query_event.rs

@@ -0,0 +1,28 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use session::context::QueryContextRef;
+
+#[derive(Debug)]
+pub struct SlowQueryEvent {
+    pub cost: u64,
+    pub threshold: u64,
+    pub query: String,
+    pub is_promql: bool,
+    pub query_ctx: QueryContextRef,
+    pub promql_range: Option<u64>,
+    pub promql_step: Option<u64>,
+    pub promql_start: Option<i64>,
+    pub promql_end: Option<i64>,
+}

src/common/function/src/admin.rs

@@ -16,6 +16,9 @@ mod add_region_follower;
 mod flush_compact_region;
 mod flush_compact_table;
 mod migrate_region;
+mod reconcile_catalog;
+mod reconcile_database;
+mod reconcile_table;
 mod remove_region_follower;
 
 use std::sync::Arc;
@@ -24,6 +27,9 @@ use add_region_follower::AddRegionFollowerFunction;
 use flush_compact_region::{CompactRegionFunction, FlushRegionFunction};
 use flush_compact_table::{CompactTableFunction, FlushTableFunction};
 use migrate_region::MigrateRegionFunction;
+use reconcile_catalog::ReconcileCatalogFunction;
+use reconcile_database::ReconcileDatabaseFunction;
+use reconcile_table::ReconcileTableFunction;
 use remove_region_follower::RemoveRegionFollowerFunction;
 
 use crate::flush_flow::FlushFlowFunction;
@@ -43,5 +49,8 @@ impl AdminFunction {
         registry.register_async(Arc::new(FlushTableFunction));
         registry.register_async(Arc::new(CompactTableFunction));
         registry.register_async(Arc::new(FlushFlowFunction));
+        registry.register_async(Arc::new(ReconcileCatalogFunction));
+        registry.register_async(Arc::new(ReconcileDatabaseFunction));
+        registry.register_async(Arc::new(ReconcileTableFunction));
     }
 }

src/common/function/src/admin/reconcile_catalog.rs

@@ -0,0 +1,179 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::meta::reconcile_request::Target;
+use api::v1::meta::{ReconcileCatalog, ReconcileRequest};
+use common_macro::admin_fn;
+use common_query::error::{
+    InvalidFuncArgsSnafu, MissingProcedureServiceHandlerSnafu, Result,
+    UnsupportedInputDataTypeSnafu,
+};
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use common_telemetry::info;
+use datatypes::prelude::*;
+use session::context::QueryContextRef;
+
+use crate::handlers::ProcedureServiceHandlerRef;
+use crate::helper::{
+    cast_u32, default_parallelism, default_resolve_strategy, get_string_from_params,
+    parse_resolve_strategy,
+};
+
+const FN_NAME: &str = "reconcile_catalog";
+
+/// A function to reconcile a catalog.
+/// Returns the procedure id if success.
+///
+/// - `reconcile_catalog(resolve_strategy)`.
+/// - `reconcile_catalog(resolve_strategy, parallelism)`.
+///
+/// - `reconcile_catalog()`.
+#[admin_fn(
+    name = ReconcileCatalogFunction,
+    display_name = reconcile_catalog,
+    sig_fn = signature,
+    ret = string
+)]
+pub(crate) async fn reconcile_catalog(
+    procedure_service_handler: &ProcedureServiceHandlerRef,
+    query_ctx: &QueryContextRef,
+    params: &[ValueRef<'_>],
+) -> Result<Value> {
+    let (resolve_strategy, parallelism) = match params.len() {
+        0 => (default_resolve_strategy(), default_parallelism()),
+        1 => (
+            parse_resolve_strategy(get_string_from_params(params, 0, FN_NAME)?)?,
+            default_parallelism(),
+        ),
+        2 => {
+            let Some(parallelism) = cast_u32(&params[1])? else {
+                return UnsupportedInputDataTypeSnafu {
+                    function: FN_NAME,
+                    datatypes: params.iter().map(|v| v.data_type()).collect::<Vec<_>>(),
+                }
+                .fail();
+            };
+            (
+                parse_resolve_strategy(get_string_from_params(params, 0, FN_NAME)?)?,
+                parallelism,
+            )
+        }
+        size => {
+            return InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 0, 1 or 2, have: {}",
+                    size
+                ),
+            }
+            .fail();
+        }
+    };
+    info!(
+        "Reconciling catalog with resolve_strategy: {:?}, parallelism: {}",
+        resolve_strategy, parallelism
+    );
+    let pid = procedure_service_handler
+        .reconcile(ReconcileRequest {
+            target: Some(Target::ReconcileCatalog(ReconcileCatalog {
+                catalog_name: query_ctx.current_catalog().to_string(),
+                parallelism,
+                resolve_strategy: resolve_strategy as i32,
+            })),
+            ..Default::default()
+        })
+        .await?;
+    match pid {
+        Some(pid) => Ok(Value::from(pid)),
+        None => Ok(Value::Null),
+    }
+}
+
+fn signature() -> Signature {
+    let nums = ConcreteDataType::numerics();
+    let mut signs = Vec::with_capacity(2 + nums.len());
+    signs.extend([
+        // reconcile_catalog()
+        TypeSignature::NullAry,
+        // reconcile_catalog(resolve_strategy)
+        TypeSignature::Exact(vec![ConcreteDataType::string_datatype()]),
+    ]);
+    for sign in nums {
+        // reconcile_catalog(resolve_strategy, parallelism)
+        signs.push(TypeSignature::Exact(vec![
+            ConcreteDataType::string_datatype(),
+            sign,
+        ]));
+    }
+    Signature::one_of(signs, Volatility::Immutable)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+    use std::sync::Arc;
+
+    use common_query::error::Error;
+    use datatypes::vectors::{StringVector, UInt64Vector, VectorRef};
+
+    use crate::admin::reconcile_catalog::ReconcileCatalogFunction;
+    use crate::function::{AsyncFunction, FunctionContext};
+
+    #[tokio::test]
+    async fn test_reconcile_catalog() {
+        common_telemetry::init_default_ut_logging();
+
+        // reconcile_catalog()
+        let f = ReconcileCatalogFunction;
+        let args = vec![];
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
+        assert_eq!(expect, result);
+
+        // reconcile_catalog(resolve_strategy)
+        let f = ReconcileCatalogFunction;
+        let args = vec![Arc::new(StringVector::from(vec!["UseMetasrv"])) as _];
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
+        assert_eq!(expect, result);
+
+        // reconcile_catalog(resolve_strategy, parallelism)
+        let f = ReconcileCatalogFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["UseLatest"])) as _,
+            Arc::new(UInt64Vector::from_slice([10])) as _,
+        ];
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
+        assert_eq!(expect, result);
+
+        // unsupported input data type
+        let f = ReconcileCatalogFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["UseLatest"])) as _,
+            Arc::new(StringVector::from(vec!["test"])) as _,
+        ];
+        let err = f.eval(FunctionContext::mock(), &args).await.unwrap_err();
+        assert_matches!(err, Error::UnsupportedInputDataType { .. });
+
+        // invalid function args
+        let f = ReconcileCatalogFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["UseLatest"])) as _,
+            Arc::new(UInt64Vector::from_slice([10])) as _,
+            Arc::new(StringVector::from(vec!["10"])) as _,
+        ];
+        let err = f.eval(FunctionContext::mock(), &args).await.unwrap_err();
+        assert_matches!(err, Error::InvalidFuncArgs { .. });
+    }
+}

src/common/function/src/admin/reconcile_database.rs

@@ -0,0 +1,198 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::meta::reconcile_request::Target;
+use api::v1::meta::{ReconcileDatabase, ReconcileRequest};
+use common_macro::admin_fn;
+use common_query::error::{
+    InvalidFuncArgsSnafu, MissingProcedureServiceHandlerSnafu, Result,
+    UnsupportedInputDataTypeSnafu,
+};
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use common_telemetry::info;
+use datatypes::prelude::*;
+use session::context::QueryContextRef;
+
+use crate::handlers::ProcedureServiceHandlerRef;
+use crate::helper::{
+    cast_u32, default_parallelism, default_resolve_strategy, get_string_from_params,
+    parse_resolve_strategy,
+};
+
+const FN_NAME: &str = "reconcile_database";
+
+/// A function to reconcile a database.
+/// Returns the procedure id if success.
+///
+/// - `reconcile_database(database_name)`.
+/// - `reconcile_database(database_name, resolve_strategy)`.
+/// - `reconcile_database(database_name, resolve_strategy, parallelism)`.
+///
+/// The parameters:
+/// - `database_name`:  the database name
+#[admin_fn(
+    name = ReconcileDatabaseFunction,
+    display_name = reconcile_database,
+    sig_fn = signature,
+    ret = string
+)]
+pub(crate) async fn reconcile_database(
+    procedure_service_handler: &ProcedureServiceHandlerRef,
+    query_ctx: &QueryContextRef,
+    params: &[ValueRef<'_>],
+) -> Result<Value> {
+    let (database_name, resolve_strategy, parallelism) = match params.len() {
+        1 => (
+            get_string_from_params(params, 0, FN_NAME)?,
+            default_resolve_strategy(),
+            default_parallelism(),
+        ),
+        2 => (
+            get_string_from_params(params, 0, FN_NAME)?,
+            parse_resolve_strategy(get_string_from_params(params, 1, FN_NAME)?)?,
+            default_parallelism(),
+        ),
+        3 => {
+            let Some(parallelism) = cast_u32(&params[2])? else {
+                return UnsupportedInputDataTypeSnafu {
+                    function: FN_NAME,
+                    datatypes: params.iter().map(|v| v.data_type()).collect::<Vec<_>>(),
+                }
+                .fail();
+            };
+            (
+                get_string_from_params(params, 0, FN_NAME)?,
+                parse_resolve_strategy(get_string_from_params(params, 1, FN_NAME)?)?,
+                parallelism,
+            )
+        }
+        size => {
+            return InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 1, 2 or 3, have: {}",
+                    size
+                ),
+            }
+            .fail();
+        }
+    };
+    info!(
+        "Reconciling database: {}, resolve_strategy: {:?}, parallelism: {}",
+        database_name, resolve_strategy, parallelism
+    );
+    let pid = procedure_service_handler
+        .reconcile(ReconcileRequest {
+            target: Some(Target::ReconcileDatabase(ReconcileDatabase {
+                catalog_name: query_ctx.current_catalog().to_string(),
+                database_name: database_name.to_string(),
+                parallelism,
+                resolve_strategy: resolve_strategy as i32,
+            })),
+            ..Default::default()
+        })
+        .await?;
+    match pid {
+        Some(pid) => Ok(Value::from(pid)),
+        None => Ok(Value::Null),
+    }
+}
+
+fn signature() -> Signature {
+    let nums = ConcreteDataType::numerics();
+    let mut signs = Vec::with_capacity(2 + nums.len());
+    signs.extend([
+        // reconcile_database(datanode_name)
+        TypeSignature::Exact(vec![ConcreteDataType::string_datatype()]),
+        // reconcile_database(database_name, resolve_strategy)
+        TypeSignature::Exact(vec![
+            ConcreteDataType::string_datatype(),
+            ConcreteDataType::string_datatype(),
+        ]),
+    ]);
+    for sign in nums {
+        // reconcile_database(database_name, resolve_strategy, parallelism)
+        signs.push(TypeSignature::Exact(vec![
+            ConcreteDataType::string_datatype(),
+            ConcreteDataType::string_datatype(),
+            sign,
+        ]));
+    }
+    Signature::one_of(signs, Volatility::Immutable)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+    use std::sync::Arc;
+
+    use common_query::error::Error;
+    use datatypes::vectors::{StringVector, UInt32Vector, VectorRef};
+
+    use crate::admin::reconcile_database::ReconcileDatabaseFunction;
+    use crate::function::{AsyncFunction, FunctionContext};
+
+    #[tokio::test]
+    async fn test_reconcile_catalog() {
+        common_telemetry::init_default_ut_logging();
+
+        // reconcile_database(database_name)
+        let f = ReconcileDatabaseFunction;
+        let args = vec![Arc::new(StringVector::from(vec!["test"])) as _];
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
+        assert_eq!(expect, result);
+
+        // reconcile_database(database_name, resolve_strategy)
+        let f = ReconcileDatabaseFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["test"])) as _,
+            Arc::new(StringVector::from(vec!["UseLatest"])) as _,
+        ];
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
+        assert_eq!(expect, result);
+
+        // reconcile_database(database_name, resolve_strategy, parallelism)
+        let f = ReconcileDatabaseFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["test"])) as _,
+            Arc::new(StringVector::from(vec!["UseLatest"])) as _,
+            Arc::new(UInt32Vector::from_slice([10])) as _,
+        ];
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
+        assert_eq!(expect, result);
+
+        // invalid function args
+        let f = ReconcileDatabaseFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["UseLatest"])) as _,
+            Arc::new(UInt32Vector::from_slice([10])) as _,
+            Arc::new(StringVector::from(vec!["v1"])) as _,
+            Arc::new(StringVector::from(vec!["v2"])) as _,
+        ];
+        let err = f.eval(FunctionContext::mock(), &args).await.unwrap_err();
+        assert_matches!(err, Error::InvalidFuncArgs { .. });
+
+        // unsupported input data type
+        let f = ReconcileDatabaseFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["UseLatest"])) as _,
+            Arc::new(UInt32Vector::from_slice([10])) as _,
+            Arc::new(StringVector::from(vec!["v1"])) as _,
+        ];
+        let err = f.eval(FunctionContext::mock(), &args).await.unwrap_err();
+        assert_matches!(err, Error::UnsupportedInputDataType { .. });
+    }
+}

src/common/function/src/admin/reconcile_table.rs

@@ -0,0 +1,149 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::meta::reconcile_request::Target;
+use api::v1::meta::{ReconcileRequest, ReconcileTable, ResolveStrategy};
+use common_catalog::format_full_table_name;
+use common_error::ext::BoxedError;
+use common_macro::admin_fn;
+use common_query::error::{
+    MissingProcedureServiceHandlerSnafu, Result, TableMutationSnafu, UnsupportedInputDataTypeSnafu,
+};
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use common_telemetry::info;
+use datatypes::prelude::*;
+use session::context::QueryContextRef;
+use session::table_name::table_name_to_full_name;
+use snafu::ResultExt;
+
+use crate::handlers::ProcedureServiceHandlerRef;
+use crate::helper::parse_resolve_strategy;
+
+const FN_NAME: &str = "reconcile_table";
+
+/// A function to reconcile a table.
+/// Returns the procedure id if success.
+///
+/// - `reconcile_table(table_name)`.
+/// - `reconcile_table(table_name, resolve_strategy)`.
+///
+/// The parameters:
+/// - `table_name`:  the table name
+#[admin_fn(
+    name = ReconcileTableFunction,
+    display_name = reconcile_table,
+    sig_fn = signature,
+    ret = string
+)]
+pub(crate) async fn reconcile_table(
+    procedure_service_handler: &ProcedureServiceHandlerRef,
+    query_ctx: &QueryContextRef,
+    params: &[ValueRef<'_>],
+) -> Result<Value> {
+    let (table_name, resolve_strategy) = match params {
+        [ValueRef::String(table_name)] => (table_name, ResolveStrategy::UseLatest),
+        [ValueRef::String(table_name), ValueRef::String(resolve_strategy)] => {
+            (table_name, parse_resolve_strategy(resolve_strategy)?)
+        }
+        _ => {
+            return UnsupportedInputDataTypeSnafu {
+                function: FN_NAME,
+                datatypes: params.iter().map(|v| v.data_type()).collect::<Vec<_>>(),
+            }
+            .fail()
+        }
+    };
+    let (catalog_name, schema_name, table_name) = table_name_to_full_name(table_name, query_ctx)
+        .map_err(BoxedError::new)
+        .context(TableMutationSnafu)?;
+    info!(
+        "Reconciling table: {} with resolve_strategy: {:?}",
+        format_full_table_name(&catalog_name, &schema_name, &table_name),
+        resolve_strategy
+    );
+    let pid = procedure_service_handler
+        .reconcile(ReconcileRequest {
+            target: Some(Target::ReconcileTable(ReconcileTable {
+                catalog_name,
+                schema_name,
+                table_name,
+                resolve_strategy: resolve_strategy as i32,
+            })),
+            ..Default::default()
+        })
+        .await?;
+    match pid {
+        Some(pid) => Ok(Value::from(pid)),
+        None => Ok(Value::Null),
+    }
+}
+
+fn signature() -> Signature {
+    Signature::one_of(
+        vec![
+            // reconcile_table(table_name)
+            TypeSignature::Exact(vec![ConcreteDataType::string_datatype()]),
+            // reconcile_table(table_name, resolve_strategy)
+            TypeSignature::Exact(vec![
+                ConcreteDataType::string_datatype(),
+                ConcreteDataType::string_datatype(),
+            ]),
+        ],
+        Volatility::Immutable,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+    use std::sync::Arc;
+
+    use common_query::error::Error;
+    use datatypes::vectors::{StringVector, VectorRef};
+
+    use crate::admin::reconcile_table::ReconcileTableFunction;
+    use crate::function::{AsyncFunction, FunctionContext};
+
+    #[tokio::test]
+    async fn test_reconcile_table() {
+        common_telemetry::init_default_ut_logging();
+
+        // reconcile_table(table_name)
+        let f = ReconcileTableFunction;
+        let args = vec![Arc::new(StringVector::from(vec!["test"])) as _];
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
+        assert_eq!(expect, result);
+
+        // reconcile_table(table_name, resolve_strategy)
+        let f = ReconcileTableFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["test"])) as _,
+            Arc::new(StringVector::from(vec!["UseMetasrv"])) as _,
+        ];
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();
+        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
+        assert_eq!(expect, result);
+
+        // unsupported input data type
+        let f = ReconcileTableFunction;
+        let args = vec![
+            Arc::new(StringVector::from(vec!["test"])) as _,
+            Arc::new(StringVector::from(vec!["UseMetasrv"])) as _,
+            Arc::new(StringVector::from(vec!["10"])) as _,
+        ];
+        let err = f.eval(FunctionContext::mock(), &args).await.unwrap_err();
+        assert_matches!(err, Error::UnsupportedInputDataType { .. });
+    }
+}

src/common/function/src/handlers.rs

@@ -14,6 +14,7 @@
 
 use std::sync::Arc;
 
+use api::v1::meta::ReconcileRequest;
 use async_trait::async_trait;
 use catalog::CatalogManagerRef;
 use common_base::AffectedRows;
@@ -65,6 +66,9 @@ pub trait ProcedureServiceHandler: Send + Sync {
     /// Migrate a region from source peer to target peer, returns the procedure id if success.
     async fn migrate_region(&self, request: MigrateRegionRequest) -> Result<Option<String>>;
 
+    /// Reconcile a table, database or catalog, returns the procedure id if success.
+    async fn reconcile(&self, request: ReconcileRequest) -> Result<Option<String>>;
+
     /// Query the procedure' state by its id
     async fn query_procedure_state(&self, pid: &str) -> Result<ProcedureStateResponse>;
 

src/common/function/src/helper.rs

@@ -12,12 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_query::error::{InvalidInputTypeSnafu, Result};
+use api::v1::meta::ResolveStrategy;
+use common_query::error::{
+    InvalidFuncArgsSnafu, InvalidInputTypeSnafu, Result, UnsupportedInputDataTypeSnafu,
+};
 use common_query::prelude::{Signature, TypeSignature, Volatility};
 use datatypes::prelude::ConcreteDataType;
 use datatypes::types::cast::cast;
 use datatypes::value::ValueRef;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 
 /// Create a function signature with oneof signatures of interleaving two arguments.
 pub fn one_of_sigs2(args1: Vec<ConcreteDataType>, args2: Vec<ConcreteDataType>) -> Signature {
@@ -43,3 +46,64 @@ pub fn cast_u64(value: &ValueRef) -> Result<Option<u64>> {
         })
         .map(|v| v.as_u64())
 }
+
+/// Cast a [`ValueRef`] to u32, returns `None` if fails
+pub fn cast_u32(value: &ValueRef) -> Result<Option<u32>> {
+    cast((*value).into(), &ConcreteDataType::uint32_datatype())
+        .context(InvalidInputTypeSnafu {
+            err_msg: format!(
+                "Failed to cast input into uint32, actual type: {:#?}",
+                value.data_type(),
+            ),
+        })
+        .map(|v| v.as_u64().map(|v| v as u32))
+}
+
+/// Parse a resolve strategy from a string.
+pub fn parse_resolve_strategy(strategy: &str) -> Result<ResolveStrategy> {
+    ResolveStrategy::from_str_name(strategy).context(InvalidFuncArgsSnafu {
+        err_msg: format!("Invalid resolve strategy: {}", strategy),
+    })
+}
+
+/// Default parallelism for reconcile operations.
+pub fn default_parallelism() -> u32 {
+    64
+}
+
+/// Default resolve strategy for reconcile operations.
+pub fn default_resolve_strategy() -> ResolveStrategy {
+    ResolveStrategy::UseLatest
+}
+
+/// Get the string value from the params.
+///
+/// # Errors
+/// Returns an error if the input type is not a string.
+pub fn get_string_from_params<'a>(
+    params: &'a [ValueRef<'a>],
+    index: usize,
+    fn_name: &'a str,
+) -> Result<&'a str> {
+    let ValueRef::String(s) = &params[index] else {
+        return UnsupportedInputDataTypeSnafu {
+            function: fn_name,
+            datatypes: params.iter().map(|v| v.data_type()).collect::<Vec<_>>(),
+        }
+        .fail();
+    };
+    Ok(s)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_resolve_strategy() {
+        assert_eq!(
+            parse_resolve_strategy("UseLatest").unwrap(),
+            ResolveStrategy::UseLatest
+        );
+    }
+}

src/common/function/src/lib.rs

@@ -14,6 +14,7 @@
 
 #![feature(let_chains)]
 #![feature(try_blocks)]
+#![feature(assert_matches)]
 
 mod admin;
 mod flush_flow;

src/common/function/src/state.rs

@@ -32,7 +32,7 @@ impl FunctionState {
     pub fn mock() -> Self {
         use std::sync::Arc;
 
-        use api::v1::meta::ProcedureStatus;
+        use api::v1::meta::{ProcedureStatus, ReconcileRequest};
         use async_trait::async_trait;
         use catalog::CatalogManagerRef;
         use common_base::AffectedRows;
@@ -63,6 +63,10 @@ impl FunctionState {
                 Ok(Some("test_pid".to_string()))
             }
 
+            async fn reconcile(&self, _request: ReconcileRequest) -> Result<Option<String>> {
+                Ok(Some("test_pid".to_string()))
+            }
+
             async fn query_procedure_state(&self, _pid: &str) -> Result<ProcedureStateResponse> {
                 Ok(ProcedureStateResponse {
                     status: ProcedureStatus::Done.into(),

src/common/function/src/system/pg_catalog/version.rs

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt;
 use std::sync::Arc;
-use std::{env, fmt};
 
 use common_query::error::Result;
 use common_query::prelude::{Signature, Volatility};
@@ -47,7 +47,7 @@ impl Function for PGVersionFunction {
     fn eval(&self, _func_ctx: &FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
         let result = StringVector::from(vec![format!(
             "PostgreSQL 16.3 GreptimeDB {}",
-            env!("CARGO_PKG_VERSION")
+            common_version::version()
         )]);
         Ok(Arc::new(result))
     }

src/common/function/src/system/version.rs

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt;
 use std::sync::Arc;
-use std::{env, fmt};
 
 use common_query::error::Result;
 use common_query::prelude::{Signature, Volatility};
@@ -52,13 +52,13 @@ impl Function for VersionFunction {
                     "{}-greptimedb-{}",
                     std::env::var("GREPTIMEDB_MYSQL_SERVER_VERSION")
                         .unwrap_or_else(|_| "8.4.2".to_string()),
-                    env!("CARGO_PKG_VERSION")
+                    common_version::version()
                 )
             }
             Channel::Postgres => {
-                format!("16.3-greptimedb-{}", env!("CARGO_PKG_VERSION"))
+                format!("16.3-greptimedb-{}", common_version::version())
             }
-            _ => env!("CARGO_PKG_VERSION").to_string(),
+            _ => common_version::version().to_string(),
         };
         let result = StringVector::from(vec![version]);
         Ok(Arc::new(result))

src/common/grpc-expr/src/alter.rs

@@ -29,12 +29,12 @@ use snafu::{ensure, OptionExt, ResultExt};
 use store_api::region_request::{SetRegionOption, UnsetRegionOption};
 use table::metadata::TableId;
 use table::requests::{
-    AddColumnRequest, AlterKind, AlterTableRequest, ModifyColumnTypeRequest, SetIndexOptions,
+    AddColumnRequest, AlterKind, AlterTableRequest, ModifyColumnTypeRequest, SetIndexOption,
-    UnsetIndexOptions,
+    UnsetIndexOption,
 };
 
 use crate::error::{
-    InvalidColumnDefSnafu, InvalidSetFulltextOptionRequestSnafu,
+    InvalidColumnDefSnafu, InvalidIndexOptionSnafu, InvalidSetFulltextOptionRequestSnafu,
     InvalidSetSkippingIndexOptionRequestSnafu, InvalidSetTableOptionRequestSnafu,
     InvalidUnsetTableOptionRequestSnafu, MissingAlterIndexOptionSnafu, MissingFieldSnafu,
     MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu,
@@ -43,6 +43,59 @@ use crate::error::{
 const LOCATION_TYPE_FIRST: i32 = LocationType::First as i32;
 const LOCATION_TYPE_AFTER: i32 = LocationType::After as i32;
 
+fn set_index_option_from_proto(set_index: api::v1::SetIndex) -> Result<SetIndexOption> {
+    let options = set_index.options.context(MissingAlterIndexOptionSnafu)?;
+    Ok(match options {
+        api::v1::set_index::Options::Fulltext(f) => SetIndexOption::Fulltext {
+            column_name: f.column_name.clone(),
+            options: FulltextOptions::new(
+                f.enable,
+                as_fulltext_option_analyzer(
+                    Analyzer::try_from(f.analyzer).context(InvalidSetFulltextOptionRequestSnafu)?,
+                ),
+                f.case_sensitive,
+                as_fulltext_option_backend(
+                    PbFulltextBackend::try_from(f.backend)
+                        .context(InvalidSetFulltextOptionRequestSnafu)?,
+                ),
+                f.granularity as u32,
+                f.false_positive_rate,
+            )
+            .context(InvalidIndexOptionSnafu)?,
+        },
+        api::v1::set_index::Options::Inverted(i) => SetIndexOption::Inverted {
+            column_name: i.column_name,
+        },
+        api::v1::set_index::Options::Skipping(s) => SetIndexOption::Skipping {
+            column_name: s.column_name,
+            options: SkippingIndexOptions::new(
+                s.granularity as u32,
+                s.false_positive_rate,
+                as_skipping_index_type(
+                    PbSkippingIndexType::try_from(s.skipping_index_type)
+                        .context(InvalidSetSkippingIndexOptionRequestSnafu)?,
+                ),
+            )
+            .context(InvalidIndexOptionSnafu)?,
+        },
+    })
+}
+
+fn unset_index_option_from_proto(unset_index: api::v1::UnsetIndex) -> Result<UnsetIndexOption> {
+    let options = unset_index.options.context(MissingAlterIndexOptionSnafu)?;
+    Ok(match options {
+        api::v1::unset_index::Options::Fulltext(f) => UnsetIndexOption::Fulltext {
+            column_name: f.column_name,
+        },
+        api::v1::unset_index::Options::Inverted(i) => UnsetIndexOption::Inverted {
+            column_name: i.column_name,
+        },
+        api::v1::unset_index::Options::Skipping(s) => UnsetIndexOption::Skipping {
+            column_name: s.column_name,
+        },
+    })
+}
+
 /// Convert an [`AlterTableExpr`] to an [`AlterTableRequest`]
 pub fn alter_expr_to_request(table_id: TableId, expr: AlterTableExpr) -> Result<AlterTableRequest> {
     let catalog_name = expr.catalog_name;
@@ -121,65 +174,34 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterTableExpr) -> Result<
                     .context(InvalidUnsetTableOptionRequestSnafu)?,
             }
         }
-        Kind::SetIndex(o) => match o.options {
+        Kind::SetIndex(o) => {
-            Some(opt) => match opt {
+            let option = set_index_option_from_proto(o)?;
-                api::v1::set_index::Options::Fulltext(f) => AlterKind::SetIndex {
+            AlterKind::SetIndexes {
-                    options: SetIndexOptions::Fulltext {
+                options: vec![option],
-                        column_name: f.column_name.clone(),
+            }
-                        options: FulltextOptions {
+        }
-                            enable: f.enable,
+        Kind::UnsetIndex(o) => {
-                            analyzer: as_fulltext_option_analyzer(
+            let option = unset_index_option_from_proto(o)?;
-                                Analyzer::try_from(f.analyzer)
+            AlterKind::UnsetIndexes {
-                                    .context(InvalidSetFulltextOptionRequestSnafu)?,
+                options: vec![option],
-                            ),
+            }
-                            case_sensitive: f.case_sensitive,
+        }
-                            backend: as_fulltext_option_backend(
+        Kind::SetIndexes(o) => {
-                                PbFulltextBackend::try_from(f.backend)
+            let options = o
-                                    .context(InvalidSetFulltextOptionRequestSnafu)?,
+                .set_indexes
-                            ),
+                .into_iter()
-                        },
+                .map(set_index_option_from_proto)
-                    },
+                .collect::<Result<Vec<_>>>()?;
-                },
+            AlterKind::SetIndexes { options }
-                api::v1::set_index::Options::Inverted(i) => AlterKind::SetIndex {
+        }
-                    options: SetIndexOptions::Inverted {
+        Kind::UnsetIndexes(o) => {
-                        column_name: i.column_name,
+            let options = o
-                    },
+                .unset_indexes
-                },
+                .into_iter()
-                api::v1::set_index::Options::Skipping(s) => AlterKind::SetIndex {
+                .map(unset_index_option_from_proto)
-                    options: SetIndexOptions::Skipping {
+                .collect::<Result<Vec<_>>>()?;
-                        column_name: s.column_name,
+            AlterKind::UnsetIndexes { options }
-                        options: SkippingIndexOptions {
+        }
-                            granularity: s.granularity as u32,
-                            index_type: as_skipping_index_type(
-                                PbSkippingIndexType::try_from(s.skipping_index_type)
-                                    .context(InvalidSetSkippingIndexOptionRequestSnafu)?,
-                            ),
-                        },
-                    },
-                },
-            },
-            None => return MissingAlterIndexOptionSnafu.fail(),
-        },
-        Kind::UnsetIndex(o) => match o.options {
-            Some(opt) => match opt {
-                api::v1::unset_index::Options::Fulltext(f) => AlterKind::UnsetIndex {
-                    options: UnsetIndexOptions::Fulltext {
-                        column_name: f.column_name,
-                    },
-                },
-                api::v1::unset_index::Options::Inverted(i) => AlterKind::UnsetIndex {
-                    options: UnsetIndexOptions::Inverted {
-                        column_name: i.column_name,
-                    },
-                },
-                api::v1::unset_index::Options::Skipping(s) => AlterKind::UnsetIndex {
-                    options: UnsetIndexOptions::Skipping {
-                        column_name: s.column_name,
-                    },
-                },
-            },
-            None => return MissingAlterIndexOptionSnafu.fail(),
-        },
         Kind::DropDefaults(o) => {
             let names = o
                 .drop_defaults

src/common/grpc-expr/src/error.rs

@@ -153,6 +153,14 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Invalid index option"))]
+    InvalidIndexOption {
+        #[snafu(implicit)]
+        location: Location,
+        #[snafu(source)]
+        error: datatypes::error::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -180,7 +188,8 @@ impl ErrorExt for Error {
             | Error::InvalidUnsetTableOptionRequest { .. }
             | Error::InvalidSetFulltextOptionRequest { .. }
             | Error::InvalidSetSkippingIndexOptionRequest { .. }
-            | Error::MissingAlterIndexOption { .. } => StatusCode::InvalidArguments,
+            | Error::MissingAlterIndexOption { .. }
+            | Error::InvalidIndexOption { .. } => StatusCode::InvalidArguments,
         }
     }
 

src/common/meta/Cargo.toml

@@ -32,6 +32,7 @@ common-procedure.workspace = true
 common-procedure-test.workspace = true
 common-query.workspace = true
 common-recordbatch.workspace = true
+common-runtime.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
 common-wal.workspace = true

src/common/meta/src/cache/flow/table_flownode.rs

@@ -15,6 +15,7 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use common_telemetry::info;
 use futures::future::BoxFuture;
 use moka::future::Cache;
 use moka::ops::compute::Op;
@@ -89,6 +90,12 @@ fn init_factory(table_flow_manager: TableFlowManagerRef) -> Initializer<TableId,
                 // we have a corresponding cache invalidation mechanism to invalidate `(Key, EmptyHashSet)`.
                 .map(Arc::new)
                 .map(Some)
+                .inspect(|set| {
+                    info!(
+                        "Initialized table_flownode cache for table_id: {}, set: {:?}",
+                        table_id, set
+                    );
+                })
         })
     })
 }
@@ -167,6 +174,13 @@ fn invalidator<'a>(
         match ident {
             CacheIdent::CreateFlow(create_flow) => handle_create_flow(cache, create_flow).await,
             CacheIdent::DropFlow(drop_flow) => handle_drop_flow(cache, drop_flow).await,
+            CacheIdent::FlowNodeAddressChange(node_id) => {
+                info!(
+                    "Invalidate flow node cache for node_id in table_flownode: {}",
+                    node_id
+                );
+                cache.invalidate_all();
+            }
             _ => {}
         }
         Ok(())
@@ -174,7 +188,10 @@ fn invalidator<'a>(
 }
 
 fn filter(ident: &CacheIdent) -> bool {
-    matches!(ident, CacheIdent::CreateFlow(_) | CacheIdent::DropFlow(_))
+    matches!(
+        ident,
+        CacheIdent::CreateFlow(_) | CacheIdent::DropFlow(_) | CacheIdent::FlowNodeAddressChange(_)
+    )
 }
 
 #[cfg(test)]

src/common/meta/src/cache_invalidator.rs

@@ -22,6 +22,7 @@ use crate::key::flow::flow_name::FlowNameKey;
 use crate::key::flow::flow_route::FlowRouteKey;
 use crate::key::flow::flownode_flow::FlownodeFlowKey;
 use crate::key::flow::table_flow::TableFlowKey;
+use crate::key::node_address::NodeAddressKey;
 use crate::key::schema_name::SchemaNameKey;
 use crate::key::table_info::TableInfoKey;
 use crate::key::table_name::TableNameKey;
@@ -53,6 +54,10 @@ pub struct Context {
 #[async_trait::async_trait]
 pub trait CacheInvalidator: Send + Sync {
     async fn invalidate(&self, ctx: &Context, caches: &[CacheIdent]) -> Result<()>;
+
+    fn name(&self) -> &'static str {
+        std::any::type_name::<Self>()
+    }
 }
 
 pub type CacheInvalidatorRef = Arc<dyn CacheInvalidator>;
@@ -137,6 +142,13 @@ where
                     let key = FlowInfoKey::new(*flow_id);
                     self.invalidate_key(&key.to_bytes()).await;
                 }
+                CacheIdent::FlowNodeAddressChange(node_id) => {
+                    // other caches doesn't need to be invalidated
+                    // since this is only for flownode address change not id change
+                    common_telemetry::info!("Invalidate flow node cache for node_id: {}", node_id);
+                    let key = NodeAddressKey::with_flownode(*node_id);
+                    self.invalidate_key(&key.to_bytes()).await;
+                }
             }
         }
         Ok(())

src/common/meta/src/ddl.rs

@@ -15,25 +15,17 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use api::v1::meta::ProcedureDetailResponse;
-use common_telemetry::tracing_context::W3cTrace;
 use store_api::storage::{RegionId, RegionNumber, TableId};
 
 use crate::cache_invalidator::CacheInvalidatorRef;
 use crate::ddl::flow_meta::FlowMetadataAllocatorRef;
 use crate::ddl::table_meta::TableMetadataAllocatorRef;
-use crate::error::{Result, UnsupportedSnafu};
 use crate::key::flow::FlowMetadataManagerRef;
 use crate::key::table_route::PhysicalTableRouteValue;
 use crate::key::TableMetadataManagerRef;
 use crate::node_manager::NodeManagerRef;
 use crate::region_keeper::MemoryRegionKeeperRef;
 use crate::region_registry::LeaderRegionRegistryRef;
-use crate::rpc::ddl::{SubmitDdlTaskRequest, SubmitDdlTaskResponse};
-use crate::rpc::procedure::{
-    AddRegionFollowerRequest, MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse,
-    RemoveRegionFollowerRequest,
-};
 use crate::DatanodeId;
 
 pub mod alter_database;
@@ -44,13 +36,13 @@ pub mod create_flow;
 pub mod create_logical_tables;
 pub mod create_table;
 mod create_table_template;
+pub(crate) use create_table_template::{build_template_from_raw_table_info, CreateRequestBuilder};
 pub mod create_view;
 pub mod drop_database;
 pub mod drop_flow;
 pub mod drop_table;
 pub mod drop_view;
 pub mod flow_meta;
-mod physical_table_metadata;
 pub mod table_meta;
 #[cfg(any(test, feature = "testing"))]
 pub mod test_util;
@@ -59,64 +51,6 @@ pub(crate) mod tests;
 pub mod truncate_table;
 pub mod utils;
 
-#[derive(Debug, Default)]
-pub struct ExecutorContext {
-    pub tracing_context: Option<W3cTrace>,
-}
-
-/// The procedure executor that accepts ddl, region migration task etc.
-#[async_trait::async_trait]
-pub trait ProcedureExecutor: Send + Sync {
-    /// Submit a ddl task
-    async fn submit_ddl_task(
-        &self,
-        ctx: &ExecutorContext,
-        request: SubmitDdlTaskRequest,
-    ) -> Result<SubmitDdlTaskResponse>;
-
-    /// Add a region follower
-    async fn add_region_follower(
-        &self,
-        _ctx: &ExecutorContext,
-        _request: AddRegionFollowerRequest,
-    ) -> Result<()> {
-        UnsupportedSnafu {
-            operation: "add_region_follower",
-        }
-        .fail()
-    }
-
-    /// Remove a region follower
-    async fn remove_region_follower(
-        &self,
-        _ctx: &ExecutorContext,
-        _request: RemoveRegionFollowerRequest,
-    ) -> Result<()> {
-        UnsupportedSnafu {
-            operation: "remove_region_follower",
-        }
-        .fail()
-    }
-
-    /// Submit a region migration task
-    async fn migrate_region(
-        &self,
-        ctx: &ExecutorContext,
-        request: MigrateRegionRequest,
-    ) -> Result<MigrateRegionResponse>;
-
-    /// Query the procedure state by its id
-    async fn query_procedure_state(
-        &self,
-        ctx: &ExecutorContext,
-        pid: &str,
-    ) -> Result<ProcedureStateResponse>;
-
-    async fn list_procedures(&self, ctx: &ExecutorContext) -> Result<ProcedureDetailResponse>;
-}
-
-pub type ProcedureExecutorRef = Arc<dyn ProcedureExecutor>;
-
 /// Metadata allocated to a table.
 #[derive(Default)]
 pub struct TableMetadata {

src/common/meta/src/ddl/alter_logical_tables.rs

@@ -12,32 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod check;
+mod executor;
-mod metadata;
-mod region_request;
-mod table_cache_keys;
 mod update_metadata;
+mod validator;
 
 use api::region::RegionResponse;
 use async_trait::async_trait;
 use common_catalog::format_full_table_name;
 use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
 use common_procedure::{Context, LockKey, Procedure, Status};
-use common_telemetry::{error, info, warn};
+use common_telemetry::{debug, error, info, warn};
-use futures_util::future;
+pub use executor::make_alter_region_request;
-pub use region_request::make_alter_region_request;
 use serde::{Deserialize, Serialize};
-use snafu::{ensure, ResultExt};
+use snafu::ResultExt;
 use store_api::metadata::ColumnMetadata;
 use store_api::metric_engine_consts::ALTER_PHYSICAL_EXTENSION_KEY;
 use strum::AsRefStr;
 use table::metadata::TableId;
 
-use crate::ddl::utils::{
+use crate::cache_invalidator::Context as CacheContext;
-    add_peer_context_if_needed, map_to_procedure_error, sync_follower_regions,
+use crate::ddl::alter_logical_tables::executor::AlterLogicalTablesExecutor;
+use crate::ddl::alter_logical_tables::validator::{
+    retain_unskipped, AlterLogicalTableValidator, ValidatorResult,
 };
+use crate::ddl::utils::{extract_column_metadatas, map_to_procedure_error, sync_follower_regions};
 use crate::ddl::DdlContext;
-use crate::error::{DecodeJsonSnafu, MetadataCorruptionSnafu, Result};
+use crate::error::Result;
 use crate::instruction::CacheIdent;
 use crate::key::table_info::TableInfoValue;
 use crate::key::table_route::PhysicalTableRouteValue;
@@ -45,13 +45,38 @@ use crate::key::DeserializedValueWithBytes;
 use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
 use crate::metrics;
 use crate::rpc::ddl::AlterTableTask;
-use crate::rpc::router::{find_leaders, RegionRoute};
+use crate::rpc::router::RegionRoute;
 
 pub struct AlterLogicalTablesProcedure {
     pub context: DdlContext,
     pub data: AlterTablesData,
 }
 
+/// Builds the validator from the [`AlterTablesData`].
+fn build_validator_from_alter_table_data<'a>(
+    data: &'a AlterTablesData,
+) -> AlterLogicalTableValidator<'a> {
+    let phsycial_table_id = data.physical_table_id;
+    let alters = data
+        .tasks
+        .iter()
+        .map(|task| &task.alter_table)
+        .collect::<Vec<_>>();
+    AlterLogicalTableValidator::new(phsycial_table_id, alters)
+}
+
+/// Builds the executor from the [`AlterTablesData`].
+fn build_executor_from_alter_expr<'a>(data: &'a AlterTablesData) -> AlterLogicalTablesExecutor<'a> {
+    debug_assert_eq!(data.tasks.len(), data.table_info_values.len());
+    let alters = data
+        .tasks
+        .iter()
+        .zip(data.table_info_values.iter())
+        .map(|(task, table_info)| (table_info.table_info.ident.table_id, &task.alter_table))
+        .collect::<Vec<_>>();
+    AlterLogicalTablesExecutor::new(alters)
+}
+
 impl AlterLogicalTablesProcedure {
     pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterLogicalTables";
 
@@ -81,35 +106,44 @@ impl AlterLogicalTablesProcedure {
     }
 
     pub(crate) async fn on_prepare(&mut self) -> Result<Status> {
-        // Checks all the tasks
+        let validator = build_validator_from_alter_table_data(&self.data);
-        self.check_input_tasks()?;
+        let ValidatorResult {
-        // Fills the table info values
+            num_skipped,
-        self.fill_table_info_values().await?;
+            skip_alter,
-        // Checks the physical table, must after [fill_table_info_values]
+            table_info_values,
-        self.check_physical_table().await?;
+            physical_table_info,
-        // Fills the physical table info
+            physical_table_route,
-        self.fill_physical_table_info().await?;
+        } = validator
-        // Filter the finished tasks
+            .validate(&self.context.table_metadata_manager)
-        let finished_tasks = self.check_finished_tasks()?;
+            .await?;
-        let already_finished_count = finished_tasks
+
-            .iter()
+        let num_tasks = self.data.tasks.len();
-            .map(|x| if *x { 1 } else { 0 })
+        if num_skipped == num_tasks {
-            .sum::<usize>();
-        let apply_tasks_count = self.data.tasks.len();
-        if already_finished_count == apply_tasks_count {
             info!("All the alter tasks are finished, will skip the procedure.");
+            let cache_ident_keys = AlterLogicalTablesExecutor::build_cache_ident_keys(
+                &physical_table_info,
+                &table_info_values
+                    .iter()
+                    .map(|v| v.get_inner_ref())
+                    .collect::<Vec<_>>(),
+            );
+            self.data.table_cache_keys_to_invalidate = cache_ident_keys;
             // Re-invalidate the table cache
             self.data.state = AlterTablesState::InvalidateTableCache;
             return Ok(Status::executing(true));
-        } else if already_finished_count > 0 {
+        } else if num_skipped > 0 {
             info!(
                 "There are {} alter tasks, {} of them were already finished.",
-                apply_tasks_count, already_finished_count
+                num_tasks, num_skipped
             );
         }
-        self.filter_task(&finished_tasks)?;
 
-        // Next state
+        // Updates the procedure state.
+        retain_unskipped(&mut self.data.tasks, &skip_alter);
+        self.data.physical_table_info = Some(physical_table_info);
+        self.data.physical_table_route = Some(physical_table_route);
+        self.data.table_info_values = table_info_values;
+        debug_assert_eq!(self.data.tasks.len(), self.data.table_info_values.len());
         self.data.state = AlterTablesState::SubmitAlterRegionRequests;
         Ok(Status::executing(true))
     }
@@ -117,57 +151,21 @@ impl AlterLogicalTablesProcedure {
     pub(crate) async fn on_submit_alter_region_requests(&mut self) -> Result<Status> {
         // Safety: we have checked the state in on_prepare
         let physical_table_route = &self.data.physical_table_route.as_ref().unwrap();
-        let leaders = find_leaders(&physical_table_route.region_routes);
+        let executor = build_executor_from_alter_expr(&self.data);
-        let mut alter_region_tasks = Vec::with_capacity(leaders.len());
+        let mut results = executor
+            .on_alter_regions(
+                &self.context.node_manager,
+                &physical_table_route.region_routes,
+            )
+            .await?;
 
-        for peer in leaders {
+        if let Some(column_metadatas) =
-            let requester = self.context.node_manager.datanode(&peer).await;
+            extract_column_metadatas(&mut results, ALTER_PHYSICAL_EXTENSION_KEY)?
-            let request = self.make_request(&peer, &physical_table_route.region_routes)?;
+        {
-
+            self.data.physical_columns = column_metadatas;
-            alter_region_tasks.push(async move {
-                requester
-                    .handle(request)
-                    .await
-                    .map_err(add_peer_context_if_needed(peer))
-            });
-        }
-
-        let mut results = future::join_all(alter_region_tasks)
-            .await
-            .into_iter()
-            .collect::<Result<Vec<_>>>()?;
-
-        // Collects responses from datanodes.
-        let phy_raw_schemas = results
-            .iter_mut()
-            .map(|res| res.extensions.remove(ALTER_PHYSICAL_EXTENSION_KEY))
-            .collect::<Vec<_>>();
-
-        if phy_raw_schemas.is_empty() {
-            self.submit_sync_region_requests(results, &physical_table_route.region_routes)
-                .await;
-            self.data.state = AlterTablesState::UpdateMetadata;
-            return Ok(Status::executing(true));
-        }
-
-        // Verify all the physical schemas are the same
-        // Safety: previous check ensures this vec is not empty
-        let first = phy_raw_schemas.first().unwrap();
-        ensure!(
-            phy_raw_schemas.iter().all(|x| x == first),
-            MetadataCorruptionSnafu {
-                err_msg: "The physical schemas from datanodes are not the same."
-            }
-        );
-
-        // Decodes the physical raw schemas
-        if let Some(phy_raw_schema) = first {
-            self.data.physical_columns =
-                ColumnMetadata::decode_list(phy_raw_schema).context(DecodeJsonSnafu)?;
         } else {
             warn!("altering logical table result doesn't contains extension key `{ALTER_PHYSICAL_EXTENSION_KEY}`,leaving the physical table's schema unchanged");
         }
-
         self.submit_sync_region_requests(results, &physical_table_route.region_routes)
             .await;
         self.data.state = AlterTablesState::UpdateMetadata;
@@ -183,7 +181,7 @@ impl AlterLogicalTablesProcedure {
         if let Err(err) = sync_follower_regions(
             &self.context,
             self.data.physical_table_id,
-            results,
+            &results,
             region_routes,
             table_info.meta.engine.as_str(),
         )
@@ -200,7 +198,18 @@ impl AlterLogicalTablesProcedure {
         self.update_physical_table_metadata().await?;
         self.update_logical_tables_metadata().await?;
 
-        self.data.build_cache_keys_to_invalidate();
+        let logical_table_info_values = self
+            .data
+            .table_info_values
+            .iter()
+            .map(|v| v.get_inner_ref())
+            .collect::<Vec<_>>();
+
+        let cache_ident_keys = AlterLogicalTablesExecutor::build_cache_ident_keys(
+            self.data.physical_table_info.as_ref().unwrap(),
+            &logical_table_info_values,
+        );
+        self.data.table_cache_keys_to_invalidate = cache_ident_keys;
         self.data.clear_metadata_fields();
 
         self.data.state = AlterTablesState::InvalidateTableCache;
@@ -210,9 +219,16 @@ impl AlterLogicalTablesProcedure {
     pub(crate) async fn on_invalidate_table_cache(&mut self) -> Result<Status> {
         let to_invalidate = &self.data.table_cache_keys_to_invalidate;
 
+        let ctx = CacheContext {
+            subject: Some(format!(
+                "Invalidate table cache by altering logical tables, physical_table_id: {}",
+                self.data.physical_table_id,
+            )),
+        };
+
         self.context
             .cache_invalidator
-            .invalidate(&Default::default(), to_invalidate)
+            .invalidate(&ctx, to_invalidate)
             .await?;
         Ok(Status::done())
     }
@@ -232,6 +248,10 @@ impl Procedure for AlterLogicalTablesProcedure {
         let _timer = metrics::METRIC_META_PROCEDURE_ALTER_TABLE
             .with_label_values(&[step])
             .start_timer();
+        debug!(
+            "Executing alter logical tables procedure, state: {:?}",
+            state
+        );
 
         match state {
             AlterTablesState::Prepare => self.on_prepare().await,

src/common/meta/src/ddl/alter_logical_tables/check.rs

@@ -1,136 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashSet;
-
-use api::v1::alter_table_expr::Kind;
-use snafu::{ensure, OptionExt};
-
-use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
-use crate::error::{AlterLogicalTablesInvalidArgumentsSnafu, Result};
-use crate::key::table_info::TableInfoValue;
-use crate::key::table_route::TableRouteValue;
-use crate::rpc::ddl::AlterTableTask;
-
-impl AlterLogicalTablesProcedure {
-    pub(crate) fn check_input_tasks(&self) -> Result<()> {
-        self.check_schema()?;
-        self.check_alter_kind()?;
-        Ok(())
-    }
-
-    pub(crate) async fn check_physical_table(&self) -> Result<()> {
-        let table_route_manager = self.context.table_metadata_manager.table_route_manager();
-        let table_ids = self
-            .data
-            .table_info_values
-            .iter()
-            .map(|v| v.table_info.ident.table_id)
-            .collect::<Vec<_>>();
-        let table_routes = table_route_manager
-            .table_route_storage()
-            .batch_get(&table_ids)
-            .await?;
-        let physical_table_id = self.data.physical_table_id;
-        let is_same_physical_table = table_routes.iter().all(|r| {
-            if let Some(TableRouteValue::Logical(r)) = r {
-                r.physical_table_id() == physical_table_id
-            } else {
-                false
-            }
-        });
-
-        ensure!(
-            is_same_physical_table,
-            AlterLogicalTablesInvalidArgumentsSnafu {
-                err_msg: "All the tasks should have the same physical table id"
-            }
-        );
-
-        Ok(())
-    }
-
-    pub(crate) fn check_finished_tasks(&self) -> Result<Vec<bool>> {
-        let task = &self.data.tasks;
-        let table_info_values = &self.data.table_info_values;
-
-        Ok(task
-            .iter()
-            .zip(table_info_values.iter())
-            .map(|(task, table)| Self::check_finished_task(task, table))
-            .collect())
-    }
-
-    // Checks if the schemas of the tasks are the same
-    fn check_schema(&self) -> Result<()> {
-        let is_same_schema = self.data.tasks.windows(2).all(|pair| {
-            pair[0].alter_table.catalog_name == pair[1].alter_table.catalog_name
-                && pair[0].alter_table.schema_name == pair[1].alter_table.schema_name
-        });
-
-        ensure!(
-            is_same_schema,
-            AlterLogicalTablesInvalidArgumentsSnafu {
-                err_msg: "Schemas of the tasks are not the same"
-            }
-        );
-
-        Ok(())
-    }
-
-    fn check_alter_kind(&self) -> Result<()> {
-        for task in &self.data.tasks {
-            let kind = task.alter_table.kind.as_ref().context(
-                AlterLogicalTablesInvalidArgumentsSnafu {
-                    err_msg: "Alter kind is missing",
-                },
-            )?;
-            let Kind::AddColumns(_) = kind else {
-                return AlterLogicalTablesInvalidArgumentsSnafu {
-                    err_msg: "Only support add columns operation",
-                }
-                .fail();
-            };
-        }
-
-        Ok(())
-    }
-
-    fn check_finished_task(task: &AlterTableTask, table: &TableInfoValue) -> bool {
-        let columns = table
-            .table_info
-            .meta
-            .schema
-            .column_schemas
-            .iter()
-            .map(|c| &c.name)
-            .collect::<HashSet<_>>();
-
-        let Some(kind) = task.alter_table.kind.as_ref() else {
-            return true; // Never get here since we have checked it in `check_alter_kind`
-        };
-        let Kind::AddColumns(add_columns) = kind else {
-            return true; // Never get here since we have checked it in `check_alter_kind`
-        };
-
-        // We only check that all columns have been finished. That is to say,
-        // if one part is finished but another part is not, it will be considered
-        // unfinished.
-        add_columns
-            .add_columns
-            .iter()
-            .map(|add_column| add_column.column_def.as_ref().map(|c| &c.name))
-            .all(|column| column.map(|c| columns.contains(c)).unwrap_or(false))
-    }
-}

src/common/meta/src/ddl/alter_logical_tables/executor.rs

@@ -0,0 +1,216 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+
+use api::region::RegionResponse;
+use api::v1::alter_table_expr::Kind;
+use api::v1::region::{
+    alter_request, region_request, AddColumn, AddColumns, AlterRequest, AlterRequests,
+    RegionColumnDef, RegionRequest, RegionRequestHeader,
+};
+use api::v1::{self, AlterTableExpr};
+use common_telemetry::tracing_context::TracingContext;
+use common_telemetry::{debug, warn};
+use futures::future;
+use store_api::metadata::ColumnMetadata;
+use store_api::storage::{RegionId, RegionNumber, TableId};
+
+use crate::ddl::utils::{add_peer_context_if_needed, raw_table_info};
+use crate::error::Result;
+use crate::instruction::CacheIdent;
+use crate::key::table_info::TableInfoValue;
+use crate::key::{DeserializedValueWithBytes, RegionDistribution, TableMetadataManagerRef};
+use crate::node_manager::NodeManagerRef;
+use crate::rpc::router::{find_leaders, region_distribution, RegionRoute};
+
+/// [AlterLogicalTablesExecutor] performs:
+/// - Alters logical regions on the datanodes.
+/// - Updates table metadata for alter table operation.
+pub struct AlterLogicalTablesExecutor<'a> {
+    /// The alter table expressions.
+    ///
+    /// The first element is the logical table id, the second element is the alter table expression.
+    alters: Vec<(TableId, &'a AlterTableExpr)>,
+}
+
+impl<'a> AlterLogicalTablesExecutor<'a> {
+    pub fn new(alters: Vec<(TableId, &'a AlterTableExpr)>) -> Self {
+        Self { alters }
+    }
+
+    /// Alters logical regions on the datanodes.
+    pub(crate) async fn on_alter_regions(
+        &self,
+        node_manager: &NodeManagerRef,
+        region_routes: &[RegionRoute],
+    ) -> Result<Vec<RegionResponse>> {
+        let region_distribution = region_distribution(region_routes);
+        let leaders = find_leaders(region_routes)
+            .into_iter()
+            .map(|p| (p.id, p))
+            .collect::<HashMap<_, _>>();
+        let mut alter_region_tasks = Vec::with_capacity(leaders.len());
+        for (datanode_id, region_role_set) in region_distribution {
+            if region_role_set.leader_regions.is_empty() {
+                continue;
+            }
+            // Safety: must exists.
+            let peer = leaders.get(&datanode_id).unwrap();
+            let requester = node_manager.datanode(peer).await;
+            let requests = self.make_alter_region_request(&region_role_set.leader_regions);
+            let requester = requester.clone();
+            let peer = peer.clone();
+
+            debug!("Sending alter region requests to datanode {}", peer);
+            alter_region_tasks.push(async move {
+                requester
+                    .handle(make_request(requests))
+                    .await
+                    .map_err(add_peer_context_if_needed(peer))
+            });
+        }
+
+        future::join_all(alter_region_tasks)
+            .await
+            .into_iter()
+            .collect::<Result<Vec<_>>>()
+    }
+
+    fn make_alter_region_request(&self, region_numbers: &[RegionNumber]) -> AlterRequests {
+        let mut requests = Vec::with_capacity(region_numbers.len() * self.alters.len());
+        for (table_id, alter) in self.alters.iter() {
+            for region_number in region_numbers {
+                let region_id = RegionId::new(*table_id, *region_number);
+                let request = make_alter_region_request(region_id, alter);
+                requests.push(request);
+            }
+        }
+
+        AlterRequests { requests }
+    }
+
+    /// Updates table metadata for alter table operation.
+    ///
+    /// ## Panic:
+    /// - If the region distribution is not set when updating table metadata.
+    pub(crate) async fn on_alter_metadata(
+        physical_table_id: TableId,
+        table_metadata_manager: &TableMetadataManagerRef,
+        current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
+        region_distribution: RegionDistribution,
+        physical_columns: &[ColumnMetadata],
+    ) -> Result<()> {
+        if physical_columns.is_empty() {
+            warn!("No physical columns found, leaving the physical table's schema unchanged when altering logical tables");
+            return Ok(());
+        }
+
+        let table_ref = current_table_info_value.table_ref();
+        let table_id = physical_table_id;
+
+        // Generates new table info
+        let old_raw_table_info = current_table_info_value.table_info.clone();
+        let new_raw_table_info =
+            raw_table_info::build_new_physical_table_info(old_raw_table_info, physical_columns);
+
+        debug!(
+            "Starting update table: {} metadata, table_id: {}, new table info: {:?}",
+            table_ref, table_id, new_raw_table_info
+        );
+
+        table_metadata_manager
+            .update_table_info(
+                current_table_info_value,
+                Some(region_distribution),
+                new_raw_table_info,
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    /// Builds the cache ident keys for the alter logical tables.
+    ///
+    /// The cache ident keys are:
+    /// - The table id of the logical tables.
+    /// - The table name of the logical tables.
+    /// - The table id of the physical table.
+    pub(crate) fn build_cache_ident_keys(
+        physical_table_info: &TableInfoValue,
+        logical_table_info_values: &[&TableInfoValue],
+    ) -> Vec<CacheIdent> {
+        let mut cache_keys = Vec::with_capacity(logical_table_info_values.len() * 2 + 2);
+        cache_keys.extend(logical_table_info_values.iter().flat_map(|table| {
+            vec![
+                CacheIdent::TableId(table.table_info.ident.table_id),
+                CacheIdent::TableName(table.table_name()),
+            ]
+        }));
+        cache_keys.push(CacheIdent::TableId(
+            physical_table_info.table_info.ident.table_id,
+        ));
+        cache_keys.push(CacheIdent::TableName(physical_table_info.table_name()));
+
+        cache_keys
+    }
+}
+
+fn make_request(alter_requests: AlterRequests) -> RegionRequest {
+    RegionRequest {
+        header: Some(RegionRequestHeader {
+            tracing_context: TracingContext::from_current_span().to_w3c(),
+            ..Default::default()
+        }),
+        body: Some(region_request::Body::Alters(alter_requests)),
+    }
+}
+
+/// Makes an alter region request.
+pub fn make_alter_region_request(
+    region_id: RegionId,
+    alter_table_expr: &AlterTableExpr,
+) -> AlterRequest {
+    let region_id = region_id.as_u64();
+    let kind = match &alter_table_expr.kind {
+        Some(Kind::AddColumns(add_columns)) => Some(alter_request::Kind::AddColumns(
+            to_region_add_columns(add_columns),
+        )),
+        _ => unreachable!(), // Safety: we have checked the kind in check_input_tasks
+    };
+
+    AlterRequest {
+        region_id,
+        schema_version: 0,
+        kind,
+    }
+}
+
+fn to_region_add_columns(add_columns: &v1::AddColumns) -> AddColumns {
+    let add_columns = add_columns
+        .add_columns
+        .iter()
+        .map(|add_column| {
+            let region_column_def = RegionColumnDef {
+                column_def: add_column.column_def.clone(),
+                ..Default::default() // other fields are not used in alter logical table
+            };
+            AddColumn {
+                column_def: Some(region_column_def),
+                ..Default::default() // other fields are not used in alter logical table
+            }
+        })
+        .collect();
+    AddColumns { add_columns }
+}

src/common/meta/src/ddl/alter_logical_tables/metadata.rs

@@ -1,158 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_catalog::format_full_table_name;
-use snafu::OptionExt;
-use table::metadata::TableId;
-
-use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
-use crate::error::{
-    AlterLogicalTablesInvalidArgumentsSnafu, Result, TableInfoNotFoundSnafu, TableNotFoundSnafu,
-    TableRouteNotFoundSnafu,
-};
-use crate::key::table_info::TableInfoValue;
-use crate::key::table_name::TableNameKey;
-use crate::key::table_route::TableRouteValue;
-use crate::key::DeserializedValueWithBytes;
-use crate::rpc::ddl::AlterTableTask;
-
-impl AlterLogicalTablesProcedure {
-    pub(crate) fn filter_task(&mut self, finished_tasks: &[bool]) -> Result<()> {
-        debug_assert_eq!(finished_tasks.len(), self.data.tasks.len());
-        debug_assert_eq!(finished_tasks.len(), self.data.table_info_values.len());
-        self.data.tasks = self
-            .data
-            .tasks
-            .drain(..)
-            .zip(finished_tasks.iter())
-            .filter_map(|(task, finished)| if *finished { None } else { Some(task) })
-            .collect();
-        self.data.table_info_values = self
-            .data
-            .table_info_values
-            .drain(..)
-            .zip(finished_tasks.iter())
-            .filter_map(|(table_info_value, finished)| {
-                if *finished {
-                    None
-                } else {
-                    Some(table_info_value)
-                }
-            })
-            .collect();
-
-        Ok(())
-    }
-
-    pub(crate) async fn fill_physical_table_info(&mut self) -> Result<()> {
-        let (physical_table_info, physical_table_route) = self
-            .context
-            .table_metadata_manager
-            .get_full_table_info(self.data.physical_table_id)
-            .await?;
-
-        let physical_table_info = physical_table_info.with_context(|| TableInfoNotFoundSnafu {
-            table: format!("table id - {}", self.data.physical_table_id),
-        })?;
-        let physical_table_route = physical_table_route
-            .context(TableRouteNotFoundSnafu {
-                table_id: self.data.physical_table_id,
-            })?
-            .into_inner();
-
-        self.data.physical_table_info = Some(physical_table_info);
-        let TableRouteValue::Physical(physical_table_route) = physical_table_route else {
-            return AlterLogicalTablesInvalidArgumentsSnafu {
-                err_msg: format!(
-                    "expected a physical table but got a logical table: {:?}",
-                    self.data.physical_table_id
-                ),
-            }
-            .fail();
-        };
-        self.data.physical_table_route = Some(physical_table_route);
-
-        Ok(())
-    }
-
-    pub(crate) async fn fill_table_info_values(&mut self) -> Result<()> {
-        let table_ids = self.get_all_table_ids().await?;
-        let table_info_values = self.get_all_table_info_values(&table_ids).await?;
-        debug_assert_eq!(table_info_values.len(), self.data.tasks.len());
-        self.data.table_info_values = table_info_values;
-
-        Ok(())
-    }
-
-    async fn get_all_table_info_values(
-        &self,
-        table_ids: &[TableId],
-    ) -> Result<Vec<DeserializedValueWithBytes<TableInfoValue>>> {
-        let table_info_manager = self.context.table_metadata_manager.table_info_manager();
-        let mut table_info_map = table_info_manager.batch_get_raw(table_ids).await?;
-        let mut table_info_values = Vec::with_capacity(table_ids.len());
-        for (table_id, task) in table_ids.iter().zip(self.data.tasks.iter()) {
-            let table_info_value =
-                table_info_map
-                    .remove(table_id)
-                    .with_context(|| TableInfoNotFoundSnafu {
-                        table: extract_table_name(task),
-                    })?;
-            table_info_values.push(table_info_value);
-        }
-
-        Ok(table_info_values)
-    }
-
-    async fn get_all_table_ids(&self) -> Result<Vec<TableId>> {
-        let table_name_manager = self.context.table_metadata_manager.table_name_manager();
-        let table_name_keys = self
-            .data
-            .tasks
-            .iter()
-            .map(|task| extract_table_name_key(task))
-            .collect();
-
-        let table_name_values = table_name_manager.batch_get(table_name_keys).await?;
-        let mut table_ids = Vec::with_capacity(table_name_values.len());
-        for (value, task) in table_name_values.into_iter().zip(self.data.tasks.iter()) {
-            let table_id = value
-                .with_context(|| TableNotFoundSnafu {
-                    table_name: extract_table_name(task),
-                })?
-                .table_id();
-            table_ids.push(table_id);
-        }
-
-        Ok(table_ids)
-    }
-}
-
-#[inline]
-fn extract_table_name(task: &AlterTableTask) -> String {
-    format_full_table_name(
-        &task.alter_table.catalog_name,
-        &task.alter_table.schema_name,
-        &task.alter_table.table_name,
-    )
-}
-
-#[inline]
-fn extract_table_name_key(task: &AlterTableTask) -> TableNameKey {
-    TableNameKey::new(
-        &task.alter_table.catalog_name,
-        &task.alter_table.schema_name,
-        &task.alter_table.table_name,
-    )
-}

src/common/meta/src/ddl/alter_logical_tables/region_request.rs

@@ -1,113 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use api::v1::alter_table_expr::Kind;
-use api::v1::region::{
-    alter_request, region_request, AddColumn, AddColumns, AlterRequest, AlterRequests,
-    RegionColumnDef, RegionRequest, RegionRequestHeader,
-};
-use api::v1::{self, AlterTableExpr};
-use common_telemetry::tracing_context::TracingContext;
-use store_api::storage::RegionId;
-
-use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
-use crate::error::Result;
-use crate::peer::Peer;
-use crate::rpc::router::{find_leader_regions, RegionRoute};
-
-impl AlterLogicalTablesProcedure {
-    pub(crate) fn make_request(
-        &self,
-        peer: &Peer,
-        region_routes: &[RegionRoute],
-    ) -> Result<RegionRequest> {
-        let alter_requests = self.make_alter_region_requests(peer, region_routes)?;
-        let request = RegionRequest {
-            header: Some(RegionRequestHeader {
-                tracing_context: TracingContext::from_current_span().to_w3c(),
-                ..Default::default()
-            }),
-            body: Some(region_request::Body::Alters(alter_requests)),
-        };
-
-        Ok(request)
-    }
-
-    fn make_alter_region_requests(
-        &self,
-        peer: &Peer,
-        region_routes: &[RegionRoute],
-    ) -> Result<AlterRequests> {
-        let tasks = &self.data.tasks;
-        let regions_on_this_peer = find_leader_regions(region_routes, peer);
-        let mut requests = Vec::with_capacity(tasks.len() * regions_on_this_peer.len());
-        for (task, table) in self
-            .data
-            .tasks
-            .iter()
-            .zip(self.data.table_info_values.iter())
-        {
-            for region_number in &regions_on_this_peer {
-                let region_id = RegionId::new(table.table_info.ident.table_id, *region_number);
-                let request = make_alter_region_request(
-                    region_id,
-                    &task.alter_table,
-                    table.table_info.ident.version,
-                );
-                requests.push(request);
-            }
-        }
-
-        Ok(AlterRequests { requests })
-    }
-}
-
-/// Makes an alter region request.
-pub fn make_alter_region_request(
-    region_id: RegionId,
-    alter_table_expr: &AlterTableExpr,
-    schema_version: u64,
-) -> AlterRequest {
-    let region_id = region_id.as_u64();
-    let kind = match &alter_table_expr.kind {
-        Some(Kind::AddColumns(add_columns)) => Some(alter_request::Kind::AddColumns(
-            to_region_add_columns(add_columns),
-        )),
-        _ => unreachable!(), // Safety: we have checked the kind in check_input_tasks
-    };
-
-    AlterRequest {
-        region_id,
-        schema_version,
-        kind,
-    }
-}
-
-fn to_region_add_columns(add_columns: &v1::AddColumns) -> AddColumns {
-    let add_columns = add_columns
-        .add_columns
-        .iter()
-        .map(|add_column| {
-            let region_column_def = RegionColumnDef {
-                column_def: add_column.column_def.clone(),
-                ..Default::default() // other fields are not used in alter logical table
-            };
-            AddColumn {
-                column_def: Some(region_column_def),
-                ..Default::default() // other fields are not used in alter logical table
-            }
-        })
-        .collect();
-    AddColumns { add_columns }
-}

src/common/meta/src/ddl/alter_logical_tables/table_cache_keys.rs

@@ -1,50 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use table::metadata::RawTableInfo;
-use table::table_name::TableName;
-
-use crate::ddl::alter_logical_tables::AlterTablesData;
-use crate::instruction::CacheIdent;
-
-impl AlterTablesData {
-    pub(crate) fn build_cache_keys_to_invalidate(&mut self) {
-        let mut cache_keys = self
-            .table_info_values
-            .iter()
-            .flat_map(|table| {
-                vec![
-                    CacheIdent::TableId(table.table_info.ident.table_id),
-                    CacheIdent::TableName(extract_table_name(&table.table_info)),
-                ]
-            })
-            .collect::<Vec<_>>();
-        cache_keys.push(CacheIdent::TableId(self.physical_table_id));
-        // Safety: physical_table_info already filled in previous steps
-        let physical_table_info = &self.physical_table_info.as_ref().unwrap().table_info;
-        cache_keys.push(CacheIdent::TableName(extract_table_name(
-            physical_table_info,
-        )));
-
-        self.table_cache_keys_to_invalidate = cache_keys;
-    }
-}
-
-fn extract_table_name(table_info: &RawTableInfo) -> TableName {
-    TableName::new(
-        &table_info.catalog_name,
-        &table_info.schema_name,
-        &table_info.name,
-    )
-}

src/common/meta/src/ddl/alter_logical_tables/update_metadata.rs

@@ -13,40 +13,34 @@
 // limitations under the License.
 
 use common_grpc_expr::alter_expr_to_request;
-use common_telemetry::warn;
-use itertools::Itertools;
 use snafu::ResultExt;
 use table::metadata::{RawTableInfo, TableInfo};
 
+use crate::ddl::alter_logical_tables::executor::AlterLogicalTablesExecutor;
 use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
-use crate::ddl::physical_table_metadata;
+use crate::ddl::utils::table_info::batch_update_table_info_values;
 use crate::error;
 use crate::error::{ConvertAlterTableRequestSnafu, Result};
 use crate::key::table_info::TableInfoValue;
 use crate::key::DeserializedValueWithBytes;
 use crate::rpc::ddl::AlterTableTask;
+use crate::rpc::router::region_distribution;
 
 impl AlterLogicalTablesProcedure {
     pub(crate) async fn update_physical_table_metadata(&mut self) -> Result<()> {
-        if self.data.physical_columns.is_empty() {
-            warn!("No physical columns found, leaving the physical table's schema unchanged when altering logical tables");
-            return Ok(());
-        }
-
         // Safety: must exist.
         let physical_table_info = self.data.physical_table_info.as_ref().unwrap();
+        let physical_table_route = self.data.physical_table_route.as_ref().unwrap();
+        let region_distribution = region_distribution(&physical_table_route.region_routes);
 
-        // Generates new table info
+        // Updates physical table's metadata.
-        let old_raw_table_info = physical_table_info.table_info.clone();
+        AlterLogicalTablesExecutor::on_alter_metadata(
-        let new_raw_table_info = physical_table_metadata::build_new_physical_table_info(
+            self.data.physical_table_id,
-            old_raw_table_info,
+            &self.context.table_metadata_manager,
+            physical_table_info,
+            region_distribution,
             &self.data.physical_columns,
-        );
+        )
-
-        // Updates physical table's metadata, and we don't need to touch per-region settings.
-        self.context
-            .table_metadata_manager
-            .update_table_info(physical_table_info, None, new_raw_table_info)
         .await?;
 
         Ok(())
@@ -54,25 +48,8 @@ impl AlterLogicalTablesProcedure {
 
     pub(crate) async fn update_logical_tables_metadata(&mut self) -> Result<()> {
         let table_info_values = self.build_update_metadata()?;
-        let manager = &self.context.table_metadata_manager;
+        batch_update_table_info_values(&self.context.table_metadata_manager, table_info_values)
-        let chunk_size = manager.batch_update_table_info_value_chunk_size();
+            .await
-        if table_info_values.len() > chunk_size {
-            let chunks = table_info_values
-                .into_iter()
-                .chunks(chunk_size)
-                .into_iter()
-                .map(|check| check.collect::<Vec<_>>())
-                .collect::<Vec<_>>();
-            for chunk in chunks {
-                manager.batch_update_table_info_values(chunk).await?;
-            }
-        } else {
-            manager
-                .batch_update_table_info_values(table_info_values)
-                .await?;
-        }
-
-        Ok(())
     }
 
     pub(crate) fn build_update_metadata(

src/common/meta/src/ddl/alter_logical_tables/validator.rs

@@ -0,0 +1,279 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashSet;
+
+use api::v1::alter_table_expr::Kind;
+use api::v1::AlterTableExpr;
+use snafu::{ensure, OptionExt};
+use store_api::storage::TableId;
+use table::table_reference::TableReference;
+
+use crate::ddl::utils::table_id::get_all_table_ids_by_names;
+use crate::ddl::utils::table_info::{
+    all_logical_table_routes_have_same_physical_id, get_all_table_info_values_by_table_ids,
+};
+use crate::error::{
+    AlterLogicalTablesInvalidArgumentsSnafu, Result, TableInfoNotFoundSnafu,
+    TableRouteNotFoundSnafu,
+};
+use crate::key::table_info::TableInfoValue;
+use crate::key::table_route::{PhysicalTableRouteValue, TableRouteManager, TableRouteValue};
+use crate::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
+
+/// [AlterLogicalTableValidator] validates the alter logical expressions.
+pub struct AlterLogicalTableValidator<'a> {
+    physical_table_id: TableId,
+    alters: Vec<&'a AlterTableExpr>,
+}
+
+impl<'a> AlterLogicalTableValidator<'a> {
+    pub fn new(physical_table_id: TableId, alters: Vec<&'a AlterTableExpr>) -> Self {
+        Self {
+            physical_table_id,
+            alters,
+        }
+    }
+
+    /// Validates all alter table expressions have the same schema and catalog.
+    fn validate_schema(&self) -> Result<()> {
+        let is_same_schema = self.alters.windows(2).all(|pair| {
+            pair[0].catalog_name == pair[1].catalog_name
+                && pair[0].schema_name == pair[1].schema_name
+        });
+
+        ensure!(
+            is_same_schema,
+            AlterLogicalTablesInvalidArgumentsSnafu {
+                err_msg: "Schemas of the alter table expressions are not the same"
+            }
+        );
+
+        Ok(())
+    }
+
+    /// Validates that all alter table expressions are of the supported kind.
+    /// Currently only supports `AddColumns` operations.
+    fn validate_alter_kind(&self) -> Result<()> {
+        for alter in &self.alters {
+            let kind = alter
+                .kind
+                .as_ref()
+                .context(AlterLogicalTablesInvalidArgumentsSnafu {
+                    err_msg: "Alter kind is missing",
+                })?;
+
+            let Kind::AddColumns(_) = kind else {
+                return AlterLogicalTablesInvalidArgumentsSnafu {
+                    err_msg: "Only support add columns operation",
+                }
+                .fail();
+            };
+        }
+
+        Ok(())
+    }
+
+    fn table_names(&self) -> Vec<TableReference> {
+        self.alters
+            .iter()
+            .map(|alter| {
+                TableReference::full(&alter.catalog_name, &alter.schema_name, &alter.table_name)
+            })
+            .collect()
+    }
+
+    /// Validates that the physical table info and route exist.
+    ///
+    /// This method performs the following validations:
+    /// 1. Retrieves the full table info and route for the given physical table id
+    /// 2. Ensures the table info and table route exists
+    /// 3. Verifies that the table route is actually a physical table route, not a logical one
+    ///
+    /// Returns a tuple containing the validated table info and physical table route.
+    async fn validate_physical_table(
+        &self,
+        table_metadata_manager: &TableMetadataManagerRef,
+    ) -> Result<(
+        DeserializedValueWithBytes<TableInfoValue>,
+        PhysicalTableRouteValue,
+    )> {
+        let (table_info, table_route) = table_metadata_manager
+            .get_full_table_info(self.physical_table_id)
+            .await?;
+
+        let table_info = table_info.with_context(|| TableInfoNotFoundSnafu {
+            table: format!("table id - {}", self.physical_table_id),
+        })?;
+
+        let physical_table_route = table_route
+            .context(TableRouteNotFoundSnafu {
+                table_id: self.physical_table_id,
+            })?
+            .into_inner();
+
+        let TableRouteValue::Physical(table_route) = physical_table_route else {
+            return AlterLogicalTablesInvalidArgumentsSnafu {
+                err_msg: format!(
+                    "expected a physical table but got a logical table: {:?}",
+                    self.physical_table_id
+                ),
+            }
+            .fail();
+        };
+
+        Ok((table_info, table_route))
+    }
+
+    /// Validates that all logical table routes have the same physical table id.
+    ///
+    /// This method performs the following validations:
+    /// 1. Retrieves table routes for all the given table ids.
+    /// 2. Ensures that all retrieved routes are logical table routes (not physical)
+    /// 3. Verifies that all logical table routes reference the same physical table id.
+    /// 4. Returns an error if any route is not logical or references a different physical table.
+    async fn validate_logical_table_routes(
+        &self,
+        table_route_manager: &TableRouteManager,
+        table_ids: &[TableId],
+    ) -> Result<()> {
+        let all_logical_table_routes_have_same_physical_id =
+            all_logical_table_routes_have_same_physical_id(
+                table_route_manager,
+                table_ids,
+                self.physical_table_id,
+            )
+            .await?;
+
+        ensure!(
+            all_logical_table_routes_have_same_physical_id,
+            AlterLogicalTablesInvalidArgumentsSnafu {
+                err_msg: "All the tasks should have the same physical table id"
+            }
+        );
+
+        Ok(())
+    }
+
+    /// Validates the alter logical expressions.
+    ///
+    /// This method performs the following validations:
+    /// 1. Validates that all alter table expressions have the same schema and catalog.
+    /// 2. Validates that all alter table expressions are of the supported kind.
+    /// 3. Validates that the physical table info and route exist.
+    /// 4. Validates that all logical table routes have the same physical table id.
+    ///
+    /// Returns a [ValidatorResult] containing the validation results.
+    pub async fn validate(
+        &self,
+        table_metadata_manager: &TableMetadataManagerRef,
+    ) -> Result<ValidatorResult> {
+        self.validate_schema()?;
+        self.validate_alter_kind()?;
+        let (physical_table_info, physical_table_route) =
+            self.validate_physical_table(table_metadata_manager).await?;
+        let table_names = self.table_names();
+        let table_ids =
+            get_all_table_ids_by_names(table_metadata_manager.table_name_manager(), &table_names)
+                .await?;
+        let mut table_info_values = get_all_table_info_values_by_table_ids(
+            table_metadata_manager.table_info_manager(),
+            &table_ids,
+            &table_names,
+        )
+        .await?;
+        self.validate_logical_table_routes(
+            table_metadata_manager.table_route_manager(),
+            &table_ids,
+        )
+        .await?;
+        let skip_alter = self
+            .alters
+            .iter()
+            .zip(table_info_values.iter())
+            .map(|(task, table)| skip_alter_logical_region(task, table))
+            .collect::<Vec<_>>();
+        retain_unskipped(&mut table_info_values, &skip_alter);
+        let num_skipped = skip_alter.iter().filter(|&&x| x).count();
+
+        Ok(ValidatorResult {
+            num_skipped,
+            skip_alter,
+            table_info_values,
+            physical_table_info,
+            physical_table_route,
+        })
+    }
+}
+
+/// The result of the validator.
+pub(crate) struct ValidatorResult {
+    pub(crate) num_skipped: usize,
+    pub(crate) skip_alter: Vec<bool>,
+    pub(crate) table_info_values: Vec<DeserializedValueWithBytes<TableInfoValue>>,
+    pub(crate) physical_table_info: DeserializedValueWithBytes<TableInfoValue>,
+    pub(crate) physical_table_route: PhysicalTableRouteValue,
+}
+
+/// Retains the elements that are not skipped.
+pub(crate) fn retain_unskipped<T>(target: &mut Vec<T>, skipped: &[bool]) {
+    debug_assert_eq!(target.len(), skipped.len());
+    let mut iter = skipped.iter();
+    target.retain(|_| !iter.next().unwrap());
+}
+
+/// Returns true if does not required to alter the logical region.
+fn skip_alter_logical_region(alter: &AlterTableExpr, table: &TableInfoValue) -> bool {
+    let existing_columns = table
+        .table_info
+        .meta
+        .schema
+        .column_schemas
+        .iter()
+        .map(|c| &c.name)
+        .collect::<HashSet<_>>();
+
+    let Some(kind) = alter.kind.as_ref() else {
+        return true; // Never get here since we have checked it in `validate_alter_kind`
+    };
+    let Kind::AddColumns(add_columns) = kind else {
+        return true; // Never get here since we have checked it in `validate_alter_kind`
+    };
+
+    // We only check that all columns have been finished. That is to say,
+    // if one part is finished but another part is not, it will be considered
+    // unfinished.
+    add_columns
+        .add_columns
+        .iter()
+        .map(|add_column| add_column.column_def.as_ref().map(|c| &c.name))
+        .all(|column| {
+            column
+                .map(|c| existing_columns.contains(c))
+                .unwrap_or(false)
+        })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_retain_unskipped() {
+        let mut target = vec![1, 2, 3, 4, 5];
+        let skipped = vec![false, true, false, true, false];
+        retain_unskipped(&mut target, &skipped);
+        assert_eq!(target, vec![1, 3, 5]);
+    }
+}

src/common/meta/src/ddl/alter_table.rs

@@ -12,10 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod check;
+mod executor;
 mod metadata;
 mod region_request;
-mod update_metadata;
 
 use std::vec;
 
@@ -29,30 +28,29 @@ use common_procedure::{
     Context as ProcedureContext, ContextProvider, Error as ProcedureError, LockKey, PoisonKey,
     PoisonKeys, Procedure, ProcedureId, Status, StringKey,
 };
-use common_telemetry::{debug, error, info};
+use common_telemetry::{error, info, warn};
-use futures::future::{self};
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, ResultExt};
-use store_api::storage::RegionId;
+use store_api::metadata::ColumnMetadata;
+use store_api::metric_engine_consts::TABLE_COLUMN_METADATA_EXTENSION_KEY;
 use strum::AsRefStr;
 use table::metadata::{RawTableInfo, TableId, TableInfo};
 use table::table_reference::TableReference;
 
-use crate::cache_invalidator::Context;
+use crate::ddl::alter_table::executor::AlterTableExecutor;
 use crate::ddl::utils::{
-    add_peer_context_if_needed, handle_multiple_results, map_to_procedure_error,
+    extract_column_metadatas, handle_multiple_results, map_to_procedure_error,
     sync_follower_regions, MultipleResults,
 };
 use crate::ddl::DdlContext;
 use crate::error::{AbortProcedureSnafu, NoLeaderSnafu, PutPoisonSnafu, Result, RetryLaterSnafu};
-use crate::instruction::CacheIdent;
 use crate::key::table_info::TableInfoValue;
 use crate::key::{DeserializedValueWithBytes, RegionDistribution};
 use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
 use crate::metrics;
 use crate::poison_key::table_poison_key;
 use crate::rpc::ddl::AlterTableTask;
-use crate::rpc::router::{find_leader_regions, find_leaders, region_distribution, RegionRoute};
+use crate::rpc::router::{find_leaders, region_distribution, RegionRoute};
 
 /// The alter table procedure
 pub struct AlterTableProcedure {
@@ -64,6 +62,24 @@ pub struct AlterTableProcedure {
     /// If we recover the procedure from json, then the table info value is not cached.
     /// But we already validated it in the prepare step.
     new_table_info: Option<TableInfo>,
+    /// The alter table executor.
+    executor: AlterTableExecutor,
+}
+
+/// Builds the executor from the [`AlterTableData`].
+///
+/// # Panics
+/// - If the alter kind is not set.
+fn build_executor_from_alter_expr(alter_data: &AlterTableData) -> AlterTableExecutor {
+    let table_name = alter_data.table_ref().into();
+    let table_id = alter_data.table_id;
+    let alter_kind = alter_data.task.alter_table.kind.as_ref().unwrap();
+    let new_table_name = if let Kind::RenameTable(RenameTable { new_table_name }) = alter_kind {
+        Some(new_table_name.to_string())
+    } else {
+        None
+    };
+    AlterTableExecutor::new(table_name, table_id, new_table_name)
 }
 
 impl AlterTableProcedure {
@@ -71,33 +87,42 @@ impl AlterTableProcedure {
 
     pub fn new(table_id: TableId, task: AlterTableTask, context: DdlContext) -> Result<Self> {
         task.validate()?;
+        let data = AlterTableData::new(task, table_id);
+        let executor = build_executor_from_alter_expr(&data);
         Ok(Self {
             context,
-            data: AlterTableData::new(task, table_id),
+            data,
             new_table_info: None,
+            executor,
         })
     }
 
     pub fn from_json(json: &str, context: DdlContext) -> ProcedureResult<Self> {
         let data: AlterTableData = serde_json::from_str(json).context(FromJsonSnafu)?;
+        let executor = build_executor_from_alter_expr(&data);
+
         Ok(AlterTableProcedure {
             context,
             data,
             new_table_info: None,
+            executor,
         })
     }
 
     // Checks whether the table exists.
     pub(crate) async fn on_prepare(&mut self) -> Result<Status> {
-        self.check_alter().await?;
+        self.executor
+            .on_prepare(&self.context.table_metadata_manager)
+            .await?;
         self.fill_table_info().await?;
 
-        // Validates the request and builds the new table info.
+        // Safety: filled in `fill_table_info`.
-        // We need to build the new table info here because we should ensure the alteration
-        // is valid in `UpdateMeta` state as we already altered the region.
-        // Safety: `fill_table_info()` already set it.
         let table_info_value = self.data.table_info_value.as_ref().unwrap();
-        self.new_table_info = Some(self.build_new_table_info(&table_info_value.table_info)?);
+        let new_table_info = AlterTableExecutor::validate_alter_table_expr(
+            &table_info_value.table_info,
+            self.data.task.alter_table.clone(),
+        )?;
+        self.new_table_info = Some(new_table_info);
 
         // Safety: Checked in `AlterTableProcedure::new`.
         let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap();
@@ -140,9 +165,7 @@ impl AlterTableProcedure {
 
         self.data.region_distribution =
             Some(region_distribution(&physical_table_route.region_routes));
-
         let leaders = find_leaders(&physical_table_route.region_routes);
-        let mut alter_region_tasks = Vec::with_capacity(leaders.len());
         let alter_kind = self.make_region_alter_kind()?;
 
         info!(
@@ -155,31 +178,14 @@ impl AlterTableProcedure {
         ensure!(!leaders.is_empty(), NoLeaderSnafu { table_id });
         // Puts the poison before submitting alter region requests to datanodes.
         self.put_poison(ctx_provider, procedure_id).await?;
-        for datanode in leaders {
+        let results = self
-            let requester = self.context.node_manager.datanode(&datanode).await;
+            .executor
-            let regions = find_leader_regions(&physical_table_route.region_routes, &datanode);
+            .on_alter_regions(
-
+                &self.context.node_manager,
-            for region in regions {
+                &physical_table_route.region_routes,
-                let region_id = RegionId::new(table_id, region);
+                alter_kind,
-                let request = self.make_alter_region_request(region_id, alter_kind.clone())?;
+            )
-                debug!("Submitting {request:?} to {datanode}");
+            .await;
-
-                let datanode = datanode.clone();
-                let requester = requester.clone();
-
-                alter_region_tasks.push(async move {
-                    requester
-                        .handle(request)
-                        .await
-                        .map_err(add_peer_context_if_needed(datanode))
-                });
-            }
-        }
-
-        let results = future::join_all(alter_region_tasks)
-            .await
-            .into_iter()
-            .collect::<Vec<_>>();
 
         match handle_multiple_results(results) {
             MultipleResults::PartialRetryable(error) => {
@@ -202,9 +208,9 @@ impl AlterTableProcedure {
                 })
             }
             MultipleResults::Ok(results) => {
-                self.submit_sync_region_requests(results, &physical_table_route.region_routes)
+                self.submit_sync_region_requests(&results, &physical_table_route.region_routes)
                     .await;
-                self.data.state = AlterTableState::UpdateMetadata;
+                self.handle_alter_region_response(results)?;
                 Ok(Status::executing_with_clean_poisons(true))
             }
             MultipleResults::AllNonRetryable(error) => {
@@ -220,9 +226,21 @@ impl AlterTableProcedure {
         }
     }
 
+    fn handle_alter_region_response(&mut self, mut results: Vec<RegionResponse>) -> Result<()> {
+        if let Some(column_metadatas) =
+            extract_column_metadatas(&mut results, TABLE_COLUMN_METADATA_EXTENSION_KEY)?
+        {
+            self.data.column_metadatas = column_metadatas;
+        } else {
+            warn!("altering table result doesn't contains extension key `{TABLE_COLUMN_METADATA_EXTENSION_KEY}`,leaving the table's column metadata unchanged");
+        }
+        self.data.state = AlterTableState::UpdateMetadata;
+        Ok(())
+    }
+
     async fn submit_sync_region_requests(
         &mut self,
-        results: Vec<RegionResponse>,
+        results: &[RegionResponse],
         region_routes: &[RegionRoute],
     ) {
         // Safety: filled in `prepare` step.
@@ -244,39 +262,34 @@ impl AlterTableProcedure {
     pub(crate) async fn on_update_metadata(&mut self) -> Result<Status> {
         let table_id = self.data.table_id();
         let table_ref = self.data.table_ref();
-        // Safety: checked before.
+        // Safety: filled in `fill_table_info`.
         let table_info_value = self.data.table_info_value.as_ref().unwrap();
+        // Safety: Checked in `AlterTableProcedure::new`.
+        let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap();
+
         // Gets the table info from the cache or builds it.
         let  new_info = match &self.new_table_info {
             Some(cached) => cached.clone(),
-            None => self.build_new_table_info(&table_info_value.table_info)
+            None => AlterTableExecutor::validate_alter_table_expr(
+                &table_info_value.table_info,
+                self.data.task.alter_table.clone(),
+               )
                 .inspect_err(|e| {
                     // We already check the table info in the prepare step so this should not happen.
                     error!(e; "Unable to build info for table {} in update metadata step, table_id: {}", table_ref, table_id);
                 })?,
         };
 
-        debug!(
+        // Safety: region distribution is set in `submit_alter_region_requests`.
-            "Starting update table: {} metadata, new table info {:?}",
+        self.executor
-            table_ref.to_string(),
+            .on_alter_metadata(
-            new_info
+                &self.context.table_metadata_manager,
-        );
-
-        // Safety: Checked in `AlterTableProcedure::new`.
-        let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap();
-        if let Kind::RenameTable(RenameTable { new_table_name }) = alter_kind {
-            self.on_update_metadata_for_rename(new_table_name.to_string(), table_info_value)
-                .await?;
-        } else {
-            // region distribution is set in submit_alter_region_requests
-            let region_distribution = self.data.region_distribution.as_ref().unwrap().clone();
-            self.on_update_metadata_for_alter(
-                new_info.into(),
-                region_distribution,
                 table_info_value,
+                self.data.region_distribution.as_ref(),
+                new_info.into(),
+                &self.data.column_metadatas,
             )
             .await?;
-        }
 
         info!("Updated table metadata for table {table_ref}, table_id: {table_id}, kind: {alter_kind:?}");
         self.data.state = AlterTableState::InvalidateTableCache;
@@ -285,18 +298,9 @@ impl AlterTableProcedure {
 
     /// Broadcasts the invalidating table cache instructions.
     async fn on_broadcast(&mut self) -> Result<Status> {
-        let cache_invalidator = &self.context.cache_invalidator;
+        self.executor
-
+            .invalidate_table_cache(&self.context.cache_invalidator)
-        cache_invalidator
-            .invalidate(
-                &Context::default(),
-                &[
-                    CacheIdent::TableId(self.data.table_id()),
-                    CacheIdent::TableName(self.data.table_ref().into()),
-                ],
-            )
             .await?;
-
         Ok(Status::done())
     }
 
@@ -318,6 +322,16 @@ impl AlterTableProcedure {
 
         lock_key
     }
+
+    #[cfg(test)]
+    pub(crate) fn data(&self) -> &AlterTableData {
+        &self.data
+    }
+
+    #[cfg(test)]
+    pub(crate) fn mut_data(&mut self) -> &mut AlterTableData {
+        &mut self.data
+    }
 }
 
 #[async_trait]
@@ -380,6 +394,8 @@ pub struct AlterTableData {
     state: AlterTableState,
     task: AlterTableTask,
     table_id: TableId,
+    #[serde(default)]
+    column_metadatas: Vec<ColumnMetadata>,
     /// Table info value before alteration.
     table_info_value: Option<DeserializedValueWithBytes<TableInfoValue>>,
     /// Region distribution for table in case we need to update region options.
@@ -392,6 +408,7 @@ impl AlterTableData {
             state: AlterTableState::Prepare,
             task,
             table_id,
+            column_metadatas: vec![],
             table_info_value: None,
             region_distribution: None,
         }
@@ -410,4 +427,14 @@ impl AlterTableData {
             .as_ref()
             .map(|value| &value.table_info)
     }
+
+    #[cfg(test)]
+    pub(crate) fn column_metadatas(&self) -> &[ColumnMetadata] {
+        &self.column_metadatas
+    }
+
+    #[cfg(test)]
+    pub(crate) fn set_column_metadatas(&mut self, column_metadatas: Vec<ColumnMetadata>) {
+        self.column_metadatas = column_metadatas;
+    }
 }

src/common/meta/src/ddl/alter_table/check.rs

@@ -1,62 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use api::v1::alter_table_expr::Kind;
-use api::v1::RenameTable;
-use common_catalog::format_full_table_name;
-use snafu::ensure;
-
-use crate::ddl::alter_table::AlterTableProcedure;
-use crate::error::{self, Result};
-use crate::key::table_name::TableNameKey;
-
-impl AlterTableProcedure {
-    /// Checks:
-    /// - The new table name doesn't exist (rename).
-    /// - Table exists.
-    pub(crate) async fn check_alter(&self) -> Result<()> {
-        let alter_expr = &self.data.task.alter_table;
-        let catalog = &alter_expr.catalog_name;
-        let schema = &alter_expr.schema_name;
-        let table_name = &alter_expr.table_name;
-        // Safety: Checked in `AlterTableProcedure::new`.
-        let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap();
-
-        let manager = &self.context.table_metadata_manager;
-        if let Kind::RenameTable(RenameTable { new_table_name }) = alter_kind {
-            let new_table_name_key = TableNameKey::new(catalog, schema, new_table_name);
-            let exists = manager
-                .table_name_manager()
-                .exists(new_table_name_key)
-                .await?;
-            ensure!(
-                !exists,
-                error::TableAlreadyExistsSnafu {
-                    table_name: format_full_table_name(catalog, schema, new_table_name),
-                }
-            )
-        }
-
-        let table_name_key = TableNameKey::new(catalog, schema, table_name);
-        let exists = manager.table_name_manager().exists(table_name_key).await?;
-        ensure!(
-            exists,
-            error::TableNotFoundSnafu {
-                table_name: format_full_table_name(catalog, schema, &alter_expr.table_name),
-            }
-        );
-
-        Ok(())
-    }
-}

src/common/meta/src/ddl/alter_table/executor.rs

@@ -0,0 +1,308 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+
+use api::region::RegionResponse;
+use api::v1::region::region_request::Body;
+use api::v1::region::{alter_request, AlterRequest, RegionRequest, RegionRequestHeader};
+use api::v1::AlterTableExpr;
+use common_catalog::format_full_table_name;
+use common_grpc_expr::alter_expr_to_request;
+use common_telemetry::debug;
+use common_telemetry::tracing_context::TracingContext;
+use futures::future;
+use snafu::{ensure, ResultExt};
+use store_api::metadata::ColumnMetadata;
+use store_api::storage::{RegionId, TableId};
+use table::metadata::{RawTableInfo, TableInfo};
+use table::requests::AlterKind;
+use table::table_name::TableName;
+
+use crate::cache_invalidator::{CacheInvalidatorRef, Context};
+use crate::ddl::utils::{add_peer_context_if_needed, raw_table_info};
+use crate::error::{self, Result, UnexpectedSnafu};
+use crate::instruction::CacheIdent;
+use crate::key::table_info::TableInfoValue;
+use crate::key::table_name::TableNameKey;
+use crate::key::{DeserializedValueWithBytes, RegionDistribution, TableMetadataManagerRef};
+use crate::node_manager::NodeManagerRef;
+use crate::rpc::router::{find_leaders, region_distribution, RegionRoute};
+
+/// [AlterTableExecutor] performs:
+/// - Alters the metadata of the table.
+/// - Alters regions on the datanode nodes.
+pub struct AlterTableExecutor {
+    table: TableName,
+    table_id: TableId,
+    /// The new table name if the alter kind is rename table.
+    new_table_name: Option<String>,
+}
+
+impl AlterTableExecutor {
+    /// Creates a new [`AlterTableExecutor`].
+    pub fn new(table: TableName, table_id: TableId, new_table_name: Option<String>) -> Self {
+        Self {
+            table,
+            table_id,
+            new_table_name,
+        }
+    }
+
+    /// Prepares to alter the table.
+    ///
+    /// ## Checks:
+    /// - The new table name doesn't exist (rename).
+    /// - Table exists.
+    pub(crate) async fn on_prepare(
+        &self,
+        table_metadata_manager: &TableMetadataManagerRef,
+    ) -> Result<()> {
+        let catalog = &self.table.catalog_name;
+        let schema = &self.table.schema_name;
+        let table_name = &self.table.table_name;
+
+        let manager = table_metadata_manager;
+        if let Some(new_table_name) = &self.new_table_name {
+            let new_table_name_key = TableNameKey::new(catalog, schema, new_table_name);
+            let exists = manager
+                .table_name_manager()
+                .exists(new_table_name_key)
+                .await?;
+            ensure!(
+                !exists,
+                error::TableAlreadyExistsSnafu {
+                    table_name: format_full_table_name(catalog, schema, new_table_name),
+                }
+            )
+        }
+
+        let table_name_key = TableNameKey::new(catalog, schema, table_name);
+        let exists = manager.table_name_manager().exists(table_name_key).await?;
+        ensure!(
+            exists,
+            error::TableNotFoundSnafu {
+                table_name: format_full_table_name(catalog, schema, table_name),
+            }
+        );
+
+        Ok(())
+    }
+
+    /// Validates the alter table expression and builds the new table info.
+    ///
+    /// This validation is performed early to ensure the alteration is valid before
+    /// proceeding to the `on_alter_metadata` state, where regions have already been altered.
+    /// Building the new table info here allows us to catch any issues with the
+    /// alteration before committing metadata changes.
+    pub(crate) fn validate_alter_table_expr(
+        table_info: &RawTableInfo,
+        alter_table_expr: AlterTableExpr,
+    ) -> Result<TableInfo> {
+        build_new_table_info(table_info, alter_table_expr)
+    }
+
+    /// Updates table metadata for alter table operation.
+    pub(crate) async fn on_alter_metadata(
+        &self,
+        table_metadata_manager: &TableMetadataManagerRef,
+        current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
+        region_distribution: Option<&RegionDistribution>,
+        mut raw_table_info: RawTableInfo,
+        column_metadatas: &[ColumnMetadata],
+    ) -> Result<()> {
+        let table_ref = self.table.table_ref();
+        let table_id = self.table_id;
+
+        if let Some(new_table_name) = &self.new_table_name {
+            debug!(
+                "Starting update table: {} metadata, table_id: {}, new table info: {:?}, new table name: {}",
+                table_ref, table_id, raw_table_info, new_table_name
+            );
+
+            table_metadata_manager
+                .rename_table(current_table_info_value, new_table_name.to_string())
+                .await?;
+        } else {
+            debug!(
+                "Starting update table: {} metadata, table_id: {}, new table info: {:?}",
+                table_ref, table_id, raw_table_info
+            );
+
+            ensure!(
+                region_distribution.is_some(),
+                UnexpectedSnafu {
+                    err_msg: "region distribution is not set when updating table metadata",
+                }
+            );
+
+            if !column_metadatas.is_empty() {
+                raw_table_info::update_table_info_column_ids(&mut raw_table_info, column_metadatas);
+            }
+            table_metadata_manager
+                .update_table_info(
+                    current_table_info_value,
+                    region_distribution.cloned(),
+                    raw_table_info,
+                )
+                .await?;
+        }
+
+        Ok(())
+    }
+
+    /// Alters regions on the datanode nodes.
+    pub(crate) async fn on_alter_regions(
+        &self,
+        node_manager: &NodeManagerRef,
+        region_routes: &[RegionRoute],
+        kind: Option<alter_request::Kind>,
+    ) -> Vec<Result<RegionResponse>> {
+        let region_distribution = region_distribution(region_routes);
+        let leaders = find_leaders(region_routes)
+            .into_iter()
+            .map(|p| (p.id, p))
+            .collect::<HashMap<_, _>>();
+        let total_num_region = region_distribution
+            .values()
+            .map(|r| r.leader_regions.len())
+            .sum::<usize>();
+        let mut alter_region_tasks = Vec::with_capacity(total_num_region);
+        for (datanode_id, region_role_set) in region_distribution {
+            if region_role_set.leader_regions.is_empty() {
+                continue;
+            }
+            // Safety: must exists.
+            let peer = leaders.get(&datanode_id).unwrap();
+            let requester = node_manager.datanode(peer).await;
+
+            for region_id in region_role_set.leader_regions {
+                let region_id = RegionId::new(self.table_id, region_id);
+                let request = make_alter_region_request(region_id, kind.clone());
+
+                let requester = requester.clone();
+                let peer = peer.clone();
+
+                alter_region_tasks.push(async move {
+                    requester
+                        .handle(request)
+                        .await
+                        .map_err(add_peer_context_if_needed(peer))
+                });
+            }
+        }
+
+        future::join_all(alter_region_tasks)
+            .await
+            .into_iter()
+            .collect::<Vec<_>>()
+    }
+
+    /// Invalidates cache for the table.
+    pub(crate) async fn invalidate_table_cache(
+        &self,
+        cache_invalidator: &CacheInvalidatorRef,
+    ) -> Result<()> {
+        let ctx = Context {
+            subject: Some(format!(
+                "Invalidate table cache by altering table {}, table_id: {}",
+                self.table.table_ref(),
+                self.table_id,
+            )),
+        };
+
+        cache_invalidator
+            .invalidate(
+                &ctx,
+                &[
+                    CacheIdent::TableName(self.table.clone()),
+                    CacheIdent::TableId(self.table_id),
+                ],
+            )
+            .await?;
+
+        Ok(())
+    }
+}
+
+/// Makes alter region request.
+pub(crate) fn make_alter_region_request(
+    region_id: RegionId,
+    kind: Option<alter_request::Kind>,
+) -> RegionRequest {
+    RegionRequest {
+        header: Some(RegionRequestHeader {
+            tracing_context: TracingContext::from_current_span().to_w3c(),
+            ..Default::default()
+        }),
+        body: Some(Body::Alter(AlterRequest {
+            region_id: region_id.as_u64(),
+            kind,
+            ..Default::default()
+        })),
+    }
+}
+
+/// Builds new table info after alteration.
+///
+/// This function creates a new table info by applying the alter table expression
+/// to the existing table info. For add column operations, it increments the
+/// `next_column_id` by the number of columns being added, which may result in gaps
+/// in the column id sequence.
+fn build_new_table_info(
+    table_info: &RawTableInfo,
+    alter_table_expr: AlterTableExpr,
+) -> Result<TableInfo> {
+    let table_info =
+        TableInfo::try_from(table_info.clone()).context(error::ConvertRawTableInfoSnafu)?;
+    let schema_name = &table_info.schema_name;
+    let catalog_name = &table_info.catalog_name;
+    let table_name = &table_info.name;
+    let table_id = table_info.ident.table_id;
+    let request = alter_expr_to_request(table_id, alter_table_expr)
+        .context(error::ConvertAlterTableRequestSnafu)?;
+
+    let new_meta = table_info
+        .meta
+        .builder_with_alter_kind(table_name, &request.alter_kind)
+        .context(error::TableSnafu)?
+        .build()
+        .with_context(|_| error::BuildTableMetaSnafu {
+            table_name: format_full_table_name(catalog_name, schema_name, table_name),
+        })?;
+
+    let mut new_info = table_info.clone();
+    new_info.meta = new_meta;
+    new_info.ident.version = table_info.ident.version + 1;
+    match request.alter_kind {
+        AlterKind::AddColumns { columns } => {
+            // Bumps the column id for the new columns.
+            // It may bump more than the actual number of columns added if there are
+            // existing columns, but it's fine.
+            new_info.meta.next_column_id += columns.len() as u32;
+        }
+        AlterKind::RenameTable { new_table_name } => {
+            new_info.name = new_table_name.to_string();
+        }
+        AlterKind::DropColumns { .. }
+        | AlterKind::ModifyColumnTypes { .. }
+        | AlterKind::SetTableOptions { .. }
+        | AlterKind::UnsetTableOptions { .. }
+        | AlterKind::SetIndexes { .. }
+        | AlterKind::UnsetIndexes { .. }
+        | AlterKind::DropDefaults { .. } => {}
+    }
+
+    Ok(new_info)
+}

src/common/meta/src/ddl/alter_table/region_request.rs

@@ -15,43 +15,16 @@
 use std::collections::HashSet;
 
 use api::v1::alter_table_expr::Kind;
-use api::v1::region::region_request::Body;
 use api::v1::region::{
-    alter_request, AddColumn, AddColumns, AlterRequest, DropColumn, DropColumns, RegionColumnDef,
+    alter_request, AddColumn, AddColumns, DropColumn, DropColumns, RegionColumnDef,
-    RegionRequest, RegionRequestHeader,
 };
-use common_telemetry::tracing_context::TracingContext;
 use snafu::OptionExt;
-use store_api::storage::RegionId;
 use table::metadata::RawTableInfo;
 
 use crate::ddl::alter_table::AlterTableProcedure;
 use crate::error::{InvalidProtoMsgSnafu, Result};
 
 impl AlterTableProcedure {
-    /// Makes alter region request from existing an alter kind.
-    /// Region alter request always add columns if not exist.
-    pub(crate) fn make_alter_region_request(
-        &self,
-        region_id: RegionId,
-        kind: Option<alter_request::Kind>,
-    ) -> Result<RegionRequest> {
-        // Safety: checked
-        let table_info = self.data.table_info().unwrap();
-
-        Ok(RegionRequest {
-            header: Some(RegionRequestHeader {
-                tracing_context: TracingContext::from_current_span().to_w3c(),
-                ..Default::default()
-            }),
-            body: Some(Body::Alter(AlterRequest {
-                region_id: region_id.as_u64(),
-                schema_version: table_info.ident.version,
-                kind,
-            })),
-        })
-    }
-
     /// Makes alter kind proto that all regions can reuse.
     /// Region alter request always add columns if not exist.
     pub(crate) fn make_region_alter_kind(&self) -> Result<Option<alter_request::Kind>> {
@@ -135,6 +108,8 @@ fn create_proto_alter_kind(
         Kind::UnsetTableOptions(v) => Ok(Some(alter_request::Kind::UnsetTableOptions(v.clone()))),
         Kind::SetIndex(v) => Ok(Some(alter_request::Kind::SetIndex(v.clone()))),
         Kind::UnsetIndex(v) => Ok(Some(alter_request::Kind::UnsetIndex(v.clone()))),
+        Kind::SetIndexes(v) => Ok(Some(alter_request::Kind::SetIndexes(v.clone()))),
+        Kind::UnsetIndexes(v) => Ok(Some(alter_request::Kind::UnsetIndexes(v.clone()))),
         Kind::DropDefaults(v) => Ok(Some(alter_request::Kind::DropDefaults(v.clone()))),
     }
 }
@@ -155,6 +130,7 @@ mod tests {
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
     use store_api::storage::{RegionId, TableId};
 
+    use crate::ddl::alter_table::executor::make_alter_region_request;
     use crate::ddl::alter_table::AlterTableProcedure;
     use crate::ddl::test_util::columns::TestColumnDefBuilder;
     use crate::ddl::test_util::create_table::{
@@ -261,15 +237,13 @@ mod tests {
         let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap();
         procedure.on_prepare().await.unwrap();
         let alter_kind = procedure.make_region_alter_kind().unwrap();
-        let Some(Body::Alter(alter_region_request)) = procedure
+        let Some(Body::Alter(alter_region_request)) =
-            .make_alter_region_request(region_id, alter_kind)
+            make_alter_region_request(region_id, alter_kind).body
-            .unwrap()
-            .body
         else {
             unreachable!()
         };
         assert_eq!(alter_region_request.region_id, region_id.as_u64());
-        assert_eq!(alter_region_request.schema_version, 1);
+        assert_eq!(alter_region_request.schema_version, 0);
         assert_eq!(
             alter_region_request.kind,
             Some(region::alter_request::Kind::AddColumns(
@@ -319,15 +293,13 @@ mod tests {
         let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap();
         procedure.on_prepare().await.unwrap();
         let alter_kind = procedure.make_region_alter_kind().unwrap();
-        let Some(Body::Alter(alter_region_request)) = procedure
+        let Some(Body::Alter(alter_region_request)) =
-            .make_alter_region_request(region_id, alter_kind)
+            make_alter_region_request(region_id, alter_kind).body
-            .unwrap()
-            .body
         else {
             unreachable!()
         };
         assert_eq!(alter_region_request.region_id, region_id.as_u64());
-        assert_eq!(alter_region_request.schema_version, 1);
+        assert_eq!(alter_region_request.schema_version, 0);
         assert_eq!(
             alter_region_request.kind,
             Some(region::alter_request::Kind::ModifyColumnTypes(

src/common/meta/src/ddl/alter_table/update_metadata.rs

@@ -1,103 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_grpc_expr::alter_expr_to_request;
-use snafu::ResultExt;
-use table::metadata::{RawTableInfo, TableInfo};
-use table::requests::AlterKind;
-
-use crate::ddl::alter_table::AlterTableProcedure;
-use crate::error::{self, Result};
-use crate::key::table_info::TableInfoValue;
-use crate::key::{DeserializedValueWithBytes, RegionDistribution};
-
-impl AlterTableProcedure {
-    /// Builds new table info after alteration.
-    /// It bumps the column id of the table by the number of the add column requests.
-    /// So there may be holes in the column id sequence.
-    pub(crate) fn build_new_table_info(&self, table_info: &RawTableInfo) -> Result<TableInfo> {
-        let table_info =
-            TableInfo::try_from(table_info.clone()).context(error::ConvertRawTableInfoSnafu)?;
-        let table_ref = self.data.table_ref();
-        let alter_expr = self.data.task.alter_table.clone();
-        let request = alter_expr_to_request(self.data.table_id(), alter_expr)
-            .context(error::ConvertAlterTableRequestSnafu)?;
-
-        let new_meta = table_info
-            .meta
-            .builder_with_alter_kind(table_ref.table, &request.alter_kind)
-            .context(error::TableSnafu)?
-            .build()
-            .with_context(|_| error::BuildTableMetaSnafu {
-                table_name: table_ref.table,
-            })?;
-
-        let mut new_info = table_info.clone();
-        new_info.meta = new_meta;
-        new_info.ident.version = table_info.ident.version + 1;
-        match request.alter_kind {
-            AlterKind::AddColumns { columns } => {
-                // Bumps the column id for the new columns.
-                // It may bump more than the actual number of columns added if there are
-                // existing columns, but it's fine.
-                new_info.meta.next_column_id += columns.len() as u32;
-            }
-            AlterKind::RenameTable { new_table_name } => {
-                new_info.name = new_table_name.to_string();
-            }
-            AlterKind::DropColumns { .. }
-            | AlterKind::ModifyColumnTypes { .. }
-            | AlterKind::SetTableOptions { .. }
-            | AlterKind::UnsetTableOptions { .. }
-            | AlterKind::SetIndex { .. }
-            | AlterKind::UnsetIndex { .. }
-            | AlterKind::DropDefaults { .. } => {}
-        }
-
-        Ok(new_info)
-    }
-
-    /// Updates table metadata for rename table operation.
-    pub(crate) async fn on_update_metadata_for_rename(
-        &self,
-        new_table_name: String,
-        current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
-    ) -> Result<()> {
-        let table_metadata_manager = &self.context.table_metadata_manager;
-        table_metadata_manager
-            .rename_table(current_table_info_value, new_table_name)
-            .await?;
-
-        Ok(())
-    }
-
-    /// Updates table metadata for alter table operation.
-    pub(crate) async fn on_update_metadata_for_alter(
-        &self,
-        new_table_info: RawTableInfo,
-        region_distribution: RegionDistribution,
-        current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
-    ) -> Result<()> {
-        let table_metadata_manager = &self.context.table_metadata_manager;
-        table_metadata_manager
-            .update_table_info(
-                current_table_info_value,
-                Some(region_distribution),
-                new_table_info,
-            )
-            .await?;
-
-        Ok(())
-    }
-}

src/common/meta/src/ddl/create_logical_tables.rs

@@ -27,7 +27,7 @@ use common_telemetry::{debug, error, warn};
 use futures::future;
 pub use region_request::create_region_request_builder;
 use serde::{Deserialize, Serialize};
-use snafu::{ensure, ResultExt};
+use snafu::ResultExt;
 use store_api::metadata::ColumnMetadata;
 use store_api::metric_engine_consts::ALTER_PHYSICAL_EXTENSION_KEY;
 use store_api::storage::{RegionId, RegionNumber};
@@ -35,10 +35,11 @@ use strum::AsRefStr;
 use table::metadata::{RawTableInfo, TableId};
 
 use crate::ddl::utils::{
-    add_peer_context_if_needed, map_to_procedure_error, sync_follower_regions,
+    add_peer_context_if_needed, extract_column_metadatas, map_to_procedure_error,
+    sync_follower_regions,
 };
 use crate::ddl::DdlContext;
-use crate::error::{DecodeJsonSnafu, MetadataCorruptionSnafu, Result};
+use crate::error::Result;
 use crate::key::table_route::TableRouteValue;
 use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
 use crate::metrics;
@@ -166,47 +167,23 @@ impl CreateLogicalTablesProcedure {
             .into_iter()
             .collect::<Result<Vec<_>>>()?;
 
-        // Collects response from datanodes.
+        if let Some(column_metadatas) =
-        let phy_raw_schemas = results
+            extract_column_metadatas(&mut results, ALTER_PHYSICAL_EXTENSION_KEY)?
-            .iter_mut()
+        {
-            .map(|res| res.extensions.remove(ALTER_PHYSICAL_EXTENSION_KEY))
+            self.data.physical_columns = column_metadatas;
-            .collect::<Vec<_>>();
-
-        if phy_raw_schemas.is_empty() {
-            self.submit_sync_region_requests(results, region_routes)
-                .await;
-            self.data.state = CreateTablesState::CreateMetadata;
-            return Ok(Status::executing(false));
-        }
-
-        // Verify all the physical schemas are the same
-        // Safety: previous check ensures this vec is not empty
-        let first = phy_raw_schemas.first().unwrap();
-        ensure!(
-            phy_raw_schemas.iter().all(|x| x == first),
-            MetadataCorruptionSnafu {
-                err_msg: "The physical schemas from datanodes are not the same."
-            }
-        );
-
-        // Decodes the physical raw schemas
-        if let Some(phy_raw_schemas) = first {
-            self.data.physical_columns =
-                ColumnMetadata::decode_list(phy_raw_schemas).context(DecodeJsonSnafu)?;
         } else {
             warn!("creating logical table result doesn't contains extension key `{ALTER_PHYSICAL_EXTENSION_KEY}`,leaving the physical table's schema unchanged");
         }
 
-        self.submit_sync_region_requests(results, region_routes)
+        self.submit_sync_region_requests(&results, region_routes)
             .await;
         self.data.state = CreateTablesState::CreateMetadata;
-
         Ok(Status::executing(true))
     }
 
     async fn submit_sync_region_requests(
         &self,
-        results: Vec<RegionResponse>,
+        results: &[RegionResponse],
         region_routes: &[RegionRoute],
     ) {
         if let Err(err) = sync_follower_regions(

src/common/meta/src/ddl/create_logical_tables/update_metadata.rs

@@ -22,7 +22,7 @@ use table::table_name::TableName;
 
 use crate::cache_invalidator::Context;
 use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure;
-use crate::ddl::physical_table_metadata;
+use crate::ddl::utils::raw_table_info;
 use crate::error::{Result, TableInfoNotFoundSnafu};
 use crate::instruction::CacheIdent;
 
@@ -47,7 +47,7 @@ impl CreateLogicalTablesProcedure {
         // Generates new table info
         let raw_table_info = physical_table_info.deref().table_info.clone();
 
-        let new_table_info = physical_table_metadata::build_new_physical_table_info(
+        let new_table_info = raw_table_info::build_new_physical_table_info(
             raw_table_info,
             &self.data.physical_columns,
         );

src/common/meta/src/ddl/create_table.rs

@@ -21,21 +21,24 @@ use common_error::ext::BoxedError;
 use common_procedure::error::{
     ExternalSnafu, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu,
 };
-use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status};
+use common_procedure::{Context as ProcedureContext, LockKey, Procedure, ProcedureId, Status};
-use common_telemetry::info;
 use common_telemetry::tracing_context::TracingContext;
+use common_telemetry::{info, warn};
 use futures::future::join_all;
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
+use store_api::metadata::ColumnMetadata;
+use store_api::metric_engine_consts::TABLE_COLUMN_METADATA_EXTENSION_KEY;
 use store_api::storage::{RegionId, RegionNumber};
 use strum::AsRefStr;
 use table::metadata::{RawTableInfo, TableId};
 use table::table_reference::TableReference;
 
 use crate::ddl::create_table_template::{build_template, CreateRequestBuilder};
+use crate::ddl::utils::raw_table_info::update_table_info_column_ids;
 use crate::ddl::utils::{
-    add_peer_context_if_needed, convert_region_routes_to_detecting_regions, map_to_procedure_error,
+    add_peer_context_if_needed, convert_region_routes_to_detecting_regions,
-    region_storage_path,
+    extract_column_metadatas, map_to_procedure_error, region_storage_path,
 };
 use crate::ddl::{DdlContext, TableMetadata};
 use crate::error::{self, Result};
@@ -243,14 +246,20 @@ impl CreateTableProcedure {
             }
         }
 
-        join_all(create_region_tasks)
+        let mut results = join_all(create_region_tasks)
             .await
             .into_iter()
             .collect::<Result<Vec<_>>>()?;
 
-        self.creator.data.state = CreateTableState::CreateMetadata;
+        if let Some(column_metadatas) =
+            extract_column_metadatas(&mut results, TABLE_COLUMN_METADATA_EXTENSION_KEY)?
+        {
+            self.creator.data.column_metadatas = column_metadatas;
+        } else {
+            warn!("creating table result doesn't contains extension key `{TABLE_COLUMN_METADATA_EXTENSION_KEY}`,leaving the table's column metadata unchanged");
+        }
 
-        // TODO(weny): Add more tests.
+        self.creator.data.state = CreateTableState::CreateMetadata;
         Ok(Status::executing(true))
     }
 
@@ -258,11 +267,15 @@ impl CreateTableProcedure {
     ///
     /// Abort(not-retry):
     /// - Failed to create table metadata.
-    async fn on_create_metadata(&mut self) -> Result<Status> {
+    async fn on_create_metadata(&mut self, pid: ProcedureId) -> Result<Status> {
         let table_id = self.table_id();
+        let table_ref = self.creator.data.table_ref();
         let manager = &self.context.table_metadata_manager;
 
-        let raw_table_info = self.table_info().clone();
+        let mut raw_table_info = self.table_info().clone();
+        if !self.creator.data.column_metadatas.is_empty() {
+            update_table_info_column_ids(&mut raw_table_info, &self.creator.data.column_metadatas);
+        }
         // Safety: the region_wal_options must be allocated.
         let region_wal_options = self.region_wal_options()?.clone();
         // Safety: the table_route must be allocated.
@@ -276,7 +289,10 @@ impl CreateTableProcedure {
         self.context
             .register_failure_detectors(detecting_regions)
             .await;
-        info!("Created table metadata for table {table_id}");
+        info!(
+            "Successfully created table: {}, table_id: {}, procedure_id: {}",
+            table_ref, table_id, pid
+        );
 
         self.creator.opening_regions.clear();
         Ok(Status::done_with_output(table_id))
@@ -304,7 +320,7 @@ impl Procedure for CreateTableProcedure {
         Ok(())
     }
 
-    async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
+    async fn execute(&mut self, ctx: &ProcedureContext) -> ProcedureResult<Status> {
         let state = &self.creator.data.state;
 
         let _timer = metrics::METRIC_META_PROCEDURE_CREATE_TABLE
@@ -314,7 +330,7 @@ impl Procedure for CreateTableProcedure {
         match state {
             CreateTableState::Prepare => self.on_prepare().await,
             CreateTableState::DatanodeCreateRegions => self.on_datanode_create_regions().await,
-            CreateTableState::CreateMetadata => self.on_create_metadata().await,
+            CreateTableState::CreateMetadata => self.on_create_metadata(ctx.procedure_id).await,
         }
         .map_err(map_to_procedure_error)
     }
@@ -346,6 +362,7 @@ impl TableCreator {
         Self {
             data: CreateTableData {
                 state: CreateTableState::Prepare,
+                column_metadatas: vec![],
                 task,
                 table_route: None,
                 region_wal_options: None,
@@ -407,6 +424,8 @@ pub enum CreateTableState {
 pub struct CreateTableData {
     pub state: CreateTableState,
     pub task: CreateTableTask,
+    #[serde(default)]
+    pub column_metadatas: Vec<ColumnMetadata>,
     /// None stands for not allocated yet.
     table_route: Option<PhysicalTableRouteValue>,
     /// None stands for not allocated yet.

src/common/meta/src/ddl/create_table_template.rs

@@ -14,17 +14,57 @@
 
 use std::collections::HashMap;
 
+use api::v1::column_def::try_as_column_def;
 use api::v1::region::{CreateRequest, RegionColumnDef};
 use api::v1::{ColumnDef, CreateTableExpr, SemanticType};
-use snafu::OptionExt;
+use snafu::{OptionExt, ResultExt};
-use store_api::metric_engine_consts::LOGICAL_TABLE_METADATA_KEY;
+use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME};
 use store_api::storage::{RegionId, RegionNumber};
-use table::metadata::TableId;
+use table::metadata::{RawTableInfo, TableId};
 
-use crate::error;
+use crate::error::{self, Result};
-use crate::error::Result;
 use crate::wal_options_allocator::prepare_wal_options;
 
+/// Builds a [CreateRequest] from a [RawTableInfo].
+///
+/// Note: **This method is only used for creating logical tables.**
+pub(crate) fn build_template_from_raw_table_info(
+    raw_table_info: &RawTableInfo,
+) -> Result<CreateRequest> {
+    let primary_key_indices = &raw_table_info.meta.primary_key_indices;
+    let column_defs = raw_table_info
+        .meta
+        .schema
+        .column_schemas
+        .iter()
+        .enumerate()
+        .map(|(i, c)| {
+            let is_primary_key = primary_key_indices.contains(&i);
+            let column_def = try_as_column_def(c, is_primary_key)
+                .context(error::ConvertColumnDefSnafu { column: &c.name })?;
+
+            Ok(RegionColumnDef {
+                column_def: Some(column_def),
+                // The column id will be overridden by the metric engine.
+                // So we just use the index as the column id.
+                column_id: i as u32,
+            })
+        })
+        .collect::<Result<Vec<_>>>()?;
+
+    let options = HashMap::from(&raw_table_info.meta.options);
+    let template = CreateRequest {
+        region_id: 0,
+        engine: METRIC_ENGINE_NAME.to_string(),
+        column_defs,
+        primary_key: primary_key_indices.iter().map(|i| *i as u32).collect(),
+        path: String::new(),
+        options,
+    };
+
+    Ok(template)
+}
+
 pub(crate) fn build_template(create_table_expr: &CreateTableExpr) -> Result<CreateRequest> {
     let column_defs = create_table_expr
         .column_defs

src/common/meta/src/ddl/drop_table/executor.rs

@@ -185,11 +185,15 @@ impl DropTableExecutor {
             .await
     }
 
-    /// Invalidates frontend caches
+    /// Invalidates caches for the table.
     pub async fn invalidate_table_cache(&self, ctx: &DdlContext) -> Result<()> {
         let cache_invalidator = &ctx.cache_invalidator;
         let ctx = Context {
-            subject: Some("Invalidate table cache by dropping table".to_string()),
+            subject: Some(format!(
+                "Invalidate table cache by dropping table {}, table_id: {}",
+                self.table.table_ref(),
+                self.table_id,
+            )),
         };
 
         cache_invalidator

src/common/meta/src/ddl/physical_table_metadata.rs

@@ -1,60 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashSet;
-
-use api::v1::SemanticType;
-use store_api::metadata::ColumnMetadata;
-use table::metadata::RawTableInfo;
-
-/// Generate the new physical table info.
-pub(crate) fn build_new_physical_table_info(
-    mut raw_table_info: RawTableInfo,
-    physical_columns: &[ColumnMetadata],
-) -> RawTableInfo {
-    let existing_columns = raw_table_info
-        .meta
-        .schema
-        .column_schemas
-        .iter()
-        .map(|col| col.name.clone())
-        .collect::<HashSet<_>>();
-    let primary_key_indices = &mut raw_table_info.meta.primary_key_indices;
-    let value_indices = &mut raw_table_info.meta.value_indices;
-    value_indices.clear();
-    let time_index = &mut raw_table_info.meta.schema.timestamp_index;
-    let columns = &mut raw_table_info.meta.schema.column_schemas;
-    columns.clear();
-
-    for (idx, col) in physical_columns.iter().enumerate() {
-        match col.semantic_type {
-            SemanticType::Tag => {
-                // push new primary key to the end.
-                if !existing_columns.contains(&col.column_schema.name) {
-                    primary_key_indices.push(idx);
-                }
-            }
-            SemanticType::Field => value_indices.push(idx),
-            SemanticType::Timestamp => *time_index = Some(idx),
-        }
-
-        columns.push(col.column_schema.clone());
-    }
-
-    if let Some(time_index) = *time_index {
-        raw_table_info.meta.schema.column_schemas[time_index].set_time_index();
-    }
-
-    raw_table_info
-}

src/common/meta/src/ddl/table_meta.rs

@@ -122,6 +122,7 @@ impl TableMetadataAllocator {
         );
 
         let peers = self.peer_allocator.alloc(regions).await?;
+        debug!("Allocated peers {:?} for table {}", peers, table_id);
         let region_routes = task
             .partitions
             .iter()
@@ -174,6 +175,10 @@ impl TableMetadataAllocator {
             region_wal_options,
         })
     }
+
+    pub fn table_id_sequence(&self) -> SequenceRef {
+        self.table_id_sequence.clone()
+    }
 }
 
 pub type PeerAllocatorRef = Arc<dyn PeerAllocator>;

src/common/meta/src/ddl/test_util.rs

@@ -17,6 +17,7 @@ pub mod columns;
 pub mod create_table;
 pub mod datanode_handler;
 pub mod flownode_handler;
+pub mod region_metadata;
 
 use std::assert_matches::assert_matches;
 use std::collections::HashMap;
@@ -24,7 +25,14 @@ use std::collections::HashMap;
 use api::v1::meta::Partition;
 use api::v1::{ColumnDataType, SemanticType};
 use common_procedure::Status;
-use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME};
+use datatypes::prelude::ConcreteDataType;
+use datatypes::schema::ColumnSchema;
+use store_api::metadata::ColumnMetadata;
+use store_api::metric_engine_consts::{
+    DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME, LOGICAL_TABLE_METADATA_KEY,
+    METRIC_ENGINE_NAME,
+};
+use store_api::storage::consts::ReservedColumnId;
 use table::metadata::{RawTableInfo, TableId};
 
 use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure;
@@ -146,6 +154,7 @@ pub fn test_create_logical_table_task(name: &str) -> CreateTableTask {
     }
 }
 
+/// Creates a physical table task with a single region.
 pub fn test_create_physical_table_task(name: &str) -> CreateTableTask {
     let create_table = TestCreateTableExprBuilder::default()
         .column_defs([
@@ -182,3 +191,95 @@ pub fn test_create_physical_table_task(name: &str) -> CreateTableTask {
         table_info,
     }
 }
+
+/// Creates a column metadata list with tag fields.
+pub fn test_column_metadatas(tag_fields: &[&str]) -> Vec<ColumnMetadata> {
+    let mut output = Vec::with_capacity(tag_fields.len() + 4);
+    output.extend([
+        ColumnMetadata {
+            column_schema: ColumnSchema::new(
+                "ts",
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                false,
+            ),
+            semantic_type: SemanticType::Timestamp,
+            column_id: 0,
+        },
+        ColumnMetadata {
+            column_schema: ColumnSchema::new("value", ConcreteDataType::float64_datatype(), false),
+            semantic_type: SemanticType::Field,
+            column_id: 1,
+        },
+        ColumnMetadata {
+            column_schema: ColumnSchema::new(
+                DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                false,
+            ),
+            semantic_type: SemanticType::Tag,
+            column_id: ReservedColumnId::table_id(),
+        },
+        ColumnMetadata {
+            column_schema: ColumnSchema::new(
+                DATA_SCHEMA_TSID_COLUMN_NAME,
+                ConcreteDataType::float64_datatype(),
+                false,
+            ),
+            semantic_type: SemanticType::Tag,
+            column_id: ReservedColumnId::tsid(),
+        },
+    ]);
+
+    for (i, name) in tag_fields.iter().enumerate() {
+        output.push(ColumnMetadata {
+            column_schema: ColumnSchema::new(
+                name.to_string(),
+                ConcreteDataType::string_datatype(),
+                true,
+            ),
+            semantic_type: SemanticType::Tag,
+            column_id: (i + 2) as u32,
+        });
+    }
+
+    output
+}
+
+/// Asserts the column names.
+pub fn assert_column_name(table_info: &RawTableInfo, expected_column_names: &[&str]) {
+    assert_eq!(
+        table_info
+            .meta
+            .schema
+            .column_schemas
+            .iter()
+            .map(|c| c.name.to_string())
+            .collect::<Vec<_>>(),
+        expected_column_names
+    );
+}
+
+/// Asserts the column metadatas
+pub fn assert_column_name_and_id(column_metadatas: &[ColumnMetadata], expected: &[(&str, u32)]) {
+    assert_eq!(expected.len(), column_metadatas.len());
+    for (name, id) in expected {
+        let column_metadata = column_metadatas
+            .iter()
+            .find(|c| c.column_id == *id)
+            .unwrap();
+        assert_eq!(column_metadata.column_schema.name, *name);
+    }
+}
+
+/// Gets the raw table info.
+pub async fn get_raw_table_info(ddl_context: &DdlContext, table_id: TableId) -> RawTableInfo {
+    ddl_context
+        .table_metadata_manager
+        .table_info_manager()
+        .get(table_id)
+        .await
+        .unwrap()
+        .unwrap()
+        .into_inner()
+        .table_info
+}

src/common/meta/src/ddl/test_util/create_table.rs

@@ -132,6 +132,7 @@ pub fn build_raw_table_info_from_expr(expr: &CreateTableExpr) -> RawTableInfo {
             options: TableOptions::try_from_iter(&expr.table_options).unwrap(),
             created_on: DateTime::default(),
             partition_key_indices: vec![],
+            column_ids: vec![],
         },
         table_type: TableType::Base,
     }

src/common/meta/src/ddl/test_util/datanode_handler.rs

@@ -12,7 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
+use std::sync::Arc;
+
 use api::region::RegionResponse;
+use api::v1::region::region_request::Body;
 use api::v1::region::RegionRequest;
 use common_error::ext::{BoxedError, ErrorExt, StackError};
 use common_error::status_code::StatusCode;
@@ -20,6 +24,8 @@ use common_query::request::QueryRequest;
 use common_recordbatch::SendableRecordBatchStream;
 use common_telemetry::debug;
 use snafu::{ResultExt, Snafu};
+use store_api::metadata::RegionMetadata;
+use store_api::storage::RegionId;
 use tokio::sync::mpsc;
 
 use crate::error::{self, Error, Result};
@@ -32,6 +38,7 @@ impl MockDatanodeHandler for () {
         Ok(RegionResponse {
             affected_rows: 0,
             extensions: Default::default(),
+            metadata: Vec::new(),
         })
     }
 
@@ -44,10 +51,13 @@ impl MockDatanodeHandler for () {
     }
 }
 
+type RegionRequestHandler =
+    Arc<dyn Fn(Peer, RegionRequest) -> Result<RegionResponse> + Send + Sync>;
+
 #[derive(Clone)]
 pub struct DatanodeWatcher {
     sender: mpsc::Sender<(Peer, RegionRequest)>,
-    handler: Option<fn(Peer, RegionRequest) -> Result<RegionResponse>>,
+    handler: Option<RegionRequestHandler>,
 }
 
 impl DatanodeWatcher {
@@ -60,9 +70,9 @@ impl DatanodeWatcher {
 
     pub fn with_handler(
         mut self,
-        user_handler: fn(Peer, RegionRequest) -> Result<RegionResponse>,
+        user_handler: impl Fn(Peer, RegionRequest) -> Result<RegionResponse> + Send + Sync + 'static,
     ) -> Self {
-        self.handler = Some(user_handler);
+        self.handler = Some(Arc::new(user_handler));
         self
     }
 }
@@ -75,7 +85,7 @@ impl MockDatanodeHandler for DatanodeWatcher {
             .send((peer.clone(), request.clone()))
             .await
             .unwrap();
-        if let Some(handler) = self.handler {
+        if let Some(handler) = self.handler.as_ref() {
             handler(peer.clone(), request)
         } else {
             Ok(RegionResponse::new(0))
@@ -272,3 +282,47 @@ impl MockDatanodeHandler for AllFailureDatanodeHandler {
         unreachable!()
     }
 }
+
+#[derive(Clone)]
+pub struct ListMetadataDatanodeHandler {
+    pub region_metadatas: HashMap<RegionId, Option<RegionMetadata>>,
+}
+
+impl ListMetadataDatanodeHandler {
+    pub fn new(region_metadatas: HashMap<RegionId, Option<RegionMetadata>>) -> Self {
+        Self { region_metadatas }
+    }
+}
+
+#[async_trait::async_trait]
+impl MockDatanodeHandler for ListMetadataDatanodeHandler {
+    async fn handle(&self, _peer: &Peer, request: RegionRequest) -> Result<RegionResponse> {
+        let Some(Body::ListMetadata(req)) = request.body else {
+            unreachable!()
+        };
+        let mut response = RegionResponse::new(0);
+
+        let mut output = Vec::with_capacity(req.region_ids.len());
+        for region_id in req.region_ids {
+            match self.region_metadatas.get(&RegionId::from_u64(region_id)) {
+                Some(metadata) => {
+                    output.push(metadata.clone());
+                }
+                None => {
+                    output.push(None);
+                }
+            }
+        }
+
+        response.metadata = serde_json::to_vec(&output).unwrap();
+        Ok(response)
+    }
+
+    async fn handle_query(
+        &self,
+        _peer: &Peer,
+        _request: QueryRequest,
+    ) -> Result<SendableRecordBatchStream> {
+        unreachable!()
+    }
+}

src/common/meta/src/ddl/test_util/region_metadata.rs

@@ -0,0 +1,34 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::SemanticType;
+use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder};
+use store_api::storage::RegionId;
+
+/// Builds a region metadata with the given column metadatas.
+pub fn build_region_metadata(
+    region_id: RegionId,
+    column_metadatas: &[ColumnMetadata],
+) -> RegionMetadata {
+    let mut builder = RegionMetadataBuilder::new(region_id);
+    let mut primary_key = vec![];
+    for column_metadata in column_metadatas {
+        builder.push_column_metadata(column_metadata.clone());
+        if column_metadata.semantic_type == SemanticType::Tag {
+            primary_key.push(column_metadata.column_id);
+        }
+    }
+    builder.primary_key(primary_key);
+    builder.build().unwrap()
+}

src/common/meta/src/ddl/tests/alter_logical_tables.rs

@@ -23,17 +23,20 @@ use api::v1::{ColumnDataType, SemanticType};
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_procedure::{Procedure, ProcedureId, Status};
 use common_procedure_test::MockContextProvider;
-use store_api::metric_engine_consts::MANIFEST_INFO_EXTENSION_KEY;
+use store_api::metadata::ColumnMetadata;
+use store_api::metric_engine_consts::{ALTER_PHYSICAL_EXTENSION_KEY, MANIFEST_INFO_EXTENSION_KEY};
 use store_api::region_engine::RegionManifestInfo;
+use store_api::storage::consts::ReservedColumnId;
 use store_api::storage::RegionId;
 use tokio::sync::mpsc;
 
 use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
 use crate::ddl::test_util::alter_table::TestAlterTableExprBuilder;
 use crate::ddl::test_util::columns::TestColumnDefBuilder;
-use crate::ddl::test_util::datanode_handler::{DatanodeWatcher, NaiveDatanodeHandler};
+use crate::ddl::test_util::datanode_handler::DatanodeWatcher;
 use crate::ddl::test_util::{
-    create_logical_table, create_physical_table, create_physical_table_metadata,
+    assert_column_name, create_logical_table, create_physical_table,
+    create_physical_table_metadata, get_raw_table_info, test_column_metadatas,
     test_create_physical_table_task,
 };
 use crate::error::Error::{AlterLogicalTablesInvalidArguments, TableNotFound};
@@ -96,6 +99,52 @@ fn make_alter_logical_table_rename_task(
     }
 }
 
+fn make_alters_request_handler(
+    column_metadatas: Vec<ColumnMetadata>,
+) -> impl Fn(Peer, RegionRequest) -> Result<RegionResponse> {
+    move |_peer: Peer, request: RegionRequest| {
+        if let region_request::Body::Alters(_) = request.body.unwrap() {
+            let mut response = RegionResponse::new(0);
+            // Default region id for physical table.
+            let region_id = RegionId::new(1000, 1);
+            response.extensions.insert(
+                MANIFEST_INFO_EXTENSION_KEY.to_string(),
+                RegionManifestInfo::encode_list(&[(
+                    region_id,
+                    RegionManifestInfo::metric(1, 0, 2, 0),
+                )])
+                .unwrap(),
+            );
+            response.extensions.insert(
+                ALTER_PHYSICAL_EXTENSION_KEY.to_string(),
+                ColumnMetadata::encode_list(&column_metadatas).unwrap(),
+            );
+            return Ok(response);
+        }
+        Ok(RegionResponse::new(0))
+    }
+}
+
+fn assert_alters_request(
+    peer: Peer,
+    request: RegionRequest,
+    expected_peer_id: u64,
+    expected_region_ids: &[RegionId],
+) {
+    assert_eq!(peer.id, expected_peer_id,);
+    let Some(region_request::Body::Alters(req)) = request.body else {
+        unreachable!();
+    };
+    for (i, region_id) in expected_region_ids.iter().enumerate() {
+        assert_eq!(
+            req.requests[i].region_id,
+            *region_id,
+            "actual region id: {}",
+            RegionId::from_u64(req.requests[i].region_id)
+        );
+    }
+}
+
 #[tokio::test]
 async fn test_on_prepare_check_schema() {
     let node_manager = Arc::new(MockDatanodeManager::new(()));
@@ -205,15 +254,20 @@ async fn test_on_prepare() {
 
 #[tokio::test]
 async fn test_on_update_metadata() {
-    let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
+    common_telemetry::init_default_ut_logging();
+    let (tx, mut rx) = mpsc::channel(8);
+    let test_column_metadatas = test_column_metadatas(&["new_col", "mew_col"]);
+    let datanode_handler =
+        DatanodeWatcher::new(tx).with_handler(make_alters_request_handler(test_column_metadatas));
+    let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
     let ddl_context = new_ddl_context(node_manager);
 
     // Creates physical table
     let phy_id = create_physical_table(&ddl_context, "phy").await;
     // Creates 3 logical tables
-    create_logical_table(ddl_context.clone(), phy_id, "table1").await;
+    let logical_table1_id = create_logical_table(ddl_context.clone(), phy_id, "table1").await;
-    create_logical_table(ddl_context.clone(), phy_id, "table2").await;
+    let logical_table2_id = create_logical_table(ddl_context.clone(), phy_id, "table2").await;
-    create_logical_table(ddl_context.clone(), phy_id, "table3").await;
+    let logical_table3_id = create_logical_table(ddl_context.clone(), phy_id, "table3").await;
     create_logical_table(ddl_context.clone(), phy_id, "table4").await;
     create_logical_table(ddl_context.clone(), phy_id, "table5").await;
 
@@ -223,7 +277,7 @@ async fn test_on_update_metadata() {
         make_alter_logical_table_add_column_task(None, "table3", vec!["new_col".to_string()]),
     ];
 
-    let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context);
+    let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone());
     let mut status = procedure.on_prepare().await.unwrap();
     assert_matches!(
         status,
@@ -255,18 +309,52 @@ async fn test_on_update_metadata() {
             clean_poisons: false
         }
     );
+    let (peer, request) = rx.try_recv().unwrap();
+    rx.try_recv().unwrap_err();
+    assert_alters_request(
+        peer,
+        request,
+        0,
+        &[
+            RegionId::new(logical_table1_id, 0),
+            RegionId::new(logical_table2_id, 0),
+            RegionId::new(logical_table3_id, 0),
+        ],
+    );
+
+    let table_info = get_raw_table_info(&ddl_context, phy_id).await;
+    assert_column_name(
+        &table_info,
+        &["ts", "value", "__table_id", "__tsid", "new_col", "mew_col"],
+    );
+    assert_eq!(
+        table_info.meta.column_ids,
+        vec![
+            0,
+            1,
+            ReservedColumnId::table_id(),
+            ReservedColumnId::tsid(),
+            2,
+            3
+        ]
+    );
 }
 
 #[tokio::test]
 async fn test_on_part_duplicate_alter_request() {
-    let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
+    common_telemetry::init_default_ut_logging();
-    let ddl_context = new_ddl_context(node_manager);
+    let (tx, mut rx) = mpsc::channel(8);
+    let column_metadatas = test_column_metadatas(&["col_0"]);
+    let handler =
+        DatanodeWatcher::new(tx).with_handler(make_alters_request_handler(column_metadatas));
+    let node_manager = Arc::new(MockDatanodeManager::new(handler));
+    let mut ddl_context = new_ddl_context(node_manager);
 
     // Creates physical table
     let phy_id = create_physical_table(&ddl_context, "phy").await;
     // Creates 3 logical tables
-    create_logical_table(ddl_context.clone(), phy_id, "table1").await;
+    let logical_table1_id = create_logical_table(ddl_context.clone(), phy_id, "table1").await;
-    create_logical_table(ddl_context.clone(), phy_id, "table2").await;
+    let logical_table2_id = create_logical_table(ddl_context.clone(), phy_id, "table2").await;
 
     let tasks = vec![
         make_alter_logical_table_add_column_task(None, "table1", vec!["col_0".to_string()]),
@@ -305,6 +393,40 @@ async fn test_on_part_duplicate_alter_request() {
             clean_poisons: false
         }
     );
+    let (peer, request) = rx.try_recv().unwrap();
+    rx.try_recv().unwrap_err();
+    assert_alters_request(
+        peer,
+        request,
+        0,
+        &[
+            RegionId::new(logical_table1_id, 0),
+            RegionId::new(logical_table2_id, 0),
+        ],
+    );
+
+    let table_info = get_raw_table_info(&ddl_context, phy_id).await;
+    assert_column_name(
+        &table_info,
+        &["ts", "value", "__table_id", "__tsid", "col_0"],
+    );
+    assert_eq!(
+        table_info.meta.column_ids,
+        vec![
+            0,
+            1,
+            ReservedColumnId::table_id(),
+            ReservedColumnId::tsid(),
+            2
+        ]
+    );
+
+    let (tx, mut rx) = mpsc::channel(8);
+    let column_metadatas = test_column_metadatas(&["col_0", "new_col_1", "new_col_2"]);
+    let handler =
+        DatanodeWatcher::new(tx).with_handler(make_alters_request_handler(column_metadatas));
+    let node_manager = Arc::new(MockDatanodeManager::new(handler));
+    ddl_context.node_manager = node_manager;
 
     // re-alter
     let tasks = vec![
@@ -357,6 +479,44 @@ async fn test_on_part_duplicate_alter_request() {
         }
     );
 
+    let (peer, request) = rx.try_recv().unwrap();
+    rx.try_recv().unwrap_err();
+    assert_alters_request(
+        peer,
+        request,
+        0,
+        &[
+            RegionId::new(logical_table1_id, 0),
+            RegionId::new(logical_table2_id, 0),
+        ],
+    );
+
+    let table_info = get_raw_table_info(&ddl_context, phy_id).await;
+    assert_column_name(
+        &table_info,
+        &[
+            "ts",
+            "value",
+            "__table_id",
+            "__tsid",
+            "col_0",
+            "new_col_1",
+            "new_col_2",
+        ],
+    );
+    assert_eq!(
+        table_info.meta.column_ids,
+        vec![
+            0,
+            1,
+            ReservedColumnId::table_id(),
+            ReservedColumnId::tsid(),
+            2,
+            3,
+            4,
+        ]
+    );
+
     let table_name_keys = vec![
         TableNameKey::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "table1"),
         TableNameKey::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "table2"),
@@ -422,27 +582,13 @@ async fn test_on_part_duplicate_alter_request() {
     );
 }
 
-fn alters_request_handler(_peer: Peer, request: RegionRequest) -> Result<RegionResponse> {
-    if let region_request::Body::Alters(_) = request.body.unwrap() {
-        let mut response = RegionResponse::new(0);
-        // Default region id for physical table.
-        let region_id = RegionId::new(1000, 1);
-        response.extensions.insert(
-            MANIFEST_INFO_EXTENSION_KEY.to_string(),
-            RegionManifestInfo::encode_list(&[(region_id, RegionManifestInfo::metric(1, 0, 2, 0))])
-                .unwrap(),
-        );
-        return Ok(response);
-    }
-
-    Ok(RegionResponse::new(0))
-}
-
 #[tokio::test]
 async fn test_on_submit_alter_region_request() {
     common_telemetry::init_default_ut_logging();
     let (tx, mut rx) = mpsc::channel(8);
-    let handler = DatanodeWatcher::new(tx).with_handler(alters_request_handler);
+    let column_metadatas = test_column_metadatas(&["new_col", "mew_col"]);
+    let handler =
+        DatanodeWatcher::new(tx).with_handler(make_alters_request_handler(column_metadatas));
     let node_manager = Arc::new(MockDatanodeManager::new(handler));
     let ddl_context = new_ddl_context(node_manager);
 

src/common/meta/src/ddl/tests/alter_table.rs

@@ -30,7 +30,12 @@ use common_error::status_code::StatusCode;
 use common_procedure::store::poison_store::PoisonStore;
 use common_procedure::{ProcedureId, Status};
 use common_procedure_test::MockContextProvider;
-use store_api::metric_engine_consts::MANIFEST_INFO_EXTENSION_KEY;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::schema::ColumnSchema;
+use store_api::metadata::ColumnMetadata;
+use store_api::metric_engine_consts::{
+    MANIFEST_INFO_EXTENSION_KEY, TABLE_COLUMN_METADATA_EXTENSION_KEY,
+};
 use store_api::region_engine::RegionManifestInfo;
 use store_api::storage::RegionId;
 use table::requests::TTL_KEY;
@@ -43,6 +48,7 @@ use crate::ddl::test_util::datanode_handler::{
     AllFailureDatanodeHandler, DatanodeWatcher, PartialSuccessDatanodeHandler,
     RequestOutdatedErrorDatanodeHandler,
 };
+use crate::ddl::test_util::{assert_column_name, assert_column_name_and_id};
 use crate::error::{Error, Result};
 use crate::key::datanode_table::DatanodeTableKey;
 use crate::key::table_name::TableNameKey;
@@ -179,6 +185,30 @@ fn alter_request_handler(_peer: Peer, request: RegionRequest) -> Result<RegionRe
             RegionManifestInfo::encode_list(&[(region_id, RegionManifestInfo::mito(1, 1))])
                 .unwrap(),
         );
+        response.extensions.insert(
+            TABLE_COLUMN_METADATA_EXTENSION_KEY.to_string(),
+            ColumnMetadata::encode_list(&[
+                ColumnMetadata {
+                    column_schema: ColumnSchema::new(
+                        "ts",
+                        ConcreteDataType::timestamp_millisecond_datatype(),
+                        false,
+                    ),
+                    semantic_type: SemanticType::Timestamp,
+                    column_id: 0,
+                },
+                ColumnMetadata {
+                    column_schema: ColumnSchema::new(
+                        "host",
+                        ConcreteDataType::float64_datatype(),
+                        false,
+                    ),
+                    semantic_type: SemanticType::Tag,
+                    column_id: 1,
+                },
+            ])
+            .unwrap(),
+        );
         return Ok(response);
     }
 
@@ -187,6 +217,7 @@ fn alter_request_handler(_peer: Peer, request: RegionRequest) -> Result<RegionRe
 
 #[tokio::test]
 async fn test_on_submit_alter_request() {
+    common_telemetry::init_default_ut_logging();
     let (tx, mut rx) = mpsc::channel(8);
     let datanode_handler = DatanodeWatcher::new(tx).with_handler(alter_request_handler);
     let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
@@ -234,6 +265,8 @@ async fn test_on_submit_alter_request() {
     assert_sync_request(peer, request, 4, RegionId::new(table_id, 2), 1);
     let (peer, request) = results.remove(0);
     assert_sync_request(peer, request, 5, RegionId::new(table_id, 1), 1);
+    let column_metadatas = procedure.data().column_metadatas();
+    assert_column_name_and_id(column_metadatas, &[("ts", 0), ("host", 1)]);
 }
 
 #[tokio::test]
@@ -378,6 +411,7 @@ async fn test_on_update_metadata_rename() {
 
 #[tokio::test]
 async fn test_on_update_metadata_add_columns() {
+    common_telemetry::init_default_ut_logging();
     let node_manager = Arc::new(MockDatanodeManager::new(()));
     let ddl_context = new_ddl_context(node_manager);
     let table_name = "foo";
@@ -431,6 +465,34 @@ async fn test_on_update_metadata_add_columns() {
         .submit_alter_region_requests(procedure_id, provider.as_ref())
         .await
         .unwrap();
+    // Returned column metadatas is empty.
+    assert!(procedure.data().column_metadatas().is_empty());
+    procedure.mut_data().set_column_metadatas(vec![
+        ColumnMetadata {
+            column_schema: ColumnSchema::new(
+                "ts",
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                false,
+            ),
+            semantic_type: SemanticType::Timestamp,
+            column_id: 0,
+        },
+        ColumnMetadata {
+            column_schema: ColumnSchema::new("host", ConcreteDataType::float64_datatype(), false),
+            semantic_type: SemanticType::Tag,
+            column_id: 1,
+        },
+        ColumnMetadata {
+            column_schema: ColumnSchema::new("cpu", ConcreteDataType::float64_datatype(), false),
+            semantic_type: SemanticType::Tag,
+            column_id: 2,
+        },
+        ColumnMetadata {
+            column_schema: ColumnSchema::new("my_tag3", ConcreteDataType::string_datatype(), true),
+            semantic_type: SemanticType::Tag,
+            column_id: 3,
+        },
+    ]);
     procedure.on_update_metadata().await.unwrap();
 
     let table_info = ddl_context
@@ -447,6 +509,8 @@ async fn test_on_update_metadata_add_columns() {
         table_info.meta.schema.column_schemas.len() as u32,
         table_info.meta.next_column_id
     );
+    assert_column_name(&table_info, &["ts", "host", "cpu", "my_tag3"]);
+    assert_eq!(table_info.meta.column_ids, vec![0, 1, 2, 3]);
 }
 
 #[tokio::test]

src/common/meta/src/ddl/tests/create_logical_tables.rs

@@ -23,15 +23,18 @@ use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_procedure::{Context as ProcedureContext, Procedure, ProcedureId, Status};
 use common_procedure_test::MockContextProvider;
-use store_api::metric_engine_consts::MANIFEST_INFO_EXTENSION_KEY;
+use store_api::metadata::ColumnMetadata;
+use store_api::metric_engine_consts::{ALTER_PHYSICAL_EXTENSION_KEY, MANIFEST_INFO_EXTENSION_KEY};
 use store_api::region_engine::RegionManifestInfo;
+use store_api::storage::consts::ReservedColumnId;
 use store_api::storage::RegionId;
 use tokio::sync::mpsc;
 
 use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure;
 use crate::ddl::test_util::datanode_handler::{DatanodeWatcher, NaiveDatanodeHandler};
 use crate::ddl::test_util::{
-    create_physical_table_metadata, test_create_logical_table_task, test_create_physical_table_task,
+    assert_column_name, create_physical_table_metadata, get_raw_table_info, test_column_metadatas,
+    test_create_logical_table_task, test_create_physical_table_task,
 };
 use crate::ddl::TableMetadata;
 use crate::error::{Error, Result};
@@ -39,6 +42,54 @@ use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue};
 use crate::rpc::router::{Region, RegionRoute};
 use crate::test_util::{new_ddl_context, MockDatanodeManager};
 
+fn make_creates_request_handler(
+    column_metadatas: Vec<ColumnMetadata>,
+) -> impl Fn(Peer, RegionRequest) -> Result<RegionResponse> {
+    move |_peer, request| {
+        let _ = _peer;
+        if let region_request::Body::Creates(_) = request.body.unwrap() {
+            let mut response = RegionResponse::new(0);
+            // Default region id for physical table.
+            let region_id = RegionId::new(1024, 1);
+            response.extensions.insert(
+                MANIFEST_INFO_EXTENSION_KEY.to_string(),
+                RegionManifestInfo::encode_list(&[(
+                    region_id,
+                    RegionManifestInfo::metric(1, 0, 2, 0),
+                )])
+                .unwrap(),
+            );
+            response.extensions.insert(
+                ALTER_PHYSICAL_EXTENSION_KEY.to_string(),
+                ColumnMetadata::encode_list(&column_metadatas).unwrap(),
+            );
+            return Ok(response);
+        }
+
+        Ok(RegionResponse::new(0))
+    }
+}
+
+fn assert_creates_request(
+    peer: Peer,
+    request: RegionRequest,
+    expected_peer_id: u64,
+    expected_region_ids: &[RegionId],
+) {
+    assert_eq!(peer.id, expected_peer_id,);
+    let Some(region_request::Body::Creates(req)) = request.body else {
+        unreachable!();
+    };
+    for (i, region_id) in expected_region_ids.iter().enumerate() {
+        assert_eq!(
+            req.requests[i].region_id,
+            *region_id,
+            "actual region id: {}",
+            RegionId::from_u64(req.requests[i].region_id)
+        );
+    }
+}
+
 #[tokio::test]
 async fn test_on_prepare_physical_table_not_found() {
     let node_manager = Arc::new(MockDatanodeManager::new(()));
@@ -227,7 +278,12 @@ async fn test_on_prepare_part_logical_tables_exist() {
 
 #[tokio::test]
 async fn test_on_create_metadata() {
-    let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
+    common_telemetry::init_default_ut_logging();
+    let (tx, mut rx) = mpsc::channel(8);
+    let column_metadatas = test_column_metadatas(&["host", "cpu"]);
+    let datanode_handler =
+        DatanodeWatcher::new(tx).with_handler(make_creates_request_handler(column_metadatas));
+    let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
     let ddl_context = new_ddl_context(node_manager);
     // Prepares physical table metadata.
     let mut create_physical_table_task = test_create_physical_table_task("phy_table");
@@ -255,7 +311,7 @@ async fn test_on_create_metadata() {
     let mut procedure = CreateLogicalTablesProcedure::new(
         vec![task, yet_another_task],
         physical_table_id,
-        ddl_context,
+        ddl_context.clone(),
     );
     let status = procedure.on_prepare().await.unwrap();
     assert_matches!(
@@ -274,11 +330,42 @@ async fn test_on_create_metadata() {
     let status = procedure.execute(&ctx).await.unwrap();
     let table_ids = status.downcast_output_ref::<Vec<u32>>().unwrap();
     assert_eq!(*table_ids, vec![1025, 1026]);
+
+    let (peer, request) = rx.try_recv().unwrap();
+    rx.try_recv().unwrap_err();
+    assert_creates_request(
+        peer,
+        request,
+        0,
+        &[RegionId::new(1025, 0), RegionId::new(1026, 0)],
+    );
+
+    let table_info = get_raw_table_info(&ddl_context, table_id).await;
+    assert_column_name(
+        &table_info,
+        &["ts", "value", "__table_id", "__tsid", "host", "cpu"],
+    );
+    assert_eq!(
+        table_info.meta.column_ids,
+        vec![
+            0,
+            1,
+            ReservedColumnId::table_id(),
+            ReservedColumnId::tsid(),
+            2,
+            3
+        ]
+    );
 }
 
 #[tokio::test]
 async fn test_on_create_metadata_part_logical_tables_exist() {
-    let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
+    common_telemetry::init_default_ut_logging();
+    let (tx, mut rx) = mpsc::channel(8);
+    let column_metadatas = test_column_metadatas(&["host", "cpu"]);
+    let datanode_handler =
+        DatanodeWatcher::new(tx).with_handler(make_creates_request_handler(column_metadatas));
+    let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
     let ddl_context = new_ddl_context(node_manager);
     // Prepares physical table metadata.
     let mut create_physical_table_task = test_create_physical_table_task("phy_table");
@@ -317,7 +404,7 @@ async fn test_on_create_metadata_part_logical_tables_exist() {
     let mut procedure = CreateLogicalTablesProcedure::new(
         vec![task, non_exist_task],
         physical_table_id,
-        ddl_context,
+        ddl_context.clone(),
     );
     let status = procedure.on_prepare().await.unwrap();
     assert_matches!(
@@ -336,6 +423,27 @@ async fn test_on_create_metadata_part_logical_tables_exist() {
     let status = procedure.execute(&ctx).await.unwrap();
     let table_ids = status.downcast_output_ref::<Vec<u32>>().unwrap();
     assert_eq!(*table_ids, vec![8192, 1025]);
+
+    let (peer, request) = rx.try_recv().unwrap();
+    rx.try_recv().unwrap_err();
+    assert_creates_request(peer, request, 0, &[RegionId::new(1025, 0)]);
+
+    let table_info = get_raw_table_info(&ddl_context, table_id).await;
+    assert_column_name(
+        &table_info,
+        &["ts", "value", "__table_id", "__tsid", "host", "cpu"],
+    );
+    assert_eq!(
+        table_info.meta.column_ids,
+        vec![
+            0,
+            1,
+            ReservedColumnId::table_id(),
+            ReservedColumnId::tsid(),
+            2,
+            3
+        ]
+    );
 }
 
 #[tokio::test]
@@ -399,27 +507,13 @@ async fn test_on_create_metadata_err() {
     assert!(!error.is_retry_later());
 }
 
-fn creates_request_handler(_peer: Peer, request: RegionRequest) -> Result<RegionResponse> {
-    if let region_request::Body::Creates(_) = request.body.unwrap() {
-        let mut response = RegionResponse::new(0);
-        // Default region id for physical table.
-        let region_id = RegionId::new(1024, 1);
-        response.extensions.insert(
-            MANIFEST_INFO_EXTENSION_KEY.to_string(),
-            RegionManifestInfo::encode_list(&[(region_id, RegionManifestInfo::metric(1, 0, 2, 0))])
-                .unwrap(),
-        );
-        return Ok(response);
-    }
-
-    Ok(RegionResponse::new(0))
-}
-
 #[tokio::test]
 async fn test_on_submit_create_request() {
     common_telemetry::init_default_ut_logging();
     let (tx, mut rx) = mpsc::channel(8);
-    let handler = DatanodeWatcher::new(tx).with_handler(creates_request_handler);
+    let column_metadatas = test_column_metadatas(&["host", "cpu"]);
+    let handler =
+        DatanodeWatcher::new(tx).with_handler(make_creates_request_handler(column_metadatas));
     let node_manager = Arc::new(MockDatanodeManager::new(handler));
     let ddl_context = new_ddl_context(node_manager);
     let mut create_physical_table_task = test_create_physical_table_task("phy_table");

src/common/meta/src/ddl/tests/create_table.rs

@@ -16,7 +16,9 @@ use std::assert_matches::assert_matches;
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use api::v1::meta::Partition;
+use api::region::RegionResponse;
+use api::v1::meta::{Partition, Peer};
+use api::v1::region::{region_request, RegionRequest};
 use api::v1::{ColumnDataType, SemanticType};
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
@@ -24,7 +26,12 @@ use common_procedure::{Context as ProcedureContext, Procedure, ProcedureId, Stat
 use common_procedure_test::{
     execute_procedure_until, execute_procedure_until_done, MockContextProvider,
 };
+use datatypes::prelude::ConcreteDataType;
+use datatypes::schema::ColumnSchema;
+use store_api::metadata::ColumnMetadata;
+use store_api::metric_engine_consts::TABLE_COLUMN_METADATA_EXTENSION_KEY;
 use store_api::storage::RegionId;
+use tokio::sync::mpsc;
 
 use crate::ddl::create_table::{CreateTableProcedure, CreateTableState};
 use crate::ddl::test_util::columns::TestColumnDefBuilder;
@@ -32,14 +39,73 @@ use crate::ddl::test_util::create_table::{
     build_raw_table_info_from_expr, TestCreateTableExprBuilder,
 };
 use crate::ddl::test_util::datanode_handler::{
-    NaiveDatanodeHandler, RetryErrorDatanodeHandler, UnexpectedErrorDatanodeHandler,
+    DatanodeWatcher, NaiveDatanodeHandler, RetryErrorDatanodeHandler,
+    UnexpectedErrorDatanodeHandler,
 };
-use crate::error::Error;
+use crate::ddl::test_util::{assert_column_name, get_raw_table_info};
+use crate::error::{Error, Result};
 use crate::key::table_route::TableRouteValue;
 use crate::kv_backend::memory::MemoryKvBackend;
 use crate::rpc::ddl::CreateTableTask;
 use crate::test_util::{new_ddl_context, new_ddl_context_with_kv_backend, MockDatanodeManager};
 
+fn create_request_handler(_peer: Peer, request: RegionRequest) -> Result<RegionResponse> {
+    let _ = _peer;
+    if let region_request::Body::Create(_) = request.body.unwrap() {
+        let mut response = RegionResponse::new(0);
+
+        response.extensions.insert(
+            TABLE_COLUMN_METADATA_EXTENSION_KEY.to_string(),
+            ColumnMetadata::encode_list(&[
+                ColumnMetadata {
+                    column_schema: ColumnSchema::new(
+                        "ts",
+                        ConcreteDataType::timestamp_millisecond_datatype(),
+                        false,
+                    ),
+                    semantic_type: SemanticType::Timestamp,
+                    column_id: 0,
+                },
+                ColumnMetadata {
+                    column_schema: ColumnSchema::new(
+                        "host",
+                        ConcreteDataType::float64_datatype(),
+                        false,
+                    ),
+                    semantic_type: SemanticType::Tag,
+                    column_id: 1,
+                },
+                ColumnMetadata {
+                    column_schema: ColumnSchema::new(
+                        "cpu",
+                        ConcreteDataType::float64_datatype(),
+                        false,
+                    ),
+                    semantic_type: SemanticType::Tag,
+                    column_id: 2,
+                },
+            ])
+            .unwrap(),
+        );
+        return Ok(response);
+    }
+
+    Ok(RegionResponse::new(0))
+}
+
+fn assert_create_request(
+    peer: Peer,
+    request: RegionRequest,
+    expected_peer_id: u64,
+    expected_region_id: RegionId,
+) {
+    assert_eq!(peer.id, expected_peer_id);
+    let Some(region_request::Body::Create(req)) = request.body else {
+        unreachable!();
+    };
+    assert_eq!(req.region_id, expected_region_id);
+}
+
 pub(crate) fn test_create_table_task(name: &str) -> CreateTableTask {
     let create_table = TestCreateTableExprBuilder::default()
         .column_defs([
@@ -230,11 +296,13 @@ async fn test_on_create_metadata_error() {
 #[tokio::test]
 async fn test_on_create_metadata() {
     common_telemetry::init_default_ut_logging();
-    let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
+    let (tx, mut rx) = mpsc::channel(8);
+    let datanode_handler = DatanodeWatcher::new(tx).with_handler(create_request_handler);
+    let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
     let ddl_context = new_ddl_context(node_manager);
     let task = test_create_table_task("foo");
     assert!(!task.create_table.create_if_not_exists);
-    let mut procedure = CreateTableProcedure::new(task, ddl_context);
+    let mut procedure = CreateTableProcedure::new(task, ddl_context.clone());
     procedure.on_prepare().await.unwrap();
     let ctx = ProcedureContext {
         procedure_id: ProcedureId::random(),
@@ -243,8 +311,16 @@ async fn test_on_create_metadata() {
     procedure.execute(&ctx).await.unwrap();
     // Triggers procedure to create table metadata
     let status = procedure.execute(&ctx).await.unwrap();
-    let table_id = status.downcast_output_ref::<u32>().unwrap();
+    let table_id = *status.downcast_output_ref::<u32>().unwrap();
-    assert_eq!(*table_id, 1024);
+    assert_eq!(table_id, 1024);
+
+    let (peer, request) = rx.try_recv().unwrap();
+    rx.try_recv().unwrap_err();
+    assert_create_request(peer, request, 0, RegionId::new(table_id, 0));
+
+    let table_info = get_raw_table_info(&ddl_context, table_id).await;
+    assert_column_name(&table_info, &["ts", "host", "cpu"]);
+    assert_eq!(table_info.meta.column_ids, vec![0, 1, 2]);
 }
 
 #[tokio::test]

src/common/meta/src/ddl/utils.rs

@@ -12,6 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub(crate) mod raw_table_info;
+#[allow(dead_code)]
+pub(crate) mod region_metadata_lister;
+pub(crate) mod table_id;
+pub(crate) mod table_info;
+
 use std::collections::HashMap;
 use std::fmt::Debug;
 
@@ -29,6 +35,7 @@ use common_telemetry::{error, info, warn};
 use common_wal::options::WalOptions;
 use futures::future::join_all;
 use snafu::{ensure, OptionExt, ResultExt};
+use store_api::metadata::ColumnMetadata;
 use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, MANIFEST_INFO_EXTENSION_KEY};
 use store_api::region_engine::RegionManifestInfo;
 use store_api::storage::{RegionId, RegionNumber};
@@ -37,8 +44,8 @@ use table::table_reference::TableReference;
 
 use crate::ddl::{DdlContext, DetectingRegion};
 use crate::error::{
-    self, Error, OperateDatanodeSnafu, ParseWalOptionsSnafu, Result, TableNotFoundSnafu,
+    self, DecodeJsonSnafu, Error, MetadataCorruptionSnafu, OperateDatanodeSnafu,
-    UnsupportedSnafu,
+    ParseWalOptionsSnafu, Result, TableNotFoundSnafu, UnsupportedSnafu,
 };
 use crate::key::datanode_table::DatanodeTableValue;
 use crate::key::table_name::TableNameKey;
@@ -314,11 +321,23 @@ pub fn parse_manifest_infos_from_extensions(
     Ok(data_manifest_version)
 }
 
+/// Parses column metadatas from extensions.
+pub fn parse_column_metadatas(
+    extensions: &HashMap<String, Vec<u8>>,
+    key: &str,
+) -> Result<Vec<ColumnMetadata>> {
+    let value = extensions.get(key).context(error::UnexpectedSnafu {
+        err_msg: format!("column metadata extension not found: {}", key),
+    })?;
+    let column_metadatas = ColumnMetadata::decode_list(value).context(error::SerdeJsonSnafu {})?;
+    Ok(column_metadatas)
+}
+
 /// Sync follower regions on datanodes.
 pub async fn sync_follower_regions(
     context: &DdlContext,
     table_id: TableId,
-    results: Vec<RegionResponse>,
+    results: &[RegionResponse],
     region_routes: &[RegionRoute],
     engine: &str,
 ) -> Result<()> {
@@ -331,7 +350,7 @@ pub async fn sync_follower_regions(
     }
 
     let results = results
-        .into_iter()
+        .iter()
         .map(|response| parse_manifest_infos_from_extensions(&response.extensions))
         .collect::<Result<Vec<_>>>()?
         .into_iter()
@@ -418,6 +437,39 @@ pub async fn sync_follower_regions(
     Ok(())
 }
 
+/// Extracts column metadatas from extensions.
+pub fn extract_column_metadatas(
+    results: &mut [RegionResponse],
+    key: &str,
+) -> Result<Option<Vec<ColumnMetadata>>> {
+    let schemas = results
+        .iter_mut()
+        .map(|r| r.extensions.remove(key))
+        .collect::<Vec<_>>();
+
+    if schemas.is_empty() {
+        warn!("extract_column_metadatas: no extension key `{key}` found in results");
+        return Ok(None);
+    }
+
+    // Verify all the physical schemas are the same
+    // Safety: previous check ensures this vec is not empty
+    let first = schemas.first().unwrap();
+    ensure!(
+        schemas.iter().all(|x| x == first),
+        MetadataCorruptionSnafu {
+            err_msg: "The table column metadata schemas from datanodes are not the same."
+        }
+    );
+
+    if let Some(first) = first {
+        let column_metadatas = ColumnMetadata::decode_list(first).context(DecodeJsonSnafu)?;
+        Ok(Some(column_metadatas))
+    } else {
+        Ok(None)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/common/meta/src/ddl/utils/raw_table_info.rs

@@ -0,0 +1,123 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet};
+
+use api::v1::SemanticType;
+use common_telemetry::debug;
+use common_telemetry::tracing::warn;
+use store_api::metadata::ColumnMetadata;
+use table::metadata::RawTableInfo;
+
+/// Generate the new physical table info.
+pub(crate) fn build_new_physical_table_info(
+    mut raw_table_info: RawTableInfo,
+    physical_columns: &[ColumnMetadata],
+) -> RawTableInfo {
+    debug!(
+        "building new physical table info for table: {}, table_id: {}",
+        raw_table_info.name, raw_table_info.ident.table_id
+    );
+    let existing_columns = raw_table_info
+        .meta
+        .schema
+        .column_schemas
+        .iter()
+        .map(|col| col.name.clone())
+        .collect::<HashSet<_>>();
+    let primary_key_indices = &mut raw_table_info.meta.primary_key_indices;
+    let value_indices = &mut raw_table_info.meta.value_indices;
+    value_indices.clear();
+    let time_index = &mut raw_table_info.meta.schema.timestamp_index;
+    let columns = &mut raw_table_info.meta.schema.column_schemas;
+    columns.clear();
+    let column_ids = &mut raw_table_info.meta.column_ids;
+    column_ids.clear();
+
+    for (idx, col) in physical_columns.iter().enumerate() {
+        match col.semantic_type {
+            SemanticType::Tag => {
+                // push new primary key to the end.
+                if !existing_columns.contains(&col.column_schema.name) {
+                    primary_key_indices.push(idx);
+                }
+            }
+            SemanticType::Field => value_indices.push(idx),
+            SemanticType::Timestamp => {
+                value_indices.push(idx);
+                *time_index = Some(idx);
+            }
+        }
+
+        columns.push(col.column_schema.clone());
+        column_ids.push(col.column_id);
+    }
+
+    if let Some(time_index) = *time_index {
+        raw_table_info.meta.schema.column_schemas[time_index].set_time_index();
+    }
+
+    raw_table_info
+}
+
+/// Updates the column IDs in the table info based on the provided column metadata.
+///
+/// This function validates that the column metadata matches the existing table schema
+/// before updating the column ids. If the column metadata doesn't match the table schema,
+/// the table info remains unchanged.
+pub(crate) fn update_table_info_column_ids(
+    raw_table_info: &mut RawTableInfo,
+    column_metadatas: &[ColumnMetadata],
+) {
+    let mut table_column_names = raw_table_info
+        .meta
+        .schema
+        .column_schemas
+        .iter()
+        .map(|c| c.name.as_str())
+        .collect::<Vec<_>>();
+    table_column_names.sort_unstable();
+
+    let mut column_names = column_metadatas
+        .iter()
+        .map(|c| c.column_schema.name.as_str())
+        .collect::<Vec<_>>();
+    column_names.sort_unstable();
+
+    if table_column_names != column_names {
+        warn!(
+            "Column metadata doesn't match the table schema for table {}, table_id: {}, column in table: {:?}, column in metadata: {:?}",
+            raw_table_info.name,
+            raw_table_info.ident.table_id,
+            table_column_names,
+            column_names,
+        );
+        return;
+    }
+
+    let name_to_id = column_metadatas
+        .iter()
+        .map(|c| (c.column_schema.name.clone(), c.column_id))
+        .collect::<HashMap<_, _>>();
+
+    let schema = &raw_table_info.meta.schema.column_schemas;
+    let mut column_ids = Vec::with_capacity(schema.len());
+    for column_schema in schema {
+        if let Some(id) = name_to_id.get(&column_schema.name) {
+            column_ids.push(*id);
+        }
+    }
+
+    raw_table_info.meta.column_ids = column_ids;
+}

src/common/meta/src/ddl/utils/region_metadata_lister.rs

@@ -0,0 +1,240 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+
+use api::v1::region::region_request::Body as PbRegionRequest;
+use api::v1::region::{ListMetadataRequest, RegionRequest, RegionRequestHeader};
+use common_telemetry::tracing_context::TracingContext;
+use futures::future::join_all;
+use snafu::ResultExt;
+use store_api::metadata::RegionMetadata;
+use store_api::storage::{RegionId, TableId};
+
+use crate::ddl::utils::add_peer_context_if_needed;
+use crate::error::{DecodeJsonSnafu, Result};
+use crate::node_manager::NodeManagerRef;
+use crate::rpc::router::{find_leaders, region_distribution, RegionRoute};
+
+/// Collects the region metadata from the datanodes.
+pub struct RegionMetadataLister {
+    node_manager: NodeManagerRef,
+}
+
+impl RegionMetadataLister {
+    /// Creates a new [`RegionMetadataLister`] with the given [`NodeManagerRef`].
+    pub fn new(node_manager: NodeManagerRef) -> Self {
+        Self { node_manager }
+    }
+
+    /// Collects the region metadata from the datanodes.
+    pub async fn list(
+        &self,
+        table_id: TableId,
+        region_routes: &[RegionRoute],
+    ) -> Result<Vec<Option<RegionMetadata>>> {
+        let region_distribution = region_distribution(region_routes);
+        let leaders = find_leaders(region_routes)
+            .into_iter()
+            .map(|p| (p.id, p))
+            .collect::<HashMap<_, _>>();
+
+        let total_num_region = region_distribution
+            .values()
+            .map(|r| r.leader_regions.len())
+            .sum::<usize>();
+
+        let mut list_metadata_tasks = Vec::with_capacity(leaders.len());
+
+        // Build requests.
+        for (datanode_id, region_role_set) in region_distribution {
+            if region_role_set.leader_regions.is_empty() {
+                continue;
+            }
+            // Safety: must exists.
+            let peer = leaders.get(&datanode_id).unwrap();
+            let requester = self.node_manager.datanode(peer).await;
+            let region_ids = region_role_set
+                .leader_regions
+                .iter()
+                .map(|r| RegionId::new(table_id, *r).as_u64())
+                .collect();
+            let request = Self::build_list_metadata_request(region_ids);
+
+            let peer = peer.clone();
+            list_metadata_tasks.push(async move {
+                requester
+                    .handle(request)
+                    .await
+                    .map_err(add_peer_context_if_needed(peer))
+            });
+        }
+
+        let results = join_all(list_metadata_tasks)
+            .await
+            .into_iter()
+            .collect::<Result<Vec<_>>>()?
+            .into_iter()
+            .map(|r| r.metadata);
+
+        let mut output = Vec::with_capacity(total_num_region);
+        for result in results {
+            let region_metadatas: Vec<Option<RegionMetadata>> =
+                serde_json::from_slice(&result).context(DecodeJsonSnafu)?;
+            output.extend(region_metadatas);
+        }
+
+        Ok(output)
+    }
+
+    fn build_list_metadata_request(region_ids: Vec<u64>) -> RegionRequest {
+        RegionRequest {
+            header: Some(RegionRequestHeader {
+                tracing_context: TracingContext::from_current_span().to_w3c(),
+                ..Default::default()
+            }),
+            body: Some(PbRegionRequest::ListMetadata(ListMetadataRequest {
+                region_ids,
+            })),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use api::region::RegionResponse;
+    use api::v1::meta::Peer;
+    use api::v1::region::region_request::Body;
+    use api::v1::region::RegionRequest;
+    use store_api::metadata::RegionMetadata;
+    use store_api::storage::RegionId;
+    use tokio::sync::mpsc;
+
+    use crate::ddl::test_util::datanode_handler::{DatanodeWatcher, ListMetadataDatanodeHandler};
+    use crate::ddl::test_util::region_metadata::build_region_metadata;
+    use crate::ddl::test_util::test_column_metadatas;
+    use crate::ddl::utils::region_metadata_lister::RegionMetadataLister;
+    use crate::error::Result;
+    use crate::rpc::router::{Region, RegionRoute};
+    use crate::test_util::MockDatanodeManager;
+
+    fn assert_list_metadata_request(req: RegionRequest, expected_region_ids: &[RegionId]) {
+        let Some(Body::ListMetadata(req)) = req.body else {
+            unreachable!()
+        };
+
+        assert_eq!(req.region_ids.len(), expected_region_ids.len());
+        for region_id in expected_region_ids {
+            assert!(req.region_ids.contains(&region_id.as_u64()));
+        }
+    }
+
+    fn empty_list_metadata_handler(_peer: Peer, request: RegionRequest) -> Result<RegionResponse> {
+        let Some(Body::ListMetadata(req)) = request.body else {
+            unreachable!()
+        };
+
+        let mut output: Vec<Option<RegionMetadata>> = Vec::with_capacity(req.region_ids.len());
+        for _region_id in req.region_ids {
+            output.push(None);
+        }
+
+        Ok(RegionResponse::from_metadata(
+            serde_json::to_vec(&output).unwrap(),
+        ))
+    }
+
+    #[tokio::test]
+    async fn test_list_request() {
+        let (tx, mut rx) = mpsc::channel(8);
+        let handler = DatanodeWatcher::new(tx).with_handler(empty_list_metadata_handler);
+        let node_manager = Arc::new(MockDatanodeManager::new(handler));
+        let lister = RegionMetadataLister::new(node_manager);
+        let region_routes = vec![
+            RegionRoute {
+                region: Region::new_test(RegionId::new(1024, 1)),
+                leader_peer: Some(Peer::empty(1)),
+                follower_peers: vec![Peer::empty(5)],
+                leader_state: None,
+                leader_down_since: None,
+            },
+            RegionRoute {
+                region: Region::new_test(RegionId::new(1024, 2)),
+                leader_peer: Some(Peer::empty(3)),
+                follower_peers: vec![Peer::empty(4)],
+                leader_state: None,
+                leader_down_since: None,
+            },
+            RegionRoute {
+                region: Region::new_test(RegionId::new(1024, 3)),
+                leader_peer: Some(Peer::empty(3)),
+                follower_peers: vec![Peer::empty(4)],
+                leader_state: None,
+                leader_down_since: None,
+            },
+        ];
+        let region_metadatas = lister.list(1024, &region_routes).await.unwrap();
+        assert_eq!(region_metadatas.len(), 3);
+
+        let mut requests = vec![];
+        for _ in 0..2 {
+            let (peer, request) = rx.try_recv().unwrap();
+            requests.push((peer, request));
+        }
+        rx.try_recv().unwrap_err();
+
+        let (peer, request) = requests.remove(0);
+        assert_eq!(peer.id, 1);
+        assert_list_metadata_request(request, &[RegionId::new(1024, 1)]);
+        let (peer, request) = requests.remove(0);
+        assert_eq!(peer.id, 3);
+        assert_list_metadata_request(request, &[RegionId::new(1024, 2), RegionId::new(1024, 3)]);
+    }
+
+    #[tokio::test]
+    async fn test_list_region_metadata() {
+        let region_metadata =
+            build_region_metadata(RegionId::new(1024, 1), &test_column_metadatas(&["tag_0"]));
+        let region_metadatas = HashMap::from([
+            (RegionId::new(1024, 0), None),
+            (RegionId::new(1024, 1), Some(region_metadata.clone())),
+        ]);
+        let handler = ListMetadataDatanodeHandler::new(region_metadatas);
+        let node_manager = Arc::new(MockDatanodeManager::new(handler));
+        let lister = RegionMetadataLister::new(node_manager);
+        let region_routes = vec![
+            RegionRoute {
+                region: Region::new_test(RegionId::new(1024, 0)),
+                leader_peer: Some(Peer::empty(1)),
+                follower_peers: vec![],
+                leader_state: None,
+                leader_down_since: None,
+            },
+            RegionRoute {
+                region: Region::new_test(RegionId::new(1024, 1)),
+                leader_peer: Some(Peer::empty(3)),
+                follower_peers: vec![],
+                leader_state: None,
+                leader_down_since: None,
+            },
+        ];
+        let region_metadatas = lister.list(1024, &region_routes).await.unwrap();
+        assert_eq!(region_metadatas.len(), 2);
+        assert_eq!(region_metadatas[0], None);
+        assert_eq!(region_metadatas[1], Some(region_metadata));
+    }
+}

src/common/meta/src/ddl/utils/table_id.rs

@@ -0,0 +1,46 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use snafu::OptionExt;
+use store_api::storage::TableId;
+use table::table_reference::TableReference;
+
+use crate::error::{Result, TableNotFoundSnafu};
+use crate::key::table_name::{TableNameKey, TableNameManager};
+
+/// Get all the table ids from the table names.
+///
+/// Returns an error if any table does not exist.
+pub(crate) async fn get_all_table_ids_by_names<'a>(
+    table_name_manager: &TableNameManager,
+    table_names: &[TableReference<'a>],
+) -> Result<Vec<TableId>> {
+    let table_name_keys = table_names
+        .iter()
+        .map(TableNameKey::from)
+        .collect::<Vec<_>>();
+    let table_name_values = table_name_manager.batch_get(table_name_keys).await?;
+    let mut table_ids = Vec::with_capacity(table_name_values.len());
+    for (value, table_name) in table_name_values.into_iter().zip(table_names) {
+        let value = value
+            .with_context(|| TableNotFoundSnafu {
+                table_name: table_name.to_string(),
+            })?
+            .table_id();
+
+        table_ids.push(value);
+    }
+
+    Ok(table_ids)
+}

src/common/meta/src/ddl/utils/table_info.rs

@@ -0,0 +1,100 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use itertools::Itertools;
+use snafu::OptionExt;
+use store_api::storage::TableId;
+use table::metadata::RawTableInfo;
+use table::table_reference::TableReference;
+
+use crate::error::{Result, TableInfoNotFoundSnafu};
+use crate::key::table_info::{TableInfoManager, TableInfoValue};
+use crate::key::table_route::{TableRouteManager, TableRouteValue};
+use crate::key::{DeserializedValueWithBytes, TableMetadataManager};
+
+/// Get all table info values by table ids.
+///
+/// Returns an error if any table does not exist.
+pub(crate) async fn get_all_table_info_values_by_table_ids<'a>(
+    table_info_manager: &TableInfoManager,
+    table_ids: &[TableId],
+    table_names: &[TableReference<'a>],
+) -> Result<Vec<DeserializedValueWithBytes<TableInfoValue>>> {
+    let mut table_info_map = table_info_manager.batch_get_raw(table_ids).await?;
+    let mut table_info_values = Vec::with_capacity(table_ids.len());
+    for (table_id, table_name) in table_ids.iter().zip(table_names) {
+        let table_info_value =
+            table_info_map
+                .remove(table_id)
+                .with_context(|| TableInfoNotFoundSnafu {
+                    table: table_name.to_string(),
+                })?;
+        table_info_values.push(table_info_value);
+    }
+
+    Ok(table_info_values)
+}
+
+/// Checks if all the logical table routes have the same physical table id.
+pub(crate) async fn all_logical_table_routes_have_same_physical_id(
+    table_route_manager: &TableRouteManager,
+    table_ids: &[TableId],
+    physical_table_id: TableId,
+) -> Result<bool> {
+    let table_routes = table_route_manager
+        .table_route_storage()
+        .batch_get(table_ids)
+        .await?;
+
+    let is_same_physical_table = table_routes.iter().all(|r| {
+        if let Some(TableRouteValue::Logical(r)) = r {
+            r.physical_table_id() == physical_table_id
+        } else {
+            false
+        }
+    });
+
+    Ok(is_same_physical_table)
+}
+
+/// Batch updates the table info values.
+///
+/// The table info values are grouped into chunks, and each chunk is updated in a single transaction.
+///
+/// Returns an error if any table info value fails to update.
+pub(crate) async fn batch_update_table_info_values(
+    table_metadata_manager: &TableMetadataManager,
+    table_info_values: Vec<(DeserializedValueWithBytes<TableInfoValue>, RawTableInfo)>,
+) -> Result<()> {
+    let chunk_size = table_metadata_manager.batch_update_table_info_value_chunk_size();
+    if table_info_values.len() > chunk_size {
+        let chunks = table_info_values
+            .into_iter()
+            .chunks(chunk_size)
+            .into_iter()
+            .map(|check| check.collect::<Vec<_>>())
+            .collect::<Vec<_>>();
+        for chunk in chunks {
+            table_metadata_manager
+                .batch_update_table_info_values(chunk)
+                .await?;
+        }
+    } else {
+        table_metadata_manager
+            .batch_update_table_info_values(table_info_values)
+            .await?;
+    }
+
+    Ok(())
+}

src/common/meta/src/ddl_manager.rs

@@ -14,7 +14,6 @@
 
 use std::sync::Arc;
 
-use api::v1::meta::ProcedureDetailResponse;
 use common_procedure::{
     watcher, BoxedProcedureLoader, Output, ProcedureId, ProcedureManagerRef, ProcedureWithId,
 };
@@ -37,16 +36,16 @@ use crate::ddl::drop_flow::DropFlowProcedure;
 use crate::ddl::drop_table::DropTableProcedure;
 use crate::ddl::drop_view::DropViewProcedure;
 use crate::ddl::truncate_table::TruncateTableProcedure;
-use crate::ddl::{utils, DdlContext, ExecutorContext, ProcedureExecutor};
+use crate::ddl::{utils, DdlContext};
 use crate::error::{
-    EmptyDdlTasksSnafu, ParseProcedureIdSnafu, ProcedureNotFoundSnafu, ProcedureOutputSnafu,
+    EmptyDdlTasksSnafu, ProcedureOutputSnafu, RegisterProcedureLoaderSnafu, Result,
-    QueryProcedureSnafu, RegisterProcedureLoaderSnafu, Result, SubmitProcedureSnafu,
+    SubmitProcedureSnafu, TableInfoNotFoundSnafu, TableNotFoundSnafu, TableRouteNotFoundSnafu,
-    TableInfoNotFoundSnafu, TableNotFoundSnafu, TableRouteNotFoundSnafu,
+    UnexpectedLogicalRouteTableSnafu, WaitProcedureSnafu,
-    UnexpectedLogicalRouteTableSnafu, UnsupportedSnafu, WaitProcedureSnafu,
 };
 use crate::key::table_info::TableInfoValue;
 use crate::key::table_name::TableNameKey;
 use crate::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
+use crate::procedure_executor::ExecutorContext;
 #[cfg(feature = "enterprise")]
 use crate::rpc::ddl::trigger::CreateTriggerTask;
 #[cfg(feature = "enterprise")]
@@ -61,8 +60,6 @@ use crate::rpc::ddl::{
     CreateViewTask, DropDatabaseTask, DropFlowTask, DropTableTask, DropViewTask, QueryContext,
     SubmitDdlTaskRequest, SubmitDdlTaskResponse, TruncateTableTask,
 };
-use crate::rpc::procedure;
-use crate::rpc::procedure::{MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse};
 use crate::rpc::router::RegionRoute;
 
 pub type DdlManagerRef = Arc<DdlManager>;
@@ -125,13 +122,12 @@ impl DdlManager {
         ddl_context: DdlContext,
         procedure_manager: ProcedureManagerRef,
         register_loaders: bool,
-        #[cfg(feature = "enterprise")] trigger_ddl_manager: Option<TriggerDdlManagerRef>,
     ) -> Result<Self> {
         let manager = Self {
             ddl_context,
             procedure_manager,
             #[cfg(feature = "enterprise")]
-            trigger_ddl_manager,
+            trigger_ddl_manager: None,
         };
         if register_loaders {
             manager.register_loaders()?;
@@ -139,6 +135,15 @@ impl DdlManager {
         Ok(manager)
     }
 
+    #[cfg(feature = "enterprise")]
+    pub fn with_trigger_ddl_manager(
+        mut self,
+        trigger_ddl_manager: Option<TriggerDdlManagerRef>,
+    ) -> Self {
+        self.trigger_ddl_manager = trigger_ddl_manager;
+        self
+    }
+
     /// Returns the [TableMetadataManagerRef].
     pub fn table_metadata_manager(&self) -> &TableMetadataManagerRef {
         &self.ddl_context.table_metadata_manager
@@ -398,6 +403,70 @@ impl DdlManager {
 
         Ok((procedure_id, output))
     }
+
+    pub async fn submit_ddl_task(
+        &self,
+        ctx: &ExecutorContext,
+        request: SubmitDdlTaskRequest,
+    ) -> Result<SubmitDdlTaskResponse> {
+        let span = ctx
+            .tracing_context
+            .as_ref()
+            .map(TracingContext::from_w3c)
+            .unwrap_or_else(TracingContext::from_current_span)
+            .attach(tracing::info_span!("DdlManager::submit_ddl_task"));
+        async move {
+            debug!("Submitting Ddl task: {:?}", request.task);
+            match request.task {
+                CreateTable(create_table_task) => {
+                    handle_create_table_task(self, create_table_task).await
+                }
+                DropTable(drop_table_task) => handle_drop_table_task(self, drop_table_task).await,
+                AlterTable(alter_table_task) => {
+                    handle_alter_table_task(self, alter_table_task).await
+                }
+                TruncateTable(truncate_table_task) => {
+                    handle_truncate_table_task(self, truncate_table_task).await
+                }
+                CreateLogicalTables(create_table_tasks) => {
+                    handle_create_logical_table_tasks(self, create_table_tasks).await
+                }
+                AlterLogicalTables(alter_table_tasks) => {
+                    handle_alter_logical_table_tasks(self, alter_table_tasks).await
+                }
+                DropLogicalTables(_) => todo!(),
+                CreateDatabase(create_database_task) => {
+                    handle_create_database_task(self, create_database_task).await
+                }
+                DropDatabase(drop_database_task) => {
+                    handle_drop_database_task(self, drop_database_task).await
+                }
+                AlterDatabase(alter_database_task) => {
+                    handle_alter_database_task(self, alter_database_task).await
+                }
+                CreateFlow(create_flow_task) => {
+                    handle_create_flow_task(self, create_flow_task, request.query_context.into())
+                        .await
+                }
+                DropFlow(drop_flow_task) => handle_drop_flow_task(self, drop_flow_task).await,
+                CreateView(create_view_task) => {
+                    handle_create_view_task(self, create_view_task).await
+                }
+                DropView(drop_view_task) => handle_drop_view_task(self, drop_view_task).await,
+                #[cfg(feature = "enterprise")]
+                CreateTrigger(create_trigger_task) => {
+                    handle_create_trigger_task(
+                        self,
+                        create_trigger_task,
+                        request.query_context.into(),
+                    )
+                    .await
+                }
+            }
+        }
+        .trace(span)
+        .await
+    }
 }
 
 async fn handle_truncate_table_task(
@@ -704,6 +773,8 @@ async fn handle_create_trigger_task(
     query_context: QueryContext,
 ) -> Result<SubmitDdlTaskResponse> {
     let Some(m) = ddl_manager.trigger_ddl_manager.as_ref() else {
+        use crate::error::UnsupportedSnafu;
+
         return UnsupportedSnafu {
             operation: "create trigger",
         }
@@ -780,114 +851,6 @@ async fn handle_create_view_task(
     })
 }
 
-/// TODO(dennis): let [`DdlManager`] implement [`ProcedureExecutor`] looks weird, find some way to refactor it.
-#[async_trait::async_trait]
-impl ProcedureExecutor for DdlManager {
-    async fn submit_ddl_task(
-        &self,
-        ctx: &ExecutorContext,
-        request: SubmitDdlTaskRequest,
-    ) -> Result<SubmitDdlTaskResponse> {
-        let span = ctx
-            .tracing_context
-            .as_ref()
-            .map(TracingContext::from_w3c)
-            .unwrap_or(TracingContext::from_current_span())
-            .attach(tracing::info_span!("DdlManager::submit_ddl_task"));
-        async move {
-            debug!("Submitting Ddl task: {:?}", request.task);
-            match request.task {
-                CreateTable(create_table_task) => {
-                    handle_create_table_task(self, create_table_task).await
-                }
-                DropTable(drop_table_task) => handle_drop_table_task(self, drop_table_task).await,
-                AlterTable(alter_table_task) => {
-                    handle_alter_table_task(self, alter_table_task).await
-                }
-                TruncateTable(truncate_table_task) => {
-                    handle_truncate_table_task(self, truncate_table_task).await
-                }
-                CreateLogicalTables(create_table_tasks) => {
-                    handle_create_logical_table_tasks(self, create_table_tasks).await
-                }
-                AlterLogicalTables(alter_table_tasks) => {
-                    handle_alter_logical_table_tasks(self, alter_table_tasks).await
-                }
-                DropLogicalTables(_) => todo!(),
-                CreateDatabase(create_database_task) => {
-                    handle_create_database_task(self, create_database_task).await
-                }
-                DropDatabase(drop_database_task) => {
-                    handle_drop_database_task(self, drop_database_task).await
-                }
-                AlterDatabase(alter_database_task) => {
-                    handle_alter_database_task(self, alter_database_task).await
-                }
-                CreateFlow(create_flow_task) => {
-                    handle_create_flow_task(self, create_flow_task, request.query_context.into())
-                        .await
-                }
-                #[cfg(feature = "enterprise")]
-                CreateTrigger(create_trigger_task) => {
-                    handle_create_trigger_task(
-                        self,
-                        create_trigger_task,
-                        request.query_context.into(),
-                    )
-                    .await
-                }
-                DropFlow(drop_flow_task) => handle_drop_flow_task(self, drop_flow_task).await,
-                CreateView(create_view_task) => {
-                    handle_create_view_task(self, create_view_task).await
-                }
-                DropView(drop_view_task) => handle_drop_view_task(self, drop_view_task).await,
-            }
-        }
-        .trace(span)
-        .await
-    }
-
-    async fn migrate_region(
-        &self,
-        _ctx: &ExecutorContext,
-        _request: MigrateRegionRequest,
-    ) -> Result<MigrateRegionResponse> {
-        UnsupportedSnafu {
-            operation: "migrate_region",
-        }
-        .fail()
-    }
-
-    async fn query_procedure_state(
-        &self,
-        _ctx: &ExecutorContext,
-        pid: &str,
-    ) -> Result<ProcedureStateResponse> {
-        let pid =
-            ProcedureId::parse_str(pid).with_context(|_| ParseProcedureIdSnafu { key: pid })?;
-
-        let state = self
-            .procedure_manager
-            .procedure_state(pid)
-            .await
-            .context(QueryProcedureSnafu)?
-            .context(ProcedureNotFoundSnafu {
-                pid: pid.to_string(),
-            })?;
-
-        Ok(procedure::procedure_state_to_pb_response(&state))
-    }
-
-    async fn list_procedures(&self, _ctx: &ExecutorContext) -> Result<ProcedureDetailResponse> {
-        let metas = self
-            .procedure_manager
-            .list_procedures()
-            .await
-            .context(QueryProcedureSnafu)?;
-        Ok(procedure::procedure_details_to_pb_response(metas))
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
@@ -948,6 +911,7 @@ mod tests {
             Default::default(),
             state_store,
             poison_manager,
+            None,
         ));
 
         let _ = DdlManager::try_new(
@@ -964,8 +928,6 @@ mod tests {
             },
             procedure_manager.clone(),
             true,
-            #[cfg(feature = "enterprise")]
-            None,
         );
 
         let expected_loaders = vec![

src/common/meta/src/error.rs

@@ -18,6 +18,7 @@ use std::sync::Arc;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
+use common_procedure::ProcedureId;
 use common_wal::options::WalOptions;
 use serde_json::error::Error as JsonError;
 use snafu::{Location, Snafu};
@@ -140,6 +141,21 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to get procedure state receiver, procedure id: {procedure_id}"))]
+    ProcedureStateReceiver {
+        procedure_id: ProcedureId,
+        #[snafu(implicit)]
+        location: Location,
+        source: common_procedure::Error,
+    },
+
+    #[snafu(display("Procedure state receiver not found: {procedure_id}"))]
+    ProcedureStateReceiverNotFound {
+        procedure_id: ProcedureId,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Failed to wait procedure done"))]
     WaitProcedure {
         #[snafu(implicit)]
@@ -387,6 +403,13 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Catalog not found, catalog: {}", catalog))]
+    CatalogNotFound {
+        catalog: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Invalid metadata, err: {}", err_msg))]
     InvalidMetadata {
         err_msg: String,
@@ -877,6 +900,93 @@ pub enum Error {
         #[snafu(source)]
         error: object_store::Error,
     },
+
+    #[snafu(display("Missing column ids"))]
+    MissingColumnIds {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display(
+        "Missing column in column metadata: {}, table: {}, table_id: {}",
+        column_name,
+        table_name,
+        table_id,
+    ))]
+    MissingColumnInColumnMetadata {
+        column_name: String,
+        #[snafu(implicit)]
+        location: Location,
+        table_name: String,
+        table_id: TableId,
+    },
+
+    #[snafu(display(
+        "Mismatch column id: column_name: {}, column_id: {}, table: {}, table_id: {}",
+        column_name,
+        column_id,
+        table_name,
+        table_id,
+    ))]
+    MismatchColumnId {
+        column_name: String,
+        column_id: u32,
+        #[snafu(implicit)]
+        location: Location,
+        table_name: String,
+        table_id: TableId,
+    },
+
+    #[snafu(display("Failed to convert column def, column: {}", column))]
+    ConvertColumnDef {
+        column: String,
+        #[snafu(implicit)]
+        location: Location,
+        source: api::error::Error,
+    },
+
+    #[snafu(display(
+        "Column metadata inconsistencies found in table: {}, table_id: {}",
+        table_name,
+        table_id
+    ))]
+    ColumnMetadataConflicts {
+        table_name: String,
+        table_id: TableId,
+    },
+
+    #[snafu(display(
+        "Column not found in column metadata, column_name: {}, column_id: {}",
+        column_name,
+        column_id
+    ))]
+    ColumnNotFound { column_name: String, column_id: u32 },
+
+    #[snafu(display(
+        "Column id mismatch, column_name: {}, expected column_id: {}, actual column_id: {}",
+        column_name,
+        expected_column_id,
+        actual_column_id
+    ))]
+    ColumnIdMismatch {
+        column_name: String,
+        expected_column_id: u32,
+        actual_column_id: u32,
+    },
+
+    #[snafu(display(
+        "Timestamp column mismatch, expected column_name: {}, expected column_id: {}, actual column_name: {}, actual column_id: {}",
+        expected_column_name,
+        expected_column_id,
+        actual_column_name,
+        actual_column_id,
+    ))]
+    TimestampMismatch {
+        expected_column_name: String,
+        expected_column_id: u32,
+        actual_column_name: String,
+        actual_column_id: u32,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -896,7 +1006,16 @@ impl ErrorExt for Error {
             | DeserializeFromJson { .. } => StatusCode::Internal,
 
             NoLeader { .. } => StatusCode::TableUnavailable,
-            ValueNotExist { .. } | ProcedurePoisonConflict { .. } => StatusCode::Unexpected,
+            ValueNotExist { .. }
+            | ProcedurePoisonConflict { .. }
+            | ProcedureStateReceiverNotFound { .. }
+            | MissingColumnIds { .. }
+            | MissingColumnInColumnMetadata { .. }
+            | MismatchColumnId { .. }
+            | ColumnMetadataConflicts { .. }
+            | ColumnNotFound { .. }
+            | ColumnIdMismatch { .. }
+            | TimestampMismatch { .. } => StatusCode::Unexpected,
 
             Unsupported { .. } => StatusCode::Unsupported,
             WriteObject { .. } | ReadObject { .. } => StatusCode::StorageUnavailable,
@@ -980,10 +1099,13 @@ impl ErrorExt for Error {
             AbortProcedure { source, .. } => source.status_code(),
             ConvertAlterTableRequest { source, .. } => source.status_code(),
             PutPoison { source, .. } => source.status_code(),
+            ConvertColumnDef { source, .. } => source.status_code(),
+            ProcedureStateReceiver { source, .. } => source.status_code(),
 
             ParseProcedureId { .. }
             | InvalidNumTopics { .. }
             | SchemaNotFound { .. }
+            | CatalogNotFound { .. }
             | InvalidNodeInfoKey { .. }
             | InvalidStatKey { .. }
             | ParseNum { .. }

src/common/meta/src/instruction.rs

@@ -174,6 +174,8 @@ pub struct UpgradeRegion {
 /// The identifier of cache.
 pub enum CacheIdent {
     FlowId(FlowId),
+    /// Indicate change of address of flownode.
+    FlowNodeAddressChange(u64),
     FlowName(FlowName),
     TableId(TableId),
     TableName(TableName),

src/common/meta/src/key.rs

@@ -100,8 +100,8 @@
 pub mod catalog_name;
 pub mod datanode_table;
 pub mod flow;
-pub mod maintenance;
 pub mod node_address;
+pub mod runtime_switch;
 mod schema_metadata_manager;
 pub mod schema_name;
 pub mod table_info;
@@ -164,7 +164,10 @@ use crate::state_store::PoisonValue;
 use crate::DatanodeId;
 
 pub const NAME_PATTERN: &str = r"[a-zA-Z_:-][a-zA-Z0-9_:\-\.@#]*";
-pub const MAINTENANCE_KEY: &str = "__maintenance";
+pub const LEGACY_MAINTENANCE_KEY: &str = "__maintenance";
+pub const MAINTENANCE_KEY: &str = "__switches/maintenance";
+pub const PAUSE_PROCEDURE_KEY: &str = "__switches/pause_procedure";
+pub const RECOVERY_MODE_KEY: &str = "__switches/recovery";
 
 pub const DATANODE_TABLE_KEY_PREFIX: &str = "__dn_table";
 pub const TABLE_INFO_KEY_PREFIX: &str = "__table_info";
@@ -179,6 +182,11 @@ pub const KAFKA_TOPIC_KEY_PREFIX: &str = "__topic_name/kafka";
 pub const LEGACY_TOPIC_KEY_PREFIX: &str = "__created_wal_topics/kafka";
 pub const TOPIC_REGION_PREFIX: &str = "__topic_region";
 
+/// The election key.
+pub const ELECTION_KEY: &str = "__metasrv_election";
+/// The root key of metasrv election candidates.
+pub const CANDIDATES_ROOT: &str = "__metasrv_election_candidates/";
+
 /// The keys with these prefixes will be loaded into the cache when the leader starts.
 pub const CACHE_KEY_PREFIXES: [&str; 5] = [
     TABLE_NAME_KEY_PREFIX,

src/common/meta/src/key/maintenance.rs

@@ -1,86 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use crate::error::Result;
-use crate::key::MAINTENANCE_KEY;
-use crate::kv_backend::KvBackendRef;
-use crate::rpc::store::PutRequest;
-
-pub type MaintenanceModeManagerRef = Arc<MaintenanceModeManager>;
-
-/// The maintenance mode manager.
-///
-/// Used to enable or disable maintenance mode.
-#[derive(Clone)]
-pub struct MaintenanceModeManager {
-    kv_backend: KvBackendRef,
-}
-
-impl MaintenanceModeManager {
-    pub fn new(kv_backend: KvBackendRef) -> Self {
-        Self { kv_backend }
-    }
-
-    /// Enables maintenance mode.
-    pub async fn set_maintenance_mode(&self) -> Result<()> {
-        let req = PutRequest {
-            key: Vec::from(MAINTENANCE_KEY),
-            value: vec![],
-            prev_kv: false,
-        };
-        self.kv_backend.put(req).await?;
-        Ok(())
-    }
-
-    /// Unsets maintenance mode.
-    pub async fn unset_maintenance_mode(&self) -> Result<()> {
-        self.kv_backend
-            .delete(MAINTENANCE_KEY.as_bytes(), false)
-            .await?;
-        Ok(())
-    }
-
-    /// Returns true if maintenance mode is enabled.
-    pub async fn maintenance_mode(&self) -> Result<bool> {
-        self.kv_backend.exists(MAINTENANCE_KEY.as_bytes()).await
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use crate::key::maintenance::MaintenanceModeManager;
-    use crate::kv_backend::memory::MemoryKvBackend;
-
-    #[tokio::test]
-    async fn test_maintenance_mode_manager() {
-        let maintenance_mode_manager = Arc::new(MaintenanceModeManager::new(Arc::new(
-            MemoryKvBackend::new(),
-        )));
-        assert!(!maintenance_mode_manager.maintenance_mode().await.unwrap());
-        maintenance_mode_manager
-            .set_maintenance_mode()
-            .await
-            .unwrap();
-        assert!(maintenance_mode_manager.maintenance_mode().await.unwrap());
-        maintenance_mode_manager
-            .unset_maintenance_mode()
-            .await
-            .unwrap();
-        assert!(!maintenance_mode_manager.maintenance_mode().await.unwrap());
-    }
-}

src/common/meta/src/key/runtime_switch.rs

@@ -0,0 +1,250 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use common_error::ext::BoxedError;
+use common_procedure::local::PauseAware;
+use moka::future::Cache;
+use snafu::ResultExt;
+
+use crate::error::{GetCacheSnafu, Result};
+use crate::key::{LEGACY_MAINTENANCE_KEY, MAINTENANCE_KEY, PAUSE_PROCEDURE_KEY, RECOVERY_MODE_KEY};
+use crate::kv_backend::KvBackendRef;
+use crate::rpc::store::{BatchDeleteRequest, PutRequest};
+
+pub type RuntimeSwitchManagerRef = Arc<RuntimeSwitchManager>;
+
+/// The runtime switch manager.
+///
+/// Used to enable or disable runtime switches.
+#[derive(Clone)]
+pub struct RuntimeSwitchManager {
+    kv_backend: KvBackendRef,
+    cache: Cache<Vec<u8>, Option<Vec<u8>>>,
+}
+
+#[async_trait::async_trait]
+impl PauseAware for RuntimeSwitchManager {
+    async fn is_paused(&self) -> std::result::Result<bool, BoxedError> {
+        self.is_procedure_paused().await.map_err(BoxedError::new)
+    }
+}
+
+const CACHE_TTL: Duration = Duration::from_secs(10);
+const MAX_CAPACITY: u64 = 32;
+
+impl RuntimeSwitchManager {
+    pub fn new(kv_backend: KvBackendRef) -> Self {
+        let cache = Cache::builder()
+            .time_to_live(CACHE_TTL)
+            .max_capacity(MAX_CAPACITY)
+            .build();
+        Self { kv_backend, cache }
+    }
+
+    async fn put_key(&self, key: &str) -> Result<()> {
+        let req = PutRequest {
+            key: Vec::from(key),
+            value: vec![],
+            prev_kv: false,
+        };
+        self.kv_backend.put(req).await?;
+        self.cache.invalidate(key.as_bytes()).await;
+        Ok(())
+    }
+
+    async fn delete_keys(&self, keys: &[&str]) -> Result<()> {
+        let req = BatchDeleteRequest::new()
+            .with_keys(keys.iter().map(|x| x.as_bytes().to_vec()).collect());
+        self.kv_backend.batch_delete(req).await?;
+        for key in keys {
+            self.cache.invalidate(key.as_bytes()).await;
+        }
+        Ok(())
+    }
+
+    /// Returns true if the key exists.
+    async fn exists(&self, key: &str) -> Result<bool> {
+        let key = key.as_bytes().to_vec();
+        let kv_backend = self.kv_backend.clone();
+        let value = self
+            .cache
+            .try_get_with(key.clone(), async move {
+                kv_backend.get(&key).await.map(|v| v.map(|v| v.value))
+            })
+            .await
+            .context(GetCacheSnafu)?;
+
+        Ok(value.is_some())
+    }
+
+    /// Enables maintenance mode.
+    pub async fn set_maintenance_mode(&self) -> Result<()> {
+        self.put_key(MAINTENANCE_KEY).await
+    }
+
+    /// Unsets maintenance mode.
+    pub async fn unset_maintenance_mode(&self) -> Result<()> {
+        self.delete_keys(&[MAINTENANCE_KEY, LEGACY_MAINTENANCE_KEY])
+            .await
+    }
+
+    /// Returns true if maintenance mode is enabled.
+    pub async fn maintenance_mode(&self) -> Result<bool> {
+        let exists = self.exists(MAINTENANCE_KEY).await?;
+        if exists {
+            return Ok(true);
+        }
+
+        let exists = self.exists(LEGACY_MAINTENANCE_KEY).await?;
+        if exists {
+            return Ok(true);
+        }
+
+        Ok(false)
+    }
+
+    // Pauses handling of incoming procedure requests.
+    pub async fn pasue_procedure(&self) -> Result<()> {
+        self.put_key(PAUSE_PROCEDURE_KEY).await
+    }
+
+    /// Resumes processing of incoming procedure requests.
+    pub async fn resume_procedure(&self) -> Result<()> {
+        self.delete_keys(&[PAUSE_PROCEDURE_KEY]).await
+    }
+
+    /// Returns true if the system is currently pausing incoming procedure requests.
+    pub async fn is_procedure_paused(&self) -> Result<bool> {
+        self.exists(PAUSE_PROCEDURE_KEY).await
+    }
+
+    /// Enables recovery mode.
+    pub async fn set_recovery_mode(&self) -> Result<()> {
+        self.put_key(RECOVERY_MODE_KEY).await
+    }
+
+    /// Unsets recovery mode.
+    pub async fn unset_recovery_mode(&self) -> Result<()> {
+        self.delete_keys(&[RECOVERY_MODE_KEY]).await
+    }
+
+    /// Returns true if the system is currently in recovery mode.
+    pub async fn recovery_mode(&self) -> Result<bool> {
+        self.exists(RECOVERY_MODE_KEY).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use crate::key::runtime_switch::RuntimeSwitchManager;
+    use crate::key::{LEGACY_MAINTENANCE_KEY, MAINTENANCE_KEY};
+    use crate::kv_backend::memory::MemoryKvBackend;
+    use crate::kv_backend::KvBackend;
+    use crate::rpc::store::PutRequest;
+
+    #[tokio::test]
+    async fn test_runtime_switch_manager_basic() {
+        let runtime_switch_manager =
+            Arc::new(RuntimeSwitchManager::new(Arc::new(MemoryKvBackend::new())));
+        runtime_switch_manager
+            .put_key(MAINTENANCE_KEY)
+            .await
+            .unwrap();
+        let v = runtime_switch_manager
+            .cache
+            .get(MAINTENANCE_KEY.as_bytes())
+            .await;
+        assert!(v.is_none());
+        runtime_switch_manager
+            .exists(MAINTENANCE_KEY)
+            .await
+            .unwrap();
+        let v = runtime_switch_manager
+            .cache
+            .get(MAINTENANCE_KEY.as_bytes())
+            .await;
+        assert!(v.is_some());
+        runtime_switch_manager
+            .delete_keys(&[MAINTENANCE_KEY])
+            .await
+            .unwrap();
+        let v = runtime_switch_manager
+            .cache
+            .get(MAINTENANCE_KEY.as_bytes())
+            .await;
+        assert!(v.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_runtime_switch_manager() {
+        let runtime_switch_manager =
+            Arc::new(RuntimeSwitchManager::new(Arc::new(MemoryKvBackend::new())));
+        assert!(!runtime_switch_manager.maintenance_mode().await.unwrap());
+        runtime_switch_manager.set_maintenance_mode().await.unwrap();
+        assert!(runtime_switch_manager.maintenance_mode().await.unwrap());
+        runtime_switch_manager
+            .unset_maintenance_mode()
+            .await
+            .unwrap();
+        assert!(!runtime_switch_manager.maintenance_mode().await.unwrap());
+    }
+
+    #[tokio::test]
+    async fn test_runtime_switch_manager_with_legacy_key() {
+        let kv_backend = Arc::new(MemoryKvBackend::new());
+        kv_backend
+            .put(PutRequest {
+                key: Vec::from(LEGACY_MAINTENANCE_KEY),
+                value: vec![],
+                prev_kv: false,
+            })
+            .await
+            .unwrap();
+        let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend));
+        assert!(runtime_switch_manager.maintenance_mode().await.unwrap());
+        runtime_switch_manager
+            .unset_maintenance_mode()
+            .await
+            .unwrap();
+        assert!(!runtime_switch_manager.maintenance_mode().await.unwrap());
+        runtime_switch_manager.set_maintenance_mode().await.unwrap();
+        assert!(runtime_switch_manager.maintenance_mode().await.unwrap());
+    }
+
+    #[tokio::test]
+    async fn test_pasue_procedure() {
+        let runtime_switch_manager =
+            Arc::new(RuntimeSwitchManager::new(Arc::new(MemoryKvBackend::new())));
+        runtime_switch_manager.pasue_procedure().await.unwrap();
+        assert!(runtime_switch_manager.is_procedure_paused().await.unwrap());
+        runtime_switch_manager.resume_procedure().await.unwrap();
+        assert!(!runtime_switch_manager.is_procedure_paused().await.unwrap());
+    }
+
+    #[tokio::test]
+    async fn test_recovery_mode() {
+        let runtime_switch_manager =
+            Arc::new(RuntimeSwitchManager::new(Arc::new(MemoryKvBackend::new())));
+        assert!(!runtime_switch_manager.recovery_mode().await.unwrap());
+        runtime_switch_manager.set_recovery_mode().await.unwrap();
+        assert!(runtime_switch_manager.recovery_mode().await.unwrap());
+        runtime_switch_manager.unset_recovery_mode().await.unwrap();
+        assert!(!runtime_switch_manager.recovery_mode().await.unwrap());
+    }
+}

src/common/meta/src/key/table_info.rs

@@ -334,6 +334,7 @@ mod tests {
             options: Default::default(),
             region_numbers: vec![1],
             partition_key_indices: vec![],
+            column_ids: vec![],
         };
 
         RawTableInfo {

src/common/meta/src/key/table_name.rs

@@ -103,6 +103,26 @@ pub fn table_decoder(kv: KeyValue) -> Result<(String, TableNameValue)> {
     Ok((table_name_key.table.to_string(), table_name_value))
 }
 
+impl<'a> From<&TableReference<'a>> for TableNameKey<'a> {
+    fn from(value: &TableReference<'a>) -> Self {
+        Self {
+            catalog: value.catalog,
+            schema: value.schema,
+            table: value.table,
+        }
+    }
+}
+
+impl<'a> From<TableReference<'a>> for TableNameKey<'a> {
+    fn from(value: TableReference<'a>) -> Self {
+        Self {
+            catalog: value.catalog,
+            schema: value.schema,
+            table: value.table,
+        }
+    }
+}
+
 impl<'a> From<&'a TableName> for TableNameKey<'a> {
     fn from(value: &'a TableName) -> Self {
         Self {

src/common/meta/src/key/table_route.rs

@@ -184,6 +184,17 @@ impl TableRouteValue {
         }
     }
 
+    /// Converts to [`LogicalTableRouteValue`].
+    ///
+    /// # Panic
+    /// If it is not the [`LogicalTableRouteValue`].
+    pub fn into_logical_table_route(self) -> LogicalTableRouteValue {
+        match self {
+            TableRouteValue::Logical(x) => x,
+            _ => unreachable!("Mistakenly been treated as a Logical TableRoute: {self:?}"),
+        }
+    }
+
     pub fn region_numbers(&self) -> Vec<RegionNumber> {
         match self {
             TableRouteValue::Physical(x) => x

src/common/meta/src/key/tombstone.rs

@@ -14,13 +14,14 @@
 
 use std::collections::HashMap;
 
+use common_telemetry::debug;
 use snafu::ensure;
 
 use crate::error::{self, Result};
 use crate::key::txn_helper::TxnOpGetResponseSet;
 use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp};
 use crate::kv_backend::KvBackendRef;
-use crate::rpc::store::BatchGetRequest;
+use crate::rpc::store::{BatchDeleteRequest, BatchGetRequest};
 
 /// [TombstoneManager] provides the ability to:
 /// - logically delete values
@@ -28,6 +29,9 @@ use crate::rpc::store::BatchGetRequest;
 pub struct TombstoneManager {
     kv_backend: KvBackendRef,
     tombstone_prefix: String,
+    // Only used for testing.
+    #[cfg(test)]
+    max_txn_ops: Option<usize>,
 }
 
 const TOMBSTONE_PREFIX: &str = "__tombstone/";
@@ -35,10 +39,7 @@ const TOMBSTONE_PREFIX: &str = "__tombstone/";
 impl TombstoneManager {
     /// Returns [TombstoneManager].
     pub fn new(kv_backend: KvBackendRef) -> Self {
-        Self {
+        Self::new_with_prefix(kv_backend, TOMBSTONE_PREFIX)
-            kv_backend,
-            tombstone_prefix: TOMBSTONE_PREFIX.to_string(),
-        }
     }
 
     /// Returns [TombstoneManager] with a custom tombstone prefix.
@@ -46,6 +47,8 @@ impl TombstoneManager {
         Self {
             kv_backend,
             tombstone_prefix: prefix.to_string(),
+            #[cfg(test)]
+            max_txn_ops: None,
         }
     }
 
@@ -53,6 +56,11 @@ impl TombstoneManager {
         [self.tombstone_prefix.as_bytes(), key].concat()
     }
 
+    #[cfg(test)]
+    pub fn set_max_txn_ops(&mut self, max_txn_ops: usize) {
+        self.max_txn_ops = Some(max_txn_ops);
+    }
+
     /// Moves value to `dest_key`.
     ///
     /// Puts `value` to `dest_key` if the value of `src_key` equals `value`.
@@ -83,7 +91,11 @@ impl TombstoneManager {
         ensure!(
             keys.len() == dest_keys.len(),
             error::UnexpectedSnafu {
-                err_msg: "The length of keys does not match the length of dest_keys."
+                err_msg: format!(
+                    "The length of keys({}) does not match the length of dest_keys({}).",
+                    keys.len(),
+                    dest_keys.len()
+                ),
             }
         );
         // The key -> dest key mapping.
@@ -136,19 +148,45 @@ impl TombstoneManager {
         .fail()
     }
 
+    fn max_txn_ops(&self) -> usize {
+        #[cfg(test)]
+        if let Some(max_txn_ops) = self.max_txn_ops {
+            return max_txn_ops;
+        }
+        self.kv_backend.max_txn_ops()
+    }
+
     /// Moves values to `dest_key`.
     ///
     /// Returns the number of keys that were moved.
     async fn move_values(&self, keys: Vec<Vec<u8>>, dest_keys: Vec<Vec<u8>>) -> Result<usize> {
-        let chunk_size = self.kv_backend.max_txn_ops() / 2;
+        ensure!(
-        if keys.len() > chunk_size {
+            keys.len() == dest_keys.len(),
-            let keys_chunks = keys.chunks(chunk_size).collect::<Vec<_>>();
+            error::UnexpectedSnafu {
-            let dest_keys_chunks = keys.chunks(chunk_size).collect::<Vec<_>>();
+                err_msg: format!(
-            for (keys, dest_keys) in keys_chunks.into_iter().zip(dest_keys_chunks) {
+                    "The length of keys({}) does not match the length of dest_keys({}).",
-                self.move_values_inner(keys, dest_keys).await?;
+                    keys.len(),
+                    dest_keys.len()
+                ),
             }
-
+        );
-            Ok(keys.len())
+        if keys.is_empty() {
+            return Ok(0);
+        }
+        let chunk_size = self.max_txn_ops() / 2;
+        if keys.len() > chunk_size {
+            debug!(
+                "Moving values with multiple chunks, keys len: {}, chunk_size: {}",
+                keys.len(),
+                chunk_size
+            );
+            let mut moved_keys = 0;
+            let keys_chunks = keys.chunks(chunk_size).collect::<Vec<_>>();
+            let dest_keys_chunks = dest_keys.chunks(chunk_size).collect::<Vec<_>>();
+            for (keys, dest_keys) in keys_chunks.into_iter().zip(dest_keys_chunks) {
+                moved_keys += self.move_values_inner(keys, dest_keys).await?;
+            }
+            Ok(moved_keys)
         } else {
             self.move_values_inner(&keys, &dest_keys).await
         }
@@ -196,15 +234,18 @@ impl TombstoneManager {
     ///
     /// Returns the number of keys that were deleted.
     pub async fn delete(&self, keys: Vec<Vec<u8>>) -> Result<usize> {
-        let operations = keys
+        let keys = keys
             .iter()
-            .map(|key| TxnOp::Delete(self.to_tombstone(key)))
+            .map(|key| self.to_tombstone(key))
             .collect::<Vec<_>>();
 
-        let txn = Txn::new().and_then(operations);
+        let num_keys = keys.len();
-        // Always success.
+        let _ = self
-        let _ = self.kv_backend.txn(txn).await?;
+            .kv_backend
-        Ok(keys.len())
+            .batch_delete(BatchDeleteRequest::new().with_keys(keys))
+            .await?;
+
+        Ok(num_keys)
     }
 }
 
@@ -392,16 +433,73 @@ mod tests {
             .into_iter()
             .map(|kv| (kv.key, kv.dest_key))
             .unzip();
-        tombstone_manager
+        let moved_keys = tombstone_manager
             .move_values(keys.clone(), dest_keys.clone())
             .await
             .unwrap();
+        assert_eq!(kvs.len(), moved_keys);
         check_moved_values(kv_backend.clone(), &move_values).await;
         // Moves again
-        tombstone_manager
+        let moved_keys = tombstone_manager
             .move_values(keys.clone(), dest_keys.clone())
             .await
             .unwrap();
+        assert_eq!(0, moved_keys);
+        check_moved_values(kv_backend.clone(), &move_values).await;
+    }
+
+    #[tokio::test]
+    async fn test_move_values_with_max_txn_ops() {
+        common_telemetry::init_default_ut_logging();
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let mut tombstone_manager = TombstoneManager::new(kv_backend.clone());
+        tombstone_manager.set_max_txn_ops(4);
+        let kvs = HashMap::from([
+            (b"bar".to_vec(), b"baz".to_vec()),
+            (b"foo".to_vec(), b"hi".to_vec()),
+            (b"baz".to_vec(), b"hello".to_vec()),
+            (b"qux".to_vec(), b"world".to_vec()),
+            (b"quux".to_vec(), b"world".to_vec()),
+            (b"quuux".to_vec(), b"world".to_vec()),
+            (b"quuuux".to_vec(), b"world".to_vec()),
+            (b"quuuuux".to_vec(), b"world".to_vec()),
+            (b"quuuuuux".to_vec(), b"world".to_vec()),
+        ]);
+        for (key, value) in &kvs {
+            kv_backend
+                .put(
+                    PutRequest::new()
+                        .with_key(key.clone())
+                        .with_value(value.clone()),
+                )
+                .await
+                .unwrap();
+        }
+        let move_values = kvs
+            .iter()
+            .map(|(key, value)| MoveValue {
+                key: key.clone(),
+                dest_key: tombstone_manager.to_tombstone(key),
+                value: value.clone(),
+            })
+            .collect::<Vec<_>>();
+        let (keys, dest_keys): (Vec<_>, Vec<_>) = move_values
+            .clone()
+            .into_iter()
+            .map(|kv| (kv.key, kv.dest_key))
+            .unzip();
+        let moved_keys = tombstone_manager
+            .move_values(keys.clone(), dest_keys.clone())
+            .await
+            .unwrap();
+        assert_eq!(kvs.len(), moved_keys);
+        check_moved_values(kv_backend.clone(), &move_values).await;
+        // Moves again
+        let moved_keys = tombstone_manager
+            .move_values(keys.clone(), dest_keys.clone())
+            .await
+            .unwrap();
+        assert_eq!(0, moved_keys);
         check_moved_values(kv_backend.clone(), &move_values).await;
     }
 
@@ -439,17 +537,19 @@ mod tests {
             .unzip();
         keys.push(b"non-exists".to_vec());
         dest_keys.push(b"hi/non-exists".to_vec());
-        tombstone_manager
+        let moved_keys = tombstone_manager
             .move_values(keys.clone(), dest_keys.clone())
             .await
             .unwrap();
         check_moved_values(kv_backend.clone(), &move_values).await;
+        assert_eq!(3, moved_keys);
         // Moves again
-        tombstone_manager
+        let moved_keys = tombstone_manager
             .move_values(keys.clone(), dest_keys.clone())
             .await
             .unwrap();
         check_moved_values(kv_backend.clone(), &move_values).await;
+        assert_eq!(0, moved_keys);
     }
 
     #[tokio::test]
@@ -490,10 +590,11 @@ mod tests {
             .into_iter()
             .map(|kv| (kv.key, kv.dest_key))
             .unzip();
-        tombstone_manager
+        let moved_keys = tombstone_manager
             .move_values(keys, dest_keys)
             .await
             .unwrap();
+        assert_eq!(kvs.len(), moved_keys);
     }
 
     #[tokio::test]
@@ -571,4 +672,24 @@ mod tests {
             .unwrap();
         check_moved_values(kv_backend.clone(), &move_values).await;
     }
+
+    #[tokio::test]
+    async fn test_move_values_with_different_lengths() {
+        let kv_backend = Arc::new(MemoryKvBackend::default());
+        let tombstone_manager = TombstoneManager::new(kv_backend.clone());
+
+        let keys = vec![b"bar".to_vec(), b"foo".to_vec()];
+        let dest_keys = vec![b"bar".to_vec(), b"foo".to_vec(), b"baz".to_vec()];
+
+        let err = tombstone_manager
+            .move_values(keys, dest_keys)
+            .await
+            .unwrap_err();
+        assert!(err
+            .to_string()
+            .contains("The length of keys(2) does not match the length of dest_keys(3)."),);
+
+        let moved_keys = tombstone_manager.move_values(vec![], vec![]).await.unwrap();
+        assert_eq!(0, moved_keys);
+    }
 }

src/common/meta/src/lib.rs

@@ -37,7 +37,9 @@ pub mod node_expiry_listener;
 pub mod node_manager;
 pub mod peer;
 pub mod poison_key;
+pub mod procedure_executor;
 pub mod range_stream;
+pub mod reconciliation;
 pub mod region_keeper;
 pub mod region_registry;
 pub mod rpc;

src/common/meta/src/metrics.rs

@@ -15,6 +15,13 @@
 use lazy_static::lazy_static;
 use prometheus::*;
 
+pub const TABLE_TYPE_PHYSICAL: &str = "physical";
+pub const TABLE_TYPE_LOGICAL: &str = "logical";
+pub const ERROR_TYPE_RETRYABLE: &str = "retryable";
+pub const ERROR_TYPE_EXTERNAL: &str = "external";
+pub const STATS_TYPE_NO_REGION_METADATA: &str = "no_region_metadata";
+pub const STATS_TYPE_REGION_NOT_OPEN: &str = "region_not_open";
+
 lazy_static! {
     pub static ref METRIC_META_TXN_REQUEST: HistogramVec = register_histogram_vec!(
         "greptime_meta_txn_request",
@@ -114,4 +121,39 @@ lazy_static! {
         &["backend", "result", "op", "type"]
     )
     .unwrap();
+    pub static ref METRIC_META_RECONCILIATION_LIST_REGION_METADATA_DURATION: HistogramVec =
+        register_histogram_vec!(
+            "greptime_meta_reconciliation_list_region_metadata_duration",
+            "reconciliation list region metadata duration",
+            &["table_type"]
+        )
+        .unwrap();
+    pub static ref METRIC_META_RECONCILIATION_RESOLVED_COLUMN_METADATA: IntCounterVec =
+        register_int_counter_vec!(
+            "greptime_meta_reconciliation_resolved_column_metadata",
+            "reconciliation resolved column metadata",
+            &["strategy"]
+        )
+        .unwrap();
+    pub static ref METRIC_META_RECONCILIATION_STATS: IntCounterVec =
+        register_int_counter_vec!(
+            "greptime_meta_reconciliation_stats",
+            "reconciliation stats",
+            &["procedure_name", "table_type", "type"]
+        )
+        .unwrap();
+    pub static ref METRIC_META_RECONCILIATION_PROCEDURE: HistogramVec =
+        register_histogram_vec!(
+            "greptime_meta_reconciliation_procedure",
+            "reconcile table procedure",
+            &["procedure_name", "step"]
+        )
+        .unwrap();
+    pub static ref METRIC_META_RECONCILIATION_PROCEDURE_ERROR: IntCounterVec =
+        register_int_counter_vec!(
+            "greptime_meta_reconciliation_procedure_error",
+            "reconciliation procedure error",
+            &["procedure_name", "step", "error_type"]
+        )
+        .unwrap();
 }

src/common/meta/src/procedure_executor.rs

@@ -0,0 +1,173 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use api::v1::meta::{ProcedureDetailResponse, ReconcileRequest, ReconcileResponse};
+use common_procedure::{ProcedureId, ProcedureManagerRef};
+use common_telemetry::tracing_context::W3cTrace;
+use snafu::{OptionExt, ResultExt};
+
+use crate::ddl_manager::DdlManagerRef;
+use crate::error::{
+    ParseProcedureIdSnafu, ProcedureNotFoundSnafu, QueryProcedureSnafu, Result, UnsupportedSnafu,
+};
+use crate::rpc::ddl::{SubmitDdlTaskRequest, SubmitDdlTaskResponse};
+use crate::rpc::procedure::{
+    self, AddRegionFollowerRequest, MigrateRegionRequest, MigrateRegionResponse,
+    ProcedureStateResponse, RemoveRegionFollowerRequest,
+};
+
+/// The context of procedure executor.
+#[derive(Debug, Default)]
+pub struct ExecutorContext {
+    pub tracing_context: Option<W3cTrace>,
+}
+
+/// The procedure executor that accepts ddl, region migration task etc.
+#[async_trait::async_trait]
+pub trait ProcedureExecutor: Send + Sync {
+    /// Submit a ddl task
+    async fn submit_ddl_task(
+        &self,
+        ctx: &ExecutorContext,
+        request: SubmitDdlTaskRequest,
+    ) -> Result<SubmitDdlTaskResponse>;
+
+    /// Add a region follower
+    async fn add_region_follower(
+        &self,
+        _ctx: &ExecutorContext,
+        _request: AddRegionFollowerRequest,
+    ) -> Result<()> {
+        UnsupportedSnafu {
+            operation: "add_region_follower",
+        }
+        .fail()
+    }
+
+    /// Remove a region follower
+    async fn remove_region_follower(
+        &self,
+        _ctx: &ExecutorContext,
+        _request: RemoveRegionFollowerRequest,
+    ) -> Result<()> {
+        UnsupportedSnafu {
+            operation: "remove_region_follower",
+        }
+        .fail()
+    }
+
+    /// Submit a region migration task
+    async fn migrate_region(
+        &self,
+        ctx: &ExecutorContext,
+        request: MigrateRegionRequest,
+    ) -> Result<MigrateRegionResponse>;
+
+    /// Submit a reconcile task.
+    async fn reconcile(
+        &self,
+        _ctx: &ExecutorContext,
+        request: ReconcileRequest,
+    ) -> Result<ReconcileResponse>;
+
+    /// Query the procedure state by its id
+    async fn query_procedure_state(
+        &self,
+        ctx: &ExecutorContext,
+        pid: &str,
+    ) -> Result<ProcedureStateResponse>;
+
+    async fn list_procedures(&self, ctx: &ExecutorContext) -> Result<ProcedureDetailResponse>;
+}
+
+pub type ProcedureExecutorRef = Arc<dyn ProcedureExecutor>;
+
+/// The local procedure executor that accepts ddl, region migration task etc.
+pub struct LocalProcedureExecutor {
+    pub ddl_manager: DdlManagerRef,
+    pub procedure_manager: ProcedureManagerRef,
+}
+
+impl LocalProcedureExecutor {
+    pub fn new(ddl_manager: DdlManagerRef, procedure_manager: ProcedureManagerRef) -> Self {
+        Self {
+            ddl_manager,
+            procedure_manager,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl ProcedureExecutor for LocalProcedureExecutor {
+    async fn submit_ddl_task(
+        &self,
+        ctx: &ExecutorContext,
+        request: SubmitDdlTaskRequest,
+    ) -> Result<SubmitDdlTaskResponse> {
+        self.ddl_manager.submit_ddl_task(ctx, request).await
+    }
+
+    async fn migrate_region(
+        &self,
+        _ctx: &ExecutorContext,
+        _request: MigrateRegionRequest,
+    ) -> Result<MigrateRegionResponse> {
+        UnsupportedSnafu {
+            operation: "migrate_region",
+        }
+        .fail()
+    }
+
+    async fn reconcile(
+        &self,
+        _ctx: &ExecutorContext,
+        _request: ReconcileRequest,
+    ) -> Result<ReconcileResponse> {
+        UnsupportedSnafu {
+            operation: "reconcile",
+        }
+        .fail()
+    }
+
+    async fn query_procedure_state(
+        &self,
+        _ctx: &ExecutorContext,
+        pid: &str,
+    ) -> Result<ProcedureStateResponse> {
+        let pid =
+            ProcedureId::parse_str(pid).with_context(|_| ParseProcedureIdSnafu { key: pid })?;
+
+        let state = self
+            .procedure_manager
+            .procedure_state(pid)
+            .await
+            .context(QueryProcedureSnafu)?
+            .with_context(|| ProcedureNotFoundSnafu {
+                pid: pid.to_string(),
+            })?;
+
+        Ok(procedure::procedure_state_to_pb_response(&state))
+    }
+
+    async fn list_procedures(&self, _ctx: &ExecutorContext) -> Result<ProcedureDetailResponse> {
+        let metas = self
+            .procedure_manager
+            .list_procedures()
+            .await
+            .context(QueryProcedureSnafu)?;
+        Ok(procedure::procedure_details_to_pb_response(metas))
+    }
+}

src/common/meta/src/reconciliation.rs

@@ -0,0 +1,20 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod manager;
+pub(crate) mod reconcile_catalog;
+pub(crate) mod reconcile_database;
+pub(crate) mod reconcile_logical_tables;
+pub(crate) mod reconcile_table;
+pub(crate) mod utils;

src/common/meta/src/reconciliation/manager.rs

@@ -0,0 +1,246 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use common_procedure::{
+    watcher, BoxedProcedure, ProcedureId, ProcedureManagerRef, ProcedureWithId,
+};
+use common_telemetry::{error, info, warn};
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::TableId;
+use table::table_name::TableName;
+use table::table_reference::TableReference;
+
+use crate::cache_invalidator::CacheInvalidatorRef;
+use crate::error::{self, Result, TableNotFoundSnafu};
+use crate::key::table_name::TableNameKey;
+use crate::key::TableMetadataManagerRef;
+use crate::node_manager::NodeManagerRef;
+use crate::reconciliation::reconcile_catalog::ReconcileCatalogProcedure;
+use crate::reconciliation::reconcile_database::{ReconcileDatabaseProcedure, DEFAULT_PARALLELISM};
+use crate::reconciliation::reconcile_logical_tables::ReconcileLogicalTablesProcedure;
+use crate::reconciliation::reconcile_table::resolve_column_metadata::ResolveStrategy;
+use crate::reconciliation::reconcile_table::ReconcileTableProcedure;
+use crate::reconciliation::utils::Context;
+
+pub type ReconciliationManagerRef = Arc<ReconciliationManager>;
+
+/// The manager for reconciliation procedures.
+pub struct ReconciliationManager {
+    procedure_manager: ProcedureManagerRef,
+    context: Context,
+}
+
+macro_rules! register_reconcile_loader {
+    ($self:ident, $procedure:ty) => {{
+        let context = $self.context.clone();
+        $self
+            .procedure_manager
+            .register_loader(
+                <$procedure>::TYPE_NAME,
+                Box::new(move |json| {
+                    let context = context.clone();
+                    let procedure = <$procedure>::from_json(context, json)?;
+                    Ok(Box::new(procedure))
+                }),
+            )
+            .context(error::RegisterProcedureLoaderSnafu {
+                type_name: <$procedure>::TYPE_NAME,
+            })?;
+    }};
+}
+
+impl ReconciliationManager {
+    pub fn new(
+        node_manager: NodeManagerRef,
+        table_metadata_manager: TableMetadataManagerRef,
+        cache_invalidator: CacheInvalidatorRef,
+        procedure_manager: ProcedureManagerRef,
+    ) -> Self {
+        Self {
+            procedure_manager,
+            context: Context {
+                node_manager,
+                table_metadata_manager,
+                cache_invalidator,
+            },
+        }
+    }
+
+    /// Try to start the reconciliation manager.
+    ///
+    /// This function will register the procedure loaders for the reconciliation procedures.
+    /// Returns an error if the procedure loaders are already registered.
+    pub fn try_start(&self) -> Result<()> {
+        register_reconcile_loader!(self, ReconcileLogicalTablesProcedure);
+        register_reconcile_loader!(self, ReconcileTableProcedure);
+        register_reconcile_loader!(self, ReconcileDatabaseProcedure);
+        register_reconcile_loader!(self, ReconcileCatalogProcedure);
+
+        Ok(())
+    }
+
+    /// Reconcile a table.
+    ///
+    /// Returns the procedure id of the reconciliation procedure.
+    pub async fn reconcile_table(
+        &self,
+        table_ref: TableReference<'_>,
+        resolve_strategy: ResolveStrategy,
+    ) -> Result<ProcedureId> {
+        let table_name_key =
+            TableNameKey::new(table_ref.catalog, table_ref.schema, table_ref.table);
+        let table_metadata_manager = &self.context.table_metadata_manager;
+        let table_id = table_metadata_manager
+            .table_name_manager()
+            .get(table_name_key)
+            .await?
+            .with_context(|| TableNotFoundSnafu {
+                table_name: table_ref.to_string(),
+            })?
+            .table_id();
+        let (physical_table_id, _) = table_metadata_manager
+            .table_route_manager()
+            .get_physical_table_route(table_id)
+            .await?;
+
+        if physical_table_id == table_id {
+            Ok(self.reconcile_physical_table(table_id, table_ref.into(), resolve_strategy))
+        } else {
+            let physical_table_info = table_metadata_manager
+                .table_info_manager()
+                .get(physical_table_id)
+                .await?
+                .with_context(|| TableNotFoundSnafu {
+                    table_name: format!("table_id: {}", physical_table_id),
+                })?;
+
+            Ok(self.reconcile_logical_tables(
+                physical_table_id,
+                physical_table_info.table_name(),
+                vec![(table_id, table_ref.into())],
+            ))
+        }
+    }
+
+    /// Reconcile a database.
+    ///
+    /// Returns the procedure id of the reconciliation procedure.
+    pub fn reconcile_database(
+        &self,
+        catalog: String,
+        schema: String,
+        resolve_strategy: ResolveStrategy,
+        parallelism: usize,
+    ) -> ProcedureId {
+        let parallelism = normalize_parallelism(parallelism);
+        let procedure = ReconcileDatabaseProcedure::new(
+            self.context.clone(),
+            catalog,
+            schema,
+            false,
+            parallelism,
+            resolve_strategy,
+            false,
+        );
+        self.spawn_procedure(Box::new(procedure))
+    }
+
+    fn reconcile_physical_table(
+        &self,
+        table_id: TableId,
+        table_name: TableName,
+        resolve_strategy: ResolveStrategy,
+    ) -> ProcedureId {
+        let procedure = ReconcileTableProcedure::new(
+            self.context.clone(),
+            table_id,
+            table_name,
+            resolve_strategy,
+            false,
+        );
+        self.spawn_procedure(Box::new(procedure))
+    }
+
+    fn reconcile_logical_tables(
+        &self,
+        physical_table_id: TableId,
+        physical_table_name: TableName,
+        logical_tables: Vec<(TableId, TableName)>,
+    ) -> ProcedureId {
+        let procedure = ReconcileLogicalTablesProcedure::new(
+            self.context.clone(),
+            physical_table_id,
+            physical_table_name,
+            logical_tables,
+            false,
+        );
+        self.spawn_procedure(Box::new(procedure))
+    }
+
+    /// Reconcile a catalog.
+    ///
+    /// Returns the procedure id of the reconciliation procedure.
+    pub fn reconcile_catalog(
+        &self,
+        catalog: String,
+        resolve_strategy: ResolveStrategy,
+        parallelism: usize,
+    ) -> ProcedureId {
+        let parallelism = normalize_parallelism(parallelism);
+        let procedure = ReconcileCatalogProcedure::new(
+            self.context.clone(),
+            catalog,
+            false,
+            resolve_strategy,
+            parallelism,
+        );
+        self.spawn_procedure(Box::new(procedure))
+    }
+
+    fn spawn_procedure(&self, procedure: BoxedProcedure) -> ProcedureId {
+        let procedure_manager = self.procedure_manager.clone();
+        let procedure_with_id = ProcedureWithId::with_random_id(procedure);
+        let procedure_id = procedure_with_id.id;
+        common_runtime::spawn_global(async move {
+            let watcher = &mut match procedure_manager.submit(procedure_with_id).await {
+                Ok(watcher) => watcher,
+                Err(e) => {
+                    error!(e; "Failed to submit reconciliation procedure {procedure_id}");
+                    return;
+                }
+            };
+            if let Err(e) = watcher::wait(watcher).await {
+                error!(e; "Failed to wait reconciliation procedure {procedure_id}");
+                return;
+            }
+
+            info!("Reconciliation procedure {procedure_id} is finished successfully!");
+        });
+        procedure_id
+    }
+}
+
+fn normalize_parallelism(parallelism: usize) -> usize {
+    if parallelism == 0 {
+        warn!(
+            "Parallelism is 0, using default parallelism: {}",
+            DEFAULT_PARALLELISM
+        );
+        DEFAULT_PARALLELISM
+    } else {
+        parallelism
+    }
+}

src/common/meta/src/reconciliation/reconcile_catalog.rs

@@ -0,0 +1,237 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::fmt::Debug;
+use std::time::Instant;
+
+use common_procedure::error::FromJsonSnafu;
+use common_procedure::{
+    Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure,
+    Result as ProcedureResult, Status,
+};
+use futures::stream::BoxStream;
+use serde::{Deserialize, Serialize};
+use snafu::ResultExt;
+
+use crate::cache_invalidator::CacheInvalidatorRef;
+use crate::error::Result;
+use crate::key::TableMetadataManagerRef;
+use crate::lock_key::CatalogLock;
+use crate::metrics;
+use crate::node_manager::NodeManagerRef;
+use crate::reconciliation::reconcile_catalog::start::ReconcileCatalogStart;
+use crate::reconciliation::reconcile_table::resolve_column_metadata::ResolveStrategy;
+use crate::reconciliation::utils::{
+    wait_for_inflight_subprocedures, Context, ReconcileCatalogMetrics, SubprocedureMeta,
+};
+
+pub(crate) mod end;
+pub(crate) mod reconcile_databases;
+pub(crate) mod start;
+
+pub(crate) struct ReconcileCatalogContext {
+    pub node_manager: NodeManagerRef,
+    pub table_metadata_manager: TableMetadataManagerRef,
+    pub cache_invalidator: CacheInvalidatorRef,
+    persistent_ctx: PersistentContext,
+    volatile_ctx: VolatileContext,
+}
+
+impl ReconcileCatalogContext {
+    pub fn new(ctx: Context, persistent_ctx: PersistentContext) -> Self {
+        Self {
+            node_manager: ctx.node_manager,
+            table_metadata_manager: ctx.table_metadata_manager,
+            cache_invalidator: ctx.cache_invalidator,
+            persistent_ctx,
+            volatile_ctx: VolatileContext::default(),
+        }
+    }
+
+    pub(crate) async fn wait_for_inflight_subprocedure(
+        &mut self,
+        procedure_ctx: &ProcedureContext,
+    ) -> Result<()> {
+        if let Some(subprocedure) = self.volatile_ctx.inflight_subprocedure.take() {
+            let subprocedures = [subprocedure];
+            let result = wait_for_inflight_subprocedures(
+                procedure_ctx,
+                &subprocedures,
+                self.persistent_ctx.fast_fail,
+            )
+            .await?;
+            self.volatile_ctx.metrics += result.into();
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct PersistentContext {
+    catalog: String,
+    fast_fail: bool,
+    resolve_strategy: ResolveStrategy,
+    parallelism: usize,
+}
+
+impl PersistentContext {
+    pub fn new(
+        catalog: String,
+        fast_fail: bool,
+        resolve_strategy: ResolveStrategy,
+        parallelism: usize,
+    ) -> Self {
+        Self {
+            catalog,
+            fast_fail,
+            resolve_strategy,
+            parallelism,
+        }
+    }
+}
+
+pub(crate) struct VolatileContext {
+    /// Stores the stream of catalogs.
+    schemas: Option<BoxStream<'static, Result<String>>>,
+    /// Stores the inflight subprocedure.
+    inflight_subprocedure: Option<SubprocedureMeta>,
+    /// Stores the metrics of reconciling catalog.
+    metrics: ReconcileCatalogMetrics,
+    /// The start time of the reconciliation.
+    start_time: Instant,
+}
+
+impl Default for VolatileContext {
+    fn default() -> Self {
+        Self {
+            schemas: None,
+            inflight_subprocedure: None,
+            metrics: Default::default(),
+            start_time: Instant::now(),
+        }
+    }
+}
+
+pub struct ReconcileCatalogProcedure {
+    pub context: ReconcileCatalogContext,
+    state: Box<dyn State>,
+}
+
+impl ReconcileCatalogProcedure {
+    pub const TYPE_NAME: &'static str = "metasrv-procedure::ReconcileCatalog";
+
+    pub fn new(
+        ctx: Context,
+        catalog: String,
+        fast_fail: bool,
+        resolve_strategy: ResolveStrategy,
+        parallelism: usize,
+    ) -> Self {
+        let persistent_ctx =
+            PersistentContext::new(catalog, fast_fail, resolve_strategy, parallelism);
+        let context = ReconcileCatalogContext::new(ctx, persistent_ctx);
+        let state = Box::new(ReconcileCatalogStart);
+        Self { context, state }
+    }
+
+    pub(crate) fn from_json(ctx: Context, json: &str) -> ProcedureResult<Self> {
+        let ProcedureDataOwned {
+            state,
+            persistent_ctx,
+        } = serde_json::from_str(json).context(FromJsonSnafu)?;
+        let context = ReconcileCatalogContext::new(ctx, persistent_ctx);
+        Ok(Self { context, state })
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct ProcedureData<'a> {
+    state: &'a dyn State,
+    persistent_ctx: &'a PersistentContext,
+}
+
+#[derive(Debug, Deserialize)]
+struct ProcedureDataOwned {
+    state: Box<dyn State>,
+    persistent_ctx: PersistentContext,
+}
+
+#[async_trait::async_trait]
+impl Procedure for ReconcileCatalogProcedure {
+    fn type_name(&self) -> &str {
+        Self::TYPE_NAME
+    }
+
+    async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
+        let state = &mut self.state;
+
+        let procedure_name = Self::TYPE_NAME;
+        let step = state.name();
+        let _timer = metrics::METRIC_META_RECONCILIATION_PROCEDURE
+            .with_label_values(&[procedure_name, step])
+            .start_timer();
+        match state.next(&mut self.context, _ctx).await {
+            Ok((next, status)) => {
+                *state = next;
+                Ok(status)
+            }
+            Err(e) => {
+                if e.is_retry_later() {
+                    metrics::METRIC_META_RECONCILIATION_PROCEDURE_ERROR
+                        .with_label_values(&[procedure_name, step, metrics::ERROR_TYPE_RETRYABLE])
+                        .inc();
+                    Err(ProcedureError::retry_later(e))
+                } else {
+                    metrics::METRIC_META_RECONCILIATION_PROCEDURE_ERROR
+                        .with_label_values(&[procedure_name, step, metrics::ERROR_TYPE_EXTERNAL])
+                        .inc();
+                    Err(ProcedureError::external(e))
+                }
+            }
+        }
+    }
+
+    fn dump(&self) -> ProcedureResult<String> {
+        let data = ProcedureData {
+            state: self.state.as_ref(),
+            persistent_ctx: &self.context.persistent_ctx,
+        };
+        serde_json::to_string(&data).context(FromJsonSnafu)
+    }
+
+    fn lock_key(&self) -> LockKey {
+        let catalog = &self.context.persistent_ctx.catalog;
+
+        LockKey::new(vec![CatalogLock::Write(catalog).into()])
+    }
+}
+
+#[async_trait::async_trait]
+#[typetag::serde(tag = "reconcile_catalog_state")]
+pub(crate) trait State: Sync + Send + Debug {
+    fn name(&self) -> &'static str {
+        let type_name = std::any::type_name::<Self>();
+        // short name
+        type_name.split("::").last().unwrap_or(type_name)
+    }
+
+    async fn next(
+        &mut self,
+        ctx: &mut ReconcileCatalogContext,
+        procedure_ctx: &ProcedureContext,
+    ) -> Result<(Box<dyn State>, Status)>;
+
+    fn as_any(&self) -> &dyn Any;
+}

src/common/meta/src/reconciliation/reconcile_catalog/end.rs

@@ -0,0 +1,48 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use common_procedure::{Context as ProcedureContext, Status};
+use common_telemetry::info;
+use serde::{Deserialize, Serialize};
+
+use crate::error::Result;
+use crate::reconciliation::reconcile_catalog::{ReconcileCatalogContext, State};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct ReconcileCatalogEnd;
+
+#[async_trait::async_trait]
+#[typetag::serde]
+impl State for ReconcileCatalogEnd {
+    async fn next(
+        &mut self,
+        ctx: &mut ReconcileCatalogContext,
+        procedure_ctx: &ProcedureContext,
+    ) -> Result<(Box<dyn State>, Status)> {
+        info!(
+            "Catalog reconciliation completed. catalog: {}, procedure_id: {}, metrics: {}, elapsed: {:?}",
+            ctx.persistent_ctx.catalog,
+            procedure_ctx.procedure_id,
+            ctx.volatile_ctx.metrics,
+            ctx.volatile_ctx.start_time.elapsed()
+        );
+        Ok((Box::new(ReconcileCatalogEnd), Status::done()))
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/common/meta/src/reconciliation/reconcile_catalog/reconcile_databases.rs

@@ -0,0 +1,104 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use common_procedure::{Context as ProcedureContext, ProcedureWithId, Status};
+use common_telemetry::info;
+use futures::TryStreamExt;
+use serde::{Deserialize, Serialize};
+
+use crate::error::Result;
+use crate::reconciliation::reconcile_catalog::end::ReconcileCatalogEnd;
+use crate::reconciliation::reconcile_catalog::{ReconcileCatalogContext, State};
+use crate::reconciliation::reconcile_database::ReconcileDatabaseProcedure;
+use crate::reconciliation::utils::{Context, SubprocedureMeta};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct ReconcileDatabases;
+
+#[async_trait::async_trait]
+#[typetag::serde]
+impl State for ReconcileDatabases {
+    async fn next(
+        &mut self,
+        ctx: &mut ReconcileCatalogContext,
+        procedure_ctx: &ProcedureContext,
+    ) -> Result<(Box<dyn State>, Status)> {
+        // Waits for inflight subprocedure first.
+        ctx.wait_for_inflight_subprocedure(procedure_ctx).await?;
+
+        if ctx.volatile_ctx.schemas.as_deref().is_none() {
+            let schemas = ctx
+                .table_metadata_manager
+                .schema_manager()
+                .schema_names(&ctx.persistent_ctx.catalog);
+            ctx.volatile_ctx.schemas = Some(schemas);
+        }
+
+        if let Some(catalog) = ctx
+            .volatile_ctx
+            .schemas
+            .as_mut()
+            .unwrap()
+            .try_next()
+            .await?
+        {
+            return Self::schedule_reconcile_database(ctx, catalog);
+        }
+
+        Ok((Box::new(ReconcileCatalogEnd), Status::executing(false)))
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl ReconcileDatabases {
+    fn schedule_reconcile_database(
+        ctx: &mut ReconcileCatalogContext,
+        schema: String,
+    ) -> Result<(Box<dyn State>, Status)> {
+        let context = Context {
+            node_manager: ctx.node_manager.clone(),
+            table_metadata_manager: ctx.table_metadata_manager.clone(),
+            cache_invalidator: ctx.cache_invalidator.clone(),
+        };
+        info!(
+            "Scheduling reconcile database: {}, catalog: {}",
+            schema, ctx.persistent_ctx.catalog
+        );
+        let procedure = ReconcileDatabaseProcedure::new(
+            context,
+            ctx.persistent_ctx.catalog.clone(),
+            schema.clone(),
+            ctx.persistent_ctx.fast_fail,
+            ctx.persistent_ctx.parallelism,
+            ctx.persistent_ctx.resolve_strategy,
+            true,
+        );
+        let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure));
+        ctx.volatile_ctx.inflight_subprocedure = Some(SubprocedureMeta::new_reconcile_database(
+            procedure_with_id.id,
+            ctx.persistent_ctx.catalog.clone(),
+            schema,
+        ));
+
+        Ok((
+            Box::new(ReconcileDatabases),
+            Status::suspended(vec![procedure_with_id], false),
+        ))
+    }
+}

src/common/meta/src/reconciliation/reconcile_catalog/start.rs

@@ -0,0 +1,58 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use common_procedure::{Context as ProcedureContext, Status};
+use serde::{Deserialize, Serialize};
+use snafu::ensure;
+
+use crate::error::{self, Result};
+use crate::key::catalog_name::CatalogNameKey;
+use crate::reconciliation::reconcile_catalog::reconcile_databases::ReconcileDatabases;
+use crate::reconciliation::reconcile_catalog::{ReconcileCatalogContext, State};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct ReconcileCatalogStart;
+
+#[async_trait::async_trait]
+#[typetag::serde]
+impl State for ReconcileCatalogStart {
+    async fn next(
+        &mut self,
+        ctx: &mut ReconcileCatalogContext,
+        _procedure_ctx: &ProcedureContext,
+    ) -> Result<(Box<dyn State>, Status)> {
+        let exists = ctx
+            .table_metadata_manager
+            .catalog_manager()
+            .exists(CatalogNameKey {
+                catalog: &ctx.persistent_ctx.catalog,
+            })
+            .await?;
+
+        ensure!(
+            exists,
+            error::CatalogNotFoundSnafu {
+                catalog: &ctx.persistent_ctx.catalog
+            },
+        );
+
+        Ok((Box::new(ReconcileDatabases), Status::executing(true)))
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/common/meta/src/reconciliation/reconcile_database.rs

@@ -0,0 +1,285 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) mod end;
+pub(crate) mod reconcile_logical_tables;
+pub(crate) mod reconcile_tables;
+pub(crate) mod start;
+
+use std::any::Any;
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::time::Instant;
+
+use async_trait::async_trait;
+use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
+use common_procedure::{
+    Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure,
+    Result as ProcedureResult, Status,
+};
+use futures::stream::BoxStream;
+use serde::{Deserialize, Serialize};
+use snafu::ResultExt;
+use store_api::storage::TableId;
+use table::table_name::TableName;
+
+use crate::cache_invalidator::CacheInvalidatorRef;
+use crate::error::Result;
+use crate::key::table_name::TableNameValue;
+use crate::key::TableMetadataManagerRef;
+use crate::lock_key::{CatalogLock, SchemaLock};
+use crate::metrics;
+use crate::node_manager::NodeManagerRef;
+use crate::reconciliation::reconcile_database::start::ReconcileDatabaseStart;
+use crate::reconciliation::reconcile_table::resolve_column_metadata::ResolveStrategy;
+use crate::reconciliation::utils::{
+    wait_for_inflight_subprocedures, Context, ReconcileDatabaseMetrics, SubprocedureMeta,
+};
+pub(crate) const DEFAULT_PARALLELISM: usize = 64;
+
+pub(crate) struct ReconcileDatabaseContext {
+    pub node_manager: NodeManagerRef,
+    pub table_metadata_manager: TableMetadataManagerRef,
+    pub cache_invalidator: CacheInvalidatorRef,
+    persistent_ctx: PersistentContext,
+    volatile_ctx: VolatileContext,
+}
+
+impl ReconcileDatabaseContext {
+    pub fn new(ctx: Context, persistent_ctx: PersistentContext) -> Self {
+        Self {
+            node_manager: ctx.node_manager,
+            table_metadata_manager: ctx.table_metadata_manager,
+            cache_invalidator: ctx.cache_invalidator,
+            persistent_ctx,
+            volatile_ctx: VolatileContext::default(),
+        }
+    }
+
+    /// Waits for inflight subprocedures to complete.
+    pub(crate) async fn wait_for_inflight_subprocedures(
+        &mut self,
+        procedure_ctx: &ProcedureContext,
+    ) -> Result<()> {
+        if !self.volatile_ctx.inflight_subprocedures.is_empty() {
+            let result = wait_for_inflight_subprocedures(
+                procedure_ctx,
+                &self.volatile_ctx.inflight_subprocedures,
+                self.persistent_ctx.fail_fast,
+            )
+            .await?;
+
+            // Collects result into metrics
+            let metrics = result.into();
+            self.volatile_ctx.inflight_subprocedures.clear();
+            self.volatile_ctx.metrics += metrics;
+        }
+
+        Ok(())
+    }
+
+    /// Returns the immutable metrics.
+    pub(crate) fn metrics(&self) -> &ReconcileDatabaseMetrics {
+        &self.volatile_ctx.metrics
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct PersistentContext {
+    catalog: String,
+    schema: String,
+    fail_fast: bool,
+    parallelism: usize,
+    resolve_strategy: ResolveStrategy,
+    is_subprocedure: bool,
+}
+
+impl PersistentContext {
+    pub fn new(
+        catalog: String,
+        schema: String,
+        fail_fast: bool,
+        parallelism: usize,
+        resolve_strategy: ResolveStrategy,
+        is_subprocedure: bool,
+    ) -> Self {
+        Self {
+            catalog,
+            schema,
+            fail_fast,
+            parallelism,
+            resolve_strategy,
+            is_subprocedure,
+        }
+    }
+}
+
+pub(crate) struct VolatileContext {
+    /// Stores pending physical tables.
+    pending_tables: Vec<(TableId, TableName)>,
+    /// Stores pending logical tables associated with each physical table.
+    ///
+    /// - Key: Table ID of the physical table.
+    /// - Value: Vector of (TableId, TableName) tuples representing logical tables belonging to the physical table.
+    pending_logical_tables: HashMap<TableId, Vec<(TableId, TableName)>>,
+    /// Stores inflight subprocedures.
+    inflight_subprocedures: Vec<SubprocedureMeta>,
+    /// Stores the stream of tables.
+    tables: Option<BoxStream<'static, Result<(String, TableNameValue)>>>,
+    /// The metrics of reconciling database.
+    metrics: ReconcileDatabaseMetrics,
+    /// The start time of the reconciliation.
+    start_time: Instant,
+}
+
+impl Default for VolatileContext {
+    fn default() -> Self {
+        Self {
+            pending_tables: vec![],
+            pending_logical_tables: HashMap::new(),
+            inflight_subprocedures: vec![],
+            tables: None,
+            metrics: ReconcileDatabaseMetrics::default(),
+            start_time: Instant::now(),
+        }
+    }
+}
+
+pub struct ReconcileDatabaseProcedure {
+    pub context: ReconcileDatabaseContext,
+    state: Box<dyn State>,
+}
+
+impl ReconcileDatabaseProcedure {
+    pub const TYPE_NAME: &'static str = "metasrv-procedure::ReconcileDatabase";
+
+    pub fn new(
+        ctx: Context,
+        catalog: String,
+        schema: String,
+        fail_fast: bool,
+        parallelism: usize,
+        resolve_strategy: ResolveStrategy,
+        is_subprocedure: bool,
+    ) -> Self {
+        let persistent_ctx = PersistentContext::new(
+            catalog,
+            schema,
+            fail_fast,
+            parallelism,
+            resolve_strategy,
+            is_subprocedure,
+        );
+        let context = ReconcileDatabaseContext::new(ctx, persistent_ctx);
+        let state = Box::new(ReconcileDatabaseStart);
+        Self { context, state }
+    }
+
+    pub(crate) fn from_json(ctx: Context, json: &str) -> ProcedureResult<Self> {
+        let ProcedureDataOwned {
+            state,
+            persistent_ctx,
+        } = serde_json::from_str(json).context(FromJsonSnafu)?;
+        let context = ReconcileDatabaseContext::new(ctx, persistent_ctx);
+        Ok(Self { context, state })
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct ProcedureData<'a> {
+    state: &'a dyn State,
+    persistent_ctx: &'a PersistentContext,
+}
+
+#[derive(Debug, Deserialize)]
+struct ProcedureDataOwned {
+    state: Box<dyn State>,
+    persistent_ctx: PersistentContext,
+}
+
+#[async_trait]
+impl Procedure for ReconcileDatabaseProcedure {
+    fn type_name(&self) -> &str {
+        Self::TYPE_NAME
+    }
+
+    async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
+        let state = &mut self.state;
+
+        let procedure_name = Self::TYPE_NAME;
+        let step = state.name();
+        let _timer = metrics::METRIC_META_RECONCILIATION_PROCEDURE
+            .with_label_values(&[procedure_name, step])
+            .start_timer();
+        match state.next(&mut self.context, _ctx).await {
+            Ok((next, status)) => {
+                *state = next;
+                Ok(status)
+            }
+            Err(e) => {
+                if e.is_retry_later() {
+                    metrics::METRIC_META_RECONCILIATION_PROCEDURE_ERROR
+                        .with_label_values(&[procedure_name, step, metrics::ERROR_TYPE_RETRYABLE])
+                        .inc();
+                    Err(ProcedureError::retry_later(e))
+                } else {
+                    metrics::METRIC_META_RECONCILIATION_PROCEDURE_ERROR
+                        .with_label_values(&[procedure_name, step, metrics::ERROR_TYPE_EXTERNAL])
+                        .inc();
+                    Err(ProcedureError::external(e))
+                }
+            }
+        }
+    }
+
+    fn dump(&self) -> ProcedureResult<String> {
+        let data = ProcedureData {
+            state: self.state.as_ref(),
+            persistent_ctx: &self.context.persistent_ctx,
+        };
+        serde_json::to_string(&data).context(ToJsonSnafu)
+    }
+
+    fn lock_key(&self) -> LockKey {
+        let catalog = &self.context.persistent_ctx.catalog;
+        let schema = &self.context.persistent_ctx.schema;
+        // If the procedure is a subprocedure, only lock the schema.
+        if self.context.persistent_ctx.is_subprocedure {
+            return LockKey::new(vec![SchemaLock::write(catalog, schema).into()]);
+        }
+
+        LockKey::new(vec![
+            CatalogLock::Read(catalog).into(),
+            SchemaLock::write(catalog, schema).into(),
+        ])
+    }
+}
+
+#[async_trait::async_trait]
+#[typetag::serde(tag = "reconcile_database_state")]
+pub(crate) trait State: Sync + Send + Debug {
+    fn name(&self) -> &'static str {
+        let type_name = std::any::type_name::<Self>();
+        // short name
+        type_name.split("::").last().unwrap_or(type_name)
+    }
+
+    async fn next(
+        &mut self,
+        ctx: &mut ReconcileDatabaseContext,
+        procedure_ctx: &ProcedureContext,
+    ) -> Result<(Box<dyn State>, Status)>;
+
+    fn as_any(&self) -> &dyn Any;
+}

src/common/meta/src/reconciliation/reconcile_database/end.rs

@@ -0,0 +1,49 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use common_procedure::{Context as ProcedureContext, Status};
+use common_telemetry::info;
+use serde::{Deserialize, Serialize};
+
+use crate::error::Result;
+use crate::reconciliation::reconcile_database::{ReconcileDatabaseContext, State};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct ReconcileDatabaseEnd;
+
+#[async_trait::async_trait]
+#[typetag::serde]
+impl State for ReconcileDatabaseEnd {
+    async fn next(
+        &mut self,
+        ctx: &mut ReconcileDatabaseContext,
+        procedure_ctx: &ProcedureContext,
+    ) -> Result<(Box<dyn State>, Status)> {
+        info!(
+            "Database reconciliation completed. schema: {}, catalog: {}, procedure_id: {}, metrics: {}, elapsed: {:?}",
+            ctx.persistent_ctx.schema,
+            ctx.persistent_ctx.catalog,
+            procedure_ctx.procedure_id,
+            ctx.metrics(),
+            ctx.volatile_ctx.start_time.elapsed(),
+        );
+        Ok((Box::new(ReconcileDatabaseEnd), Status::done()))
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}