Revert "build(deps): upgrade opendal to 0.46 (#4037 )"

This reverts commit f9db5ff0d6.
Merge branch 'main' into avoid-query-meta
2025-12-25 07:30:02 +00:00 · 2024-06-03 20:28:59 +08:00 · 2024-05-31 18:16:36 +08:00 · 2024-05-31 18:16:08 +08:00 · 2024-05-31 09:25:14 +00:00 · 2024-05-31 06:50:22 +00:00
229 changed files with 6696 additions and 4054 deletions
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -212,7 +212,14 @@ jobs:
          path: .
      - name: Unzip binaries
        run: tar -xvf ./bins.tar.gz
-      - name: Fuzz Test
+      - name: Build Fuzz Test
+        shell: bash
+        run: |
+          cd tests-fuzz &
+          cargo install cargo-gc-bin &
+          cargo gc &
+          cd ..
+      - name: Run Fuzz Test
        uses: ./.github/actions/fuzz-test
        env:
          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -214,7 +214,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"

 [[package]]
 name = "api"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "common-base",
 "common-decimal",
@@ -703,7 +703,7 @@ dependencies = [

 [[package]]
 name = "auth"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
@@ -877,7 +877,7 @@ dependencies = [

 [[package]]
 name = "benchmarks"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arrow",
@@ -904,7 +904,6 @@ dependencies = [
 "rskafka",
 "serde",
 "store-api",
- "tests-integration",
 "tokio",
 "toml 0.8.12",
 "uuid",
@@ -1220,7 +1219,7 @@ dependencies = [

 [[package]]
 name = "cache"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "catalog",
 "common-error",
@@ -1228,6 +1227,7 @@ dependencies = [
 "common-meta",
 "moka",
 "snafu 0.8.2",
+ "substrait 0.8.1",
 ]

 [[package]]
@@ -1254,13 +1254,15 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"

 [[package]]
 name = "catalog"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arrow",
 "arrow-schema",
 "async-stream",
 "async-trait",
+ "bytes",
+ "cache",
 "catalog",
 "chrono",
 "common-catalog",
@@ -1534,7 +1536,7 @@ checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"

 [[package]]
 name = "client"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arc-swap",
@@ -1563,7 +1565,7 @@ dependencies = [
 "serde_json",
 "snafu 0.8.2",
 "substrait 0.17.1",
- "substrait 0.8.0",
+ "substrait 0.8.1",
 "tokio",
 "tokio-stream",
 "tonic 0.11.0",
@@ -1593,7 +1595,7 @@ dependencies = [

 [[package]]
 name = "cmd"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "auth",
@@ -1649,7 +1651,7 @@ dependencies = [
 "session",
 "snafu 0.8.2",
 "store-api",
- "substrait 0.8.0",
+ "substrait 0.8.1",
 "table",
 "temp-env",
 "tempfile",
@@ -1694,7 +1696,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"

 [[package]]
 name = "common-base"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "anymap",
 "bitvec",
@@ -1710,7 +1712,7 @@ dependencies = [

 [[package]]
 name = "common-catalog"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "chrono",
 "common-error",
@@ -1721,7 +1723,7 @@ dependencies = [

 [[package]]
 name = "common-config"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "common-base",
 "common-error",
@@ -1744,7 +1746,7 @@ dependencies = [

 [[package]]
 name = "common-datasource"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arrow",
 "arrow-schema",
@@ -1776,7 +1778,7 @@ dependencies = [

 [[package]]
 name = "common-decimal"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "bigdecimal",
 "common-error",
@@ -1789,7 +1791,7 @@ dependencies = [

 [[package]]
 name = "common-error"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "snafu 0.8.2",
 "strum 0.25.0",
@@ -1797,7 +1799,7 @@ dependencies = [

 [[package]]
 name = "common-frontend"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
@@ -1812,7 +1814,7 @@ dependencies = [

 [[package]]
 name = "common-function"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arc-swap",
@@ -1845,7 +1847,7 @@ dependencies = [

 [[package]]
 name = "common-greptimedb-telemetry"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "common-runtime",
@@ -1862,7 +1864,7 @@ dependencies = [

 [[package]]
 name = "common-grpc"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arrow-flight",
@@ -1888,7 +1890,7 @@ dependencies = [

 [[package]]
 name = "common-grpc-expr"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "common-base",
@@ -1905,7 +1907,7 @@ dependencies = [

 [[package]]
 name = "common-macro"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arc-swap",
 "common-query",
@@ -1920,7 +1922,7 @@ dependencies = [

 [[package]]
 name = "common-mem-prof"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "common-error",
 "common-macro",
@@ -1933,7 +1935,7 @@ dependencies = [

 [[package]]
 name = "common-meta"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "anymap2",
 "api",
@@ -1986,11 +1988,11 @@ dependencies = [

 [[package]]
 name = "common-plugins"
-version = "0.8.0"
+version = "0.8.1"

 [[package]]
 name = "common-procedure"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-stream",
 "async-trait",
@@ -2015,7 +2017,7 @@ dependencies = [

 [[package]]
 name = "common-procedure-test"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "common-procedure",
@@ -2023,10 +2025,11 @@ dependencies = [

 [[package]]
 name = "common-query"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
+ "bytes",
 "common-base",
 "common-error",
 "common-macro",
@@ -2046,7 +2049,7 @@ dependencies = [

 [[package]]
 name = "common-recordbatch"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arc-swap",
 "common-error",
@@ -2065,7 +2068,7 @@ dependencies = [

 [[package]]
 name = "common-runtime"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "common-error",
@@ -2085,7 +2088,7 @@ dependencies = [

 [[package]]
 name = "common-telemetry"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "atty",
 "backtrace",
@@ -2112,7 +2115,7 @@ dependencies = [

 [[package]]
 name = "common-test-util"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "client",
 "common-query",
@@ -2124,7 +2127,7 @@ dependencies = [

 [[package]]
 name = "common-time"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arrow",
 "chrono",
@@ -2140,7 +2143,7 @@ dependencies = [

 [[package]]
 name = "common-version"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "build-data",
 "schemars",
@@ -2149,7 +2152,7 @@ dependencies = [

 [[package]]
 name = "common-wal"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "common-base",
 "common-error",
@@ -3149,7 +3152,7 @@ dependencies = [

 [[package]]
 name = "datanode"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arrow-flight",
@@ -3198,7 +3201,6 @@ dependencies = [
 "session",
 "snafu 0.8.2",
 "store-api",
- "substrait 0.8.0",
 "table",
 "tokio",
 "toml 0.8.12",
@@ -3207,7 +3209,7 @@ dependencies = [

 [[package]]
 name = "datatypes"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -3698,7 +3700,7 @@ dependencies = [

 [[package]]
 name = "file-engine"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
@@ -3800,7 +3802,7 @@ dependencies = [

 [[package]]
 name = "flow"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
@@ -3841,7 +3843,7 @@ dependencies = [
 "snafu 0.8.2",
 "store-api",
 "strum 0.25.0",
- "substrait 0.8.0",
+ "substrait 0.8.1",
 "table",
 "tokio",
 "tonic 0.11.0",
@@ -3879,7 +3881,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"

 [[package]]
 name = "frontend"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arc-swap",
@@ -4195,7 +4197,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=902f75fdd170c572e90b1f640161d90995f20218#902f75fdd170c572e90b1f640161d90995f20218"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=ae26136accd82fbdf8be540cd502f2e94951077e#ae26136accd82fbdf8be540cd502f2e94951077e"
 dependencies = [
 "prost 0.12.4",
 "serde",
@@ -4685,7 +4687,7 @@ dependencies = [

 [[package]]
 name = "index"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "asynchronous-codec",
@@ -5252,7 +5254,7 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"

 [[package]]
 name = "log-store"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-stream",
 "async-trait",
@@ -5549,7 +5551,7 @@ dependencies = [

 [[package]]
 name = "meta-client"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
@@ -5575,7 +5577,7 @@ dependencies = [

 [[package]]
 name = "meta-srv"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
@@ -5651,7 +5653,7 @@ dependencies = [

 [[package]]
 name = "metric-engine"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "aquamarine",
@@ -5733,7 +5735,7 @@ dependencies = [

 [[package]]
 name = "mito2"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "aquamarine",
@@ -6361,7 +6363,7 @@ dependencies = [

 [[package]]
 name = "object-store"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "anyhow",
 "async-trait",
@@ -6602,7 +6604,7 @@ dependencies = [

 [[package]]
 name = "operator"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
@@ -6648,7 +6650,7 @@ dependencies = [
 "sql",
 "sqlparser 0.44.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e4e496b8d62416ad50ce70a1b460c7313610cf5d)",
 "store-api",
- "substrait 0.8.0",
+ "substrait 0.8.1",
 "table",
 "tokio",
 "tonic 0.11.0",
@@ -6892,7 +6894,7 @@ dependencies = [

 [[package]]
 name = "partition"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "async-trait",
@@ -7238,7 +7240,7 @@ dependencies = [

 [[package]]
 name = "plugins"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "auth",
 "common-base",
@@ -7516,26 +7518,20 @@ dependencies = [

 [[package]]
 name = "promql"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "ahash 0.8.11",
- "async-recursion",
 "async-trait",
 "bytemuck",
- "catalog",
- "common-catalog",
 "common-error",
 "common-macro",
- "common-query",
 "common-recordbatch",
 "common-telemetry",
 "datafusion 37.0.0",
 "datafusion-expr 37.0.0",
- "datafusion-functions 37.0.0",
 "datatypes",
 "futures",
 "greptime-proto",
- "itertools 0.10.5",
 "lazy_static",
 "prometheus",
 "promql-parser",
@@ -7543,7 +7539,6 @@ dependencies = [
 "query",
 "session",
 "snafu 0.8.2",
- "table",
 "tokio",
 ]

@@ -7729,7 +7724,7 @@ dependencies = [

 [[package]]
 name = "puffin"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "bitflags 2.5.0",
@@ -7840,7 +7835,7 @@ dependencies = [

 [[package]]
 name = "query"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "ahash 0.8.11",
 "api",
@@ -7851,6 +7846,7 @@ dependencies = [
 "async-recursion",
 "async-stream",
 "async-trait",
+ "bytes",
 "catalog",
 "chrono",
 "common-base",
@@ -7863,11 +7859,13 @@ dependencies = [
 "common-plugins",
 "common-query",
 "common-recordbatch",
+ "common-runtime",
 "common-telemetry",
 "common-time",
 "datafusion 37.0.0",
 "datafusion-common 37.0.0",
 "datafusion-expr 37.0.0",
+ "datafusion-functions 37.0.0",
 "datafusion-optimizer 37.0.0",
 "datafusion-physical-expr 37.0.0",
 "datafusion-sql 37.0.0",
@@ -7877,6 +7875,7 @@ dependencies = [
 "futures-util",
 "greptime-proto",
 "humantime",
+ "itertools 0.10.5",
 "lazy_static",
 "meter-core",
 "meter-macros",
@@ -7888,6 +7887,7 @@ dependencies = [
 "prometheus",
 "promql",
 "promql-parser",
+ "prost 0.12.4",
 "rand",
 "regex",
 "session",
@@ -7897,7 +7897,7 @@ dependencies = [
 "stats-cli",
 "store-api",
 "streaming-stats",
- "substrait 0.8.0",
+ "substrait 0.8.1",
 "table",
 "tokio",
 "tokio-stream",
@@ -9204,7 +9204,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

 [[package]]
 name = "script"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arc-swap",
@@ -9474,7 +9474,7 @@ dependencies = [

 [[package]]
 name = "servers"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "aide",
 "api",
@@ -9560,7 +9560,6 @@ dependencies = [
 "strum 0.25.0",
 "table",
 "tempfile",
- "tests-integration",
 "tikv-jemalloc-ctl",
 "tokio",
 "tokio-postgres",
@@ -9578,7 +9577,7 @@ dependencies = [

 [[package]]
 name = "session"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arc-swap",
@@ -9856,7 +9855,7 @@ dependencies = [

 [[package]]
 name = "sql"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "chrono",
@@ -9912,7 +9911,7 @@ dependencies = [

 [[package]]
 name = "sqlness-runner"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "clap 4.5.4",
@@ -9925,7 +9924,6 @@ dependencies = [
 "serde_json",
 "sqlness",
 "tempfile",
- "tests-integration",
 "tinytemplate",
 "tokio",
 ]
@@ -10130,7 +10128,7 @@ dependencies = [

 [[package]]
 name = "store-api"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "aquamarine",
@@ -10298,13 +10296,11 @@ dependencies = [

 [[package]]
 name = "substrait"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "async-trait",
 "bytes",
- "catalog",
 "common-error",
- "common-function",
 "common-macro",
 "common-telemetry",
 "datafusion 37.0.0",
@@ -10314,7 +10310,6 @@ dependencies = [
 "datatypes",
 "promql",
 "prost 0.12.4",
- "session",
 "snafu 0.8.2",
 "substrait 0.17.1",
 "tokio",
@@ -10489,8 +10484,9 @@ dependencies = [

 [[package]]
 name = "table"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
+ "api",
 "async-trait",
 "chrono",
 "common-base",
@@ -10599,7 +10595,7 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"

 [[package]]
 name = "tests-fuzz"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "arbitrary",
 "async-trait",
@@ -10632,7 +10628,7 @@ dependencies = [

 [[package]]
 name = "tests-integration"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
 "api",
 "arrow-flight",
@@ -10691,7 +10687,7 @@ dependencies = [
 "sql",
 "sqlx",
 "store-api",
- "substrait 0.8.0",
+ "substrait 0.8.1",
 "table",
 "tempfile",
 "time",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,7 +64,7 @@ members = [
 resolver = "2"

 [workspace.package]
-version = "0.8.0"
+version = "0.8.1"
 edition = "2021"
 license = "Apache-2.0"

@@ -120,7 +120,7 @@ etcd-client = { git = "https://github.com/MichaelScofield/etcd-client.git", rev
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "902f75fdd170c572e90b1f640161d90995f20218" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "ae26136accd82fbdf8be540cd502f2e94951077e" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
@@ -172,6 +172,7 @@ tokio-stream = { version = "0.1" }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
 tonic = { version = "0.11", features = ["tls", "gzip", "zstd"] }
+tower = { version = "0.4" }
 uuid = { version = "1.7", features = ["serde", "v4", "fast-rng"] }
 zstd = "0.13"

@@ -232,8 +233,6 @@ sql = { path = "src/sql" }
 store-api = { path = "src/store-api" }
 substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }
-# TODO some code depends on this
-tests-integration = { path = "tests-integration" }

 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -12,7 +12,7 @@ api.workspace = true
 arrow.workspace = true
 chrono.workspace = true
 clap.workspace = true
-client.workspace = true
+client = { workspace = true, features = ["testing"] }
 common-base.workspace = true
 common-telemetry.workspace = true
 common-wal.workspace = true
@@ -33,8 +33,6 @@ rand.workspace = true
 rskafka.workspace = true
 serde.workspace = true
 store-api.workspace = true
-# TODO depend `Database` client
-tests-integration.workspace = true
 tokio.workspace = true
 toml.workspace = true
 uuid.workspace = true
--- a/benchmarks/src/wal_bench.rs
+++ b/benchmarks/src/wal_bench.rs
@@ -28,6 +28,7 @@ use rand::distributions::{Alphanumeric, DistString, Uniform};
 use rand::rngs::SmallRng;
 use rand::{Rng, SeedableRng};
 use serde::{Deserialize, Serialize};
+use store_api::logstore::provider::Provider;
 use store_api::logstore::LogStore;
 use store_api::storage::RegionId;

@@ -210,7 +211,7 @@ impl From<Args> for Config {
 pub struct Region {
    id: RegionId,
    schema: Vec<ColumnSchema>,
-    wal_options: WalOptions,
+    provider: Provider,
    next_sequence: AtomicU64,
    next_entry_id: AtomicU64,
    next_timestamp: AtomicI64,
@@ -227,10 +228,14 @@ impl Region {
        num_rows: u32,
        rng_seed: u64,
    ) -> Self {
+        let provider = match wal_options {
+            WalOptions::RaftEngine => Provider::raft_engine_provider(id.as_u64()),
+            WalOptions::Kafka(opts) => Provider::kafka_provider(opts.topic),
+        };
        Self {
            id,
            schema,
-            wal_options,
+            provider,
            next_sequence: AtomicU64::new(1),
            next_entry_id: AtomicU64::new(1),
            next_timestamp: AtomicI64::new(1655276557000),
@@ -258,14 +263,14 @@ impl Region {
                self.id,
                self.next_entry_id.fetch_add(1, Ordering::Relaxed),
                &entry,
-                &self.wal_options,
+                &self.provider,
            )
            .unwrap();
    }

    /// Replays the region.
    pub async fn replay<S: LogStore>(&self, wal: &Arc<Wal<S>>) {
-        let mut wal_stream = wal.scan(self.id, 0, &self.wal_options).unwrap();
+        let mut wal_stream = wal.scan(self.id, 0, &self.provider).unwrap();
        while let Some(res) = wal_stream.next().await {
            let (_, entry) = res.unwrap();
            metrics::METRIC_WAL_READ_BYTES_TOTAL.inc_by(Self::entry_estimated_size(&entry) as u64);
--- a/src/cache/Cargo.toml
+++ b/src/cache/Cargo.toml
@@ -11,3 +11,4 @@ common-macro.workspace = true
 common-meta.workspace = true
 moka.workspace = true
 snafu.workspace = true
+substrait.workspace = true
--- a/src/cache/src/lib.rs
+++ b/src/cache/src/lib.rs
@@ -20,7 +20,8 @@ use std::time::Duration;
 use catalog::kvbackend::new_table_cache;
 use common_meta::cache::{
    new_table_flownode_set_cache, new_table_info_cache, new_table_name_cache,
-    new_table_route_cache, CacheRegistry, CacheRegistryBuilder, LayeredCacheRegistryBuilder,
+    new_table_route_cache, new_view_info_cache, CacheRegistry, CacheRegistryBuilder,
+    LayeredCacheRegistryBuilder,
 };
 use common_meta::kv_backend::KvBackendRef;
 use moka::future::CacheBuilder;
@@ -33,6 +34,7 @@ const DEFAULT_CACHE_TTL: Duration = Duration::from_secs(10 * 60);
 const DEFAULT_CACHE_TTI: Duration = Duration::from_secs(5 * 60);

 pub const TABLE_INFO_CACHE_NAME: &str = "table_info_cache";
+pub const VIEW_INFO_CACHE_NAME: &str = "view_info_cache";
 pub const TABLE_NAME_CACHE_NAME: &str = "table_name_cache";
 pub const TABLE_CACHE_NAME: &str = "table_cache";
 pub const TABLE_FLOWNODE_SET_CACHE_NAME: &str = "table_flownode_set_cache";
@@ -82,11 +84,22 @@ pub fn build_fundamental_cache_registry(kv_backend: KvBackendRef) -> CacheRegist
        cache,
        kv_backend.clone(),
    ));
+    // Builds the view info cache
+    let cache = CacheBuilder::new(DEFAULT_CACHE_MAX_CAPACITY)
+        .time_to_live(DEFAULT_CACHE_TTL)
+        .time_to_idle(DEFAULT_CACHE_TTI)
+        .build();
+    let view_info_cache = Arc::new(new_view_info_cache(
+        VIEW_INFO_CACHE_NAME.to_string(),
+        cache,
+        kv_backend.clone(),
+    ));

    CacheRegistryBuilder::default()
        .add_cache(table_info_cache)
        .add_cache(table_name_cache)
        .add_cache(table_route_cache)
+        .add_cache(view_info_cache)
        .add_cache(table_flownode_set_cache)
        .build()
 }
--- a/src/catalog/Cargo.toml
+++ b/src/catalog/Cargo.toml
@@ -16,6 +16,7 @@ arrow.workspace = true
 arrow-schema.workspace = true
 async-stream.workspace = true
 async-trait = "0.1"
+bytes.workspace = true
 common-catalog.workspace = true
 common-config.workspace = true
 common-error.workspace = true
@@ -48,8 +49,11 @@ table.workspace = true
 tokio.workspace = true

 [dev-dependencies]
+cache.workspace = true
 catalog = { workspace = true, features = ["testing"] }
 chrono.workspace = true
+common-meta = { workspace = true, features = ["testing"] }
+common-query = { workspace = true, features = ["testing"] }
 common-test-util.workspace = true
 log-store.workspace = true
 object-store.workspace = true
--- a/src/catalog/src/error.rs
+++ b/src/catalog/src/error.rs
@@ -19,10 +19,7 @@ use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
-use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
-use table::metadata::TableId;
-use tokio::task::JoinError;

 #[derive(Snafu)]
 #[snafu(visibility(pub))]
@@ -65,19 +62,6 @@ pub enum Error {
        location: Location,
        source: BoxedError,
    },
-    #[snafu(display("Failed to open system catalog table"))]
-    OpenSystemCatalog {
-        #[snafu(implicit)]
-        location: Location,
-        source: table::error::Error,
-    },
-
-    #[snafu(display("Failed to create system catalog table"))]
-    CreateSystemCatalog {
-        #[snafu(implicit)]
-        location: Location,
-        source: table::error::Error,
-    },

    #[snafu(display("Failed to create table, table info: {}", table_info))]
    CreateTable {
@@ -94,52 +78,6 @@ pub enum Error {
        location: Location,
    },

-    #[snafu(display(
-        "System catalog table type mismatch, expected: binary, found: {:?}",
-        data_type,
-    ))]
-    SystemCatalogTypeMismatch {
-        data_type: ConcreteDataType,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Invalid system catalog entry type: {:?}", entry_type))]
-    InvalidEntryType {
-        entry_type: Option<u8>,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Invalid system catalog key: {:?}", key))]
-    InvalidKey {
-        key: Option<String>,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Catalog value is not present"))]
-    EmptyValue {
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to deserialize value"))]
-    ValueDeserialize {
-        #[snafu(source)]
-        error: serde_json::error::Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Table engine not found: {}", engine_name))]
-    TableEngineNotFound {
-        engine_name: String,
-        #[snafu(implicit)]
-        location: Location,
-        source: table::error::Error,
-    },
-
    #[snafu(display("Cannot find catalog by name: {}", catalog_name))]
    CatalogNotFound {
        catalog_name: String,
@@ -169,44 +107,9 @@ pub enum Error {
        location: Location,
    },

-    #[snafu(display("Schema {} already exists", schema))]
-    SchemaExists {
-        schema: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Operation {} not implemented yet", operation))]
-    Unimplemented {
-        operation: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Operation {} not supported", op))]
-    NotSupported {
-        op: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to open table {table_id}"))]
-    OpenTable {
-        table_id: TableId,
-        #[snafu(implicit)]
-        location: Location,
-        source: table::error::Error,
-    },
-
-    #[snafu(display("Failed to open table in parallel"))]
-    ParallelOpenTable {
-        #[snafu(source)]
-        error: JoinError,
-    },
-
-    #[snafu(display("Table not found while opening table, table info: {}", table_info))]
-    TableNotFound {
-        table_info: String,
+    #[snafu(display("View info not found: {}", name))]
+    ViewInfoNotFound {
+        name: String,
        #[snafu(implicit)]
        location: Location,
    },
@@ -217,13 +120,6 @@ pub enum Error {
    #[snafu(display("Failed to find region routes"))]
    FindRegionRoutes { source: partition::error::Error },

-    #[snafu(display("Failed to read system catalog table records"))]
-    ReadSystemCatalog {
-        #[snafu(implicit)]
-        location: Location,
-        source: common_recordbatch::error::Error,
-    },
-
    #[snafu(display("Failed to create recordbatch"))]
    CreateRecordBatch {
        #[snafu(implicit)]
@@ -231,20 +127,6 @@ pub enum Error {
        source: common_recordbatch::error::Error,
    },

-    #[snafu(display("Failed to insert table creation record to system catalog"))]
-    InsertCatalogRecord {
-        #[snafu(implicit)]
-        location: Location,
-        source: table::error::Error,
-    },
-
-    #[snafu(display("Failed to scan system catalog table"))]
-    SystemCatalogTableScan {
-        #[snafu(implicit)]
-        location: Location,
-        source: table::error::Error,
-    },
-
    #[snafu(display("Internal error"))]
    Internal {
        #[snafu(implicit)]
@@ -258,20 +140,14 @@ pub enum Error {
        location: Location,
    },

-    #[snafu(display("Failed to execute system catalog table scan"))]
-    SystemCatalogTableScanExec {
+    #[snafu(display("Failed to decode logical plan for view: {}", name))]
+    DecodePlan {
+        name: String,
        #[snafu(implicit)]
        location: Location,
        source: common_query::error::Error,
    },

-    #[snafu(display("Cannot parse catalog value"))]
-    InvalidCatalogValue {
-        #[snafu(implicit)]
-        location: Location,
-        source: common_catalog::error::Error,
-    },
-
    #[snafu(display("Failed to perform metasrv operation"))]
    Metasrv {
        #[snafu(implicit)]
@@ -297,20 +173,6 @@ pub enum Error {
        location: Location,
    },

-    #[snafu(display("Table schema mismatch"))]
-    TableSchemaMismatch {
-        #[snafu(implicit)]
-        location: Location,
-        source: table::error::Error,
-    },
-
-    #[snafu(display("A generic error has occurred, msg: {}", msg))]
-    Generic {
-        msg: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
    #[snafu(display("Table metadata manager error"))]
    TableMetadataManager {
        source: common_meta::error::Error,
@@ -324,6 +186,26 @@ pub enum Error {
        #[snafu(implicit)]
        location: Location,
    },
+
+    #[snafu(display("Failed to get view info from cache"))]
+    GetViewCache {
+        source: common_meta::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Cache not found: {name}"))]
+    CacheNotFound {
+        name: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Failed to cast the catalog manager"))]
+    CastManager {
+        #[snafu(implicit)]
+        location: Location,
+    },
 }

 pub type Result<T> = std::result::Result<T, Error>;
@@ -331,61 +213,43 @@ pub type Result<T> = std::result::Result<T, Error>;
 impl ErrorExt for Error {
    fn status_code(&self) -> StatusCode {
        match self {
-            Error::InvalidKey { .. }
-            | Error::SchemaNotFound { .. }
+            Error::SchemaNotFound { .. }
            | Error::CatalogNotFound { .. }
            | Error::FindPartitions { .. }
            | Error::FindRegionRoutes { .. }
-            | Error::InvalidEntryType { .. }
-            | Error::ParallelOpenTable { .. } => StatusCode::Unexpected,
+            | Error::CacheNotFound { .. }
+            | Error::CastManager { .. } => StatusCode::Unexpected,

-            Error::TableNotFound { .. } => StatusCode::TableNotFound,
+            Error::ViewInfoNotFound { .. } => StatusCode::TableNotFound,

-            Error::SystemCatalog { .. }
-            | Error::EmptyValue { .. }
-            | Error::ValueDeserialize { .. } => StatusCode::StorageUnavailable,
+            Error::SystemCatalog { .. } => StatusCode::StorageUnavailable,

-            Error::Generic { .. }
-            | Error::SystemCatalogTypeMismatch { .. }
-            | Error::UpgradeWeakCatalogManagerRef { .. } => StatusCode::Internal,
-
-            Error::ReadSystemCatalog { source, .. } | Error::CreateRecordBatch { source, .. } => {
-                source.status_code()
-            }
-            Error::InvalidCatalogValue { source, .. } => source.status_code(),
+            Error::UpgradeWeakCatalogManagerRef { .. } => StatusCode::Internal,

+            Error::CreateRecordBatch { source, .. } => source.status_code(),
            Error::TableExists { .. } => StatusCode::TableAlreadyExists,
            Error::TableNotExist { .. } => StatusCode::TableNotFound,
-            Error::SchemaExists { .. } | Error::TableEngineNotFound { .. } => {
-                StatusCode::InvalidArguments
-            }
-
            Error::ListCatalogs { source, .. }
            | Error::ListNodes { source, .. }
            | Error::ListSchemas { source, .. }
            | Error::ListTables { source, .. } => source.status_code(),

-            Error::OpenSystemCatalog { source, .. }
-            | Error::CreateSystemCatalog { source, .. }
-            | Error::InsertCatalogRecord { source, .. }
-            | Error::OpenTable { source, .. }
-            | Error::CreateTable { source, .. }
-            | Error::TableSchemaMismatch { source, .. } => source.status_code(),
+            Error::CreateTable { source, .. } => source.status_code(),

            Error::Metasrv { source, .. } => source.status_code(),
-            Error::SystemCatalogTableScan { source, .. } => source.status_code(),
-            Error::SystemCatalogTableScanExec { source, .. } => source.status_code(),
+            Error::DecodePlan { source, .. } => source.status_code(),
            Error::InvalidTableInfoInCatalog { source, .. } => source.status_code(),

            Error::CompileScriptInternal { source, .. } | Error::Internal { source, .. } => {
                source.status_code()
            }

-            Error::Unimplemented { .. } | Error::NotSupported { .. } => StatusCode::Unsupported,
            Error::QueryAccessDenied { .. } => StatusCode::AccessDenied,
            Error::Datafusion { .. } => StatusCode::EngineExecuteQuery,
            Error::TableMetadataManager { source, .. } => source.status_code(),
-            Error::GetTableCache { .. } => StatusCode::Internal,
+            Error::GetViewCache { source, .. } | Error::GetTableCache { source, .. } => {
+                source.status_code()
+            }
        }
    }

@@ -417,11 +281,6 @@ mod tests {
            .status_code()
        );

-        assert_eq!(
-            StatusCode::Unexpected,
-            InvalidKeySnafu { key: None }.build().status_code()
-        );
-
        assert_eq!(
            StatusCode::StorageUnavailable,
            Error::SystemCatalog {
@@ -430,19 +289,6 @@ mod tests {
            }
            .status_code()
        );
-
-        assert_eq!(
-            StatusCode::Internal,
-            Error::SystemCatalogTypeMismatch {
-                data_type: ConcreteDataType::binary_datatype(),
-                location: Location::generate(),
-            }
-            .status_code()
-        );
-        assert_eq!(
-            StatusCode::StorageUnavailable,
-            EmptyValueSnafu {}.build().status_code()
-        );
    }

    #[test]
--- a/src/catalog/src/kvbackend/manager.rs
+++ b/src/catalog/src/kvbackend/manager.rs
@@ -22,14 +22,13 @@ use common_catalog::consts::{
 };
 use common_config::Mode;
 use common_error::ext::BoxedError;
-use common_meta::cache::TableRouteCacheRef;
+use common_meta::cache::{LayeredCacheRegistryRef, ViewInfoCacheRef};
 use common_meta::key::catalog_name::CatalogNameKey;
 use common_meta::key::schema_name::SchemaNameKey;
 use common_meta::key::table_info::TableInfoValue;
 use common_meta::key::table_name::TableNameKey;
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
-use common_meta::table_name::TableName;
 use futures_util::stream::BoxStream;
 use futures_util::{StreamExt, TryStreamExt};
 use meta_client::client::MetaClient;
@@ -38,11 +37,12 @@ use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
 use snafu::prelude::*;
 use table::dist_table::DistTable;
 use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+use table::table_name::TableName;
 use table::TableRef;

 use crate::error::{
-    GetTableCacheSnafu, InvalidTableInfoInCatalogSnafu, ListCatalogsSnafu, ListSchemasSnafu,
-    ListTablesSnafu, Result, TableMetadataManagerSnafu,
+    CacheNotFoundSnafu, GetTableCacheSnafu, InvalidTableInfoInCatalogSnafu, ListCatalogsSnafu,
+    ListSchemasSnafu, ListTablesSnafu, Result, TableMetadataManagerSnafu,
 };
 use crate::information_schema::InformationSchemaProvider;
 use crate::kvbackend::TableCacheRef;
@@ -61,25 +61,26 @@ pub struct KvBackendCatalogManager {
    table_metadata_manager: TableMetadataManagerRef,
    /// A sub-CatalogManager that handles system tables
    system_catalog: SystemCatalog,
-    table_cache: TableCacheRef,
+    cache_registry: LayeredCacheRegistryRef,
 }

 const CATALOG_CACHE_MAX_CAPACITY: u64 = 128;

 impl KvBackendCatalogManager {
-    pub async fn new(
+    pub fn new(
        mode: Mode,
        meta_client: Option<Arc<MetaClient>>,
        backend: KvBackendRef,
-        table_cache: TableCacheRef,
-        table_route_cache: TableRouteCacheRef,
+        cache_registry: LayeredCacheRegistryRef,
    ) -> Arc<Self> {
        Arc::new_cyclic(|me| Self {
            mode,
            meta_client,
            partition_manager: Arc::new(PartitionRuleManager::new(
                backend.clone(),
-                table_route_cache,
+                cache_registry
+                    .get()
+                    .expect("Failed to get table_route_cache"),
            )),
            table_metadata_manager: Arc::new(TableMetadataManager::new(backend)),
            system_catalog: SystemCatalog {
@@ -90,7 +91,7 @@ impl KvBackendCatalogManager {
                    me.clone(),
                )),
            },
-            table_cache,
+            cache_registry,
        })
    }

@@ -99,6 +100,12 @@ impl KvBackendCatalogManager {
        &self.mode
    }

+    pub fn view_info_cache(&self) -> Result<ViewInfoCacheRef> {
+        self.cache_registry.get().context(CacheNotFoundSnafu {
+            name: "view_info_cache",
+        })
+    }
+
    /// Returns the `[MetaClient]`.
    pub fn meta_client(&self) -> Option<Arc<MetaClient>> {
        self.meta_client.clone()
@@ -215,7 +222,11 @@ impl CatalogManager for KvBackendCatalogManager {
            return Ok(Some(table));
        }

-        self.table_cache
+        let table_cache: TableCacheRef = self.cache_registry.get().context(CacheNotFoundSnafu {
+            name: "table_cache",
+        })?;
+
+        table_cache
            .get_by_ref(&TableName {
                catalog_name: catalog_name.to_string(),
                schema_name: schema_name.to_string(),
--- a/src/catalog/src/kvbackend/table_cache.rs
+++ b/src/catalog/src/kvbackend/table_cache.rs
@@ -17,11 +17,11 @@ use std::sync::Arc;
 use common_meta::cache::{CacheContainer, Initializer, TableInfoCacheRef, TableNameCacheRef};
 use common_meta::error::{Result as MetaResult, ValueNotExistSnafu};
 use common_meta::instruction::CacheIdent;
-use common_meta::table_name::TableName;
 use futures::future::BoxFuture;
 use moka::future::Cache;
 use snafu::OptionExt;
 use table::dist_table::DistTable;
+use table::table_name::TableName;
 use table::TableRef;

 pub type TableCacheRef = Arc<TableCache>;
--- a/src/catalog/src/table_source.rs
+++ b/src/catalog/src/table_source.rs
@@ -15,15 +15,25 @@
 use std::collections::HashMap;
 use std::sync::Arc;

+use bytes::Bytes;
 use common_catalog::format_full_table_name;
+use common_query::logical_plan::SubstraitPlanDecoderRef;
 use datafusion::common::{ResolvedTableReference, TableReference};
-use datafusion::datasource::provider_as_source;
+use datafusion::datasource::view::ViewTable;
+use datafusion::datasource::{provider_as_source, TableProvider};
 use datafusion::logical_expr::TableSource;
 use session::context::QueryContext;
-use snafu::{ensure, OptionExt};
+use snafu::{ensure, OptionExt, ResultExt};
+use table::metadata::TableType;
 use table::table::adapter::DfTableProviderAdapter;
+mod dummy_catalog;
+use dummy_catalog::DummyCatalogList;

-use crate::error::{QueryAccessDeniedSnafu, Result, TableNotExistSnafu};
+use crate::error::{
+    CastManagerSnafu, DatafusionSnafu, DecodePlanSnafu, GetViewCacheSnafu, QueryAccessDeniedSnafu,
+    Result, TableNotExistSnafu, ViewInfoNotFoundSnafu,
+};
+use crate::kvbackend::KvBackendCatalogManager;
 use crate::CatalogManagerRef;

 pub struct DfTableSourceProvider {
@@ -32,6 +42,7 @@ pub struct DfTableSourceProvider {
    disallow_cross_catalog_query: bool,
    default_catalog: String,
    default_schema: String,
+    plan_decoder: SubstraitPlanDecoderRef,
 }

 impl DfTableSourceProvider {
@@ -39,6 +50,7 @@ impl DfTableSourceProvider {
        catalog_manager: CatalogManagerRef,
        disallow_cross_catalog_query: bool,
        query_ctx: &QueryContext,
+        plan_decoder: SubstraitPlanDecoderRef,
    ) -> Self {
        Self {
            catalog_manager,
@@ -46,6 +58,7 @@ impl DfTableSourceProvider {
            resolved_tables: HashMap::new(),
            default_catalog: query_ctx.current_catalog().to_owned(),
            default_schema: query_ctx.current_schema().to_owned(),
+            plan_decoder,
        }
    }

@@ -94,8 +107,39 @@ impl DfTableSourceProvider {
                table: format_full_table_name(catalog_name, schema_name, table_name),
            })?;

-        let provider = DfTableProviderAdapter::new(table);
-        let source = provider_as_source(Arc::new(provider));
+        let provider: Arc<dyn TableProvider> = if table.table_info().table_type == TableType::View {
+            let catalog_manager = self
+                .catalog_manager
+                .as_any()
+                .downcast_ref::<KvBackendCatalogManager>()
+                .context(CastManagerSnafu)?;
+
+            let view_info = catalog_manager
+                .view_info_cache()?
+                .get(table.table_info().ident.table_id)
+                .await
+                .context(GetViewCacheSnafu)?
+                .context(ViewInfoNotFoundSnafu {
+                    name: &table.table_info().name,
+                })?;
+
+            // Build the catalog list provider for deserialization.
+            let catalog_list = Arc::new(DummyCatalogList::new(self.catalog_manager.clone()));
+            let logical_plan = self
+                .plan_decoder
+                .decode(Bytes::from(view_info.view_info.clone()), catalog_list, true)
+                .await
+                .context(DecodePlanSnafu {
+                    name: &table.table_info().name,
+                })?;
+
+            Arc::new(ViewTable::try_new(logical_plan, None).context(DatafusionSnafu)?)
+        } else {
+            Arc::new(DfTableProviderAdapter::new(table))
+        };
+
+        let source = provider_as_source(provider);
+
        let _ = self.resolved_tables.insert(resolved_name, source.clone());
        Ok(source)
    }
@@ -103,6 +147,7 @@ impl DfTableSourceProvider {

 #[cfg(test)]
 mod tests {
+    use common_query::test_util::DummyDecoder;
    use session::context::QueryContext;

    use super::*;
@@ -112,8 +157,12 @@ mod tests {
    fn test_validate_table_ref() {
        let query_ctx = &QueryContext::with("greptime", "public");

-        let table_provider =
-            DfTableSourceProvider::new(MemoryCatalogManager::with_default_setup(), true, query_ctx);
+        let table_provider = DfTableSourceProvider::new(
+            MemoryCatalogManager::with_default_setup(),
+            true,
+            query_ctx,
+            DummyDecoder::arc(),
+        );

        let table_ref = TableReference::bare("table_name");
        let result = table_provider.resolve_table_ref(table_ref);
@@ -148,4 +197,99 @@ mod tests {
        let table_ref = TableReference::full("greptime", "greptime_private", "columns");
        assert!(table_provider.resolve_table_ref(table_ref).is_ok());
    }
+
+    use std::collections::HashSet;
+
+    use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+    use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
+    use common_config::Mode;
+    use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder};
+    use common_meta::key::TableMetadataManager;
+    use common_meta::kv_backend::memory::MemoryKvBackend;
+    use common_query::error::Result as QueryResult;
+    use common_query::logical_plan::SubstraitPlanDecoder;
+    use datafusion::catalog::CatalogProviderList;
+    use datafusion::logical_expr::builder::LogicalTableSource;
+    use datafusion::logical_expr::{col, lit, LogicalPlan, LogicalPlanBuilder};
+
+    struct MockDecoder;
+    impl MockDecoder {
+        pub fn arc() -> Arc<Self> {
+            Arc::new(MockDecoder)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl SubstraitPlanDecoder for MockDecoder {
+        async fn decode(
+            &self,
+            _message: bytes::Bytes,
+            _catalog_list: Arc<dyn CatalogProviderList>,
+            _optimize: bool,
+        ) -> QueryResult<LogicalPlan> {
+            Ok(mock_plan())
+        }
+    }
+
+    fn mock_plan() -> LogicalPlan {
+        let schema = Schema::new(vec![
+            Field::new("id", DataType::Int32, true),
+            Field::new("name", DataType::Utf8, true),
+        ]);
+        let table_source = LogicalTableSource::new(SchemaRef::new(schema));
+
+        let projection = None;
+
+        let builder =
+            LogicalPlanBuilder::scan("person", Arc::new(table_source), projection).unwrap();
+
+        builder
+            .filter(col("id").gt(lit(500)))
+            .unwrap()
+            .build()
+            .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_resolve_view() {
+        let query_ctx = &QueryContext::with("greptime", "public");
+        let backend = Arc::new(MemoryKvBackend::default());
+        let layered_cache_builder = LayeredCacheRegistryBuilder::default()
+            .add_cache_registry(CacheRegistryBuilder::default().build());
+        let fundamental_cache_registry = build_fundamental_cache_registry(backend.clone());
+        let layered_cache_registry = Arc::new(
+            with_default_composite_cache_registry(
+                layered_cache_builder.add_cache_registry(fundamental_cache_registry),
+            )
+            .unwrap()
+            .build(),
+        );
+
+        let catalog_manager = KvBackendCatalogManager::new(
+            Mode::Standalone,
+            None,
+            backend.clone(),
+            layered_cache_registry,
+        );
+        let table_metadata_manager = TableMetadataManager::new(backend);
+        let mut view_info = common_meta::key::test_utils::new_test_table_info(1024, vec![]);
+        view_info.table_type = TableType::View;
+        let logical_plan = vec![1, 2, 3];
+        // Create view metadata
+        table_metadata_manager
+            .create_view_metadata(view_info.clone().into(), logical_plan, HashSet::new())
+            .await
+            .unwrap();
+
+        let mut table_provider =
+            DfTableSourceProvider::new(catalog_manager, true, query_ctx, MockDecoder::arc());
+
+        // View not found
+        let table_ref = TableReference::bare("not_exists_view");
+        assert!(table_provider.resolve_table(table_ref).await.is_err());
+
+        let table_ref = TableReference::bare(view_info.name);
+        let source = table_provider.resolve_table(table_ref).await.unwrap();
+        assert_eq!(*source.get_logical_plan().unwrap(), mock_plan());
+    }
 }
--- a/src/catalog/src/table_source/dummy_catalog.rs
+++ b/src/catalog/src/table_source/dummy_catalog.rs
@@ -0,0 +1,129 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Dummy catalog for region server.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_catalog::format_full_table_name;
+use datafusion::catalog::schema::SchemaProvider;
+use datafusion::catalog::{CatalogProvider, CatalogProviderList};
+use datafusion::datasource::TableProvider;
+use snafu::OptionExt;
+use table::table::adapter::DfTableProviderAdapter;
+
+use crate::error::TableNotExistSnafu;
+use crate::CatalogManagerRef;
+
+/// Delegate the resolving requests to the `[CatalogManager]` unconditionally.
+#[derive(Clone)]
+pub struct DummyCatalogList {
+    catalog_manager: CatalogManagerRef,
+}
+
+impl DummyCatalogList {
+    /// Creates a new catalog list with the given catalog manager.
+    pub fn new(catalog_manager: CatalogManagerRef) -> Self {
+        Self { catalog_manager }
+    }
+}
+
+impl CatalogProviderList for DummyCatalogList {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn register_catalog(
+        &self,
+        _name: String,
+        _catalog: Arc<dyn CatalogProvider>,
+    ) -> Option<Arc<dyn CatalogProvider>> {
+        None
+    }
+
+    fn catalog_names(&self) -> Vec<String> {
+        vec![]
+    }
+
+    fn catalog(&self, catalog_name: &str) -> Option<Arc<dyn CatalogProvider>> {
+        Some(Arc::new(DummyCatalogProvider {
+            catalog_name: catalog_name.to_string(),
+            catalog_manager: self.catalog_manager.clone(),
+        }))
+    }
+}
+
+/// A dummy catalog provider for [DummyCatalogList].
+#[derive(Clone)]
+struct DummyCatalogProvider {
+    catalog_name: String,
+    catalog_manager: CatalogManagerRef,
+}
+
+impl CatalogProvider for DummyCatalogProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema_names(&self) -> Vec<String> {
+        vec![]
+    }
+
+    fn schema(&self, schema_name: &str) -> Option<Arc<dyn SchemaProvider>> {
+        Some(Arc::new(DummySchemaProvider {
+            catalog_name: self.catalog_name.clone(),
+            schema_name: schema_name.to_string(),
+            catalog_manager: self.catalog_manager.clone(),
+        }))
+    }
+}
+
+/// A dummy schema provider for [DummyCatalogList].
+#[derive(Clone)]
+struct DummySchemaProvider {
+    catalog_name: String,
+    schema_name: String,
+    catalog_manager: CatalogManagerRef,
+}
+
+#[async_trait]
+impl SchemaProvider for DummySchemaProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn table_names(&self) -> Vec<String> {
+        vec![]
+    }
+
+    async fn table(&self, name: &str) -> datafusion::error::Result<Option<Arc<dyn TableProvider>>> {
+        let table = self
+            .catalog_manager
+            .table(&self.catalog_name, &self.schema_name, name)
+            .await?
+            .with_context(|| TableNotExistSnafu {
+                table: format_full_table_name(&self.catalog_name, &self.schema_name, name),
+            })?;
+
+        let table_provider: Arc<dyn TableProvider> = Arc::new(DfTableProviderAdapter::new(table));
+
+        Ok(Some(table_provider))
+    }
+
+    fn table_exist(&self, _name: &str) -> bool {
+        true
+    }
+}
--- a/src/client/src/client.rs
+++ b/src/client/src/client.rs
@@ -173,14 +173,14 @@ impl Client {
        Ok(FlightClient { addr, client })
    }

-    pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
-        let (_, channel) = self.find_channel()?;
+    pub(crate) fn raw_region_client(&self) -> Result<(String, PbRegionClient<Channel>)> {
+        let (addr, channel) = self.find_channel()?;
        let client = PbRegionClient::new(channel)
            .max_decoding_message_size(self.max_grpc_recv_message_size())
            .max_encoding_message_size(self.max_grpc_send_message_size())
            .accept_compressed(CompressionEncoding::Zstd)
            .send_compressed(CompressionEncoding::Zstd);
-        Ok(client)
+        Ok((addr, client))
    }

    pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {
--- a/tests-integration/src/database.rs
+++ b/tests-integration/src/database.rs
@@ -23,8 +23,6 @@ use api::v1::{
 };
 use arrow_flight::Ticket;
 use async_stream::stream;
-use client::error::{ConvertFlightDataSnafu, Error, IllegalFlightMessagesSnafu, ServerSnafu};
-use client::{from_grpc_response, Client, Result};
 use common_error::ext::{BoxedError, ErrorExt};
 use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_query::Output;
@@ -37,7 +35,8 @@ use prost::Message;
 use snafu::{ensure, ResultExt};
 use tonic::transport::Channel;

-pub const DEFAULT_LOOKBACK_STRING: &str = "5m";
+use crate::error::{ConvertFlightDataSnafu, Error, IllegalFlightMessagesSnafu, ServerSnafu};
+use crate::{from_grpc_response, Client, Result};

 #[derive(Clone, Debug, Default)]
 pub struct Database {
@@ -105,10 +104,18 @@ impl Database {
        self.catalog = catalog.into();
    }

+    pub fn catalog(&self) -> &String {
+        &self.catalog
+    }
+
    pub fn set_schema(&mut self, schema: impl Into<String>) {
        self.schema = schema.into();
    }

+    pub fn schema(&self) -> &String {
+        &self.schema
+    }
+
    pub fn set_timezone(&mut self, timezone: impl Into<String>) {
        self.timezone = timezone.into();
    }
@@ -156,6 +163,13 @@ impl Database {
        .await
    }

+    pub async fn logical_plan(&self, logical_plan: Vec<u8>) -> Result<Output> {
+        self.do_get(Request::Query(QueryRequest {
+            query: Some(Query::LogicalPlan(logical_plan)),
+        }))
+        .await
+    }
+
    pub async fn create(&self, expr: CreateTableExpr) -> Result<Output> {
        self.do_get(Request::Ddl(DdlRequest {
            expr: Some(DdlExpr::CreateTable(expr)),
@@ -269,17 +283,12 @@ struct FlightContext {

 #[cfg(test)]
 mod tests {
+    use std::assert_matches::assert_matches;
+
    use api::v1::auth_header::AuthScheme;
    use api::v1::{AuthHeader, Basic};
-    use clap::Parser;
-    use client::Client;
-    use cmd::error::Result as CmdResult;
-    use cmd::options::GlobalOptions;
-    use cmd::{cli, standalone, App};
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-    use common_telemetry::logging::LoggingOptions;

-    use super::{Database, FlightContext};
+    use super::*;

    #[test]
    fn test_flight_ctx() {
@@ -295,76 +304,11 @@ mod tests {
            auth_scheme: Some(basic),
        });

-        assert!(matches!(
+        assert_matches!(
            ctx.auth_header,
            Some(AuthHeader {
                auth_scheme: Some(AuthScheme::Basic(_)),
            })
-        ))
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_export_create_table_with_quoted_names() -> CmdResult<()> {
-        let output_dir = tempfile::tempdir().unwrap();
-
-        let standalone = standalone::Command::parse_from([
-            "standalone",
-            "start",
-            "--data-home",
-            &*output_dir.path().to_string_lossy(),
-        ]);
-
-        let standalone_opts = standalone.load_options(&GlobalOptions::default()).unwrap();
-        let mut instance = standalone.build(standalone_opts).await?;
-        instance.start().await?;
-
-        let client = Client::with_urls(["127.0.0.1:4001"]);
-        let database = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
-        database
-            .sql(r#"CREATE DATABASE "cli.export.create_table";"#)
-            .await
-            .unwrap();
-        database
-            .sql(
-                r#"CREATE TABLE "cli.export.create_table"."a.b.c"(
-                        ts TIMESTAMP,
-                        TIME INDEX (ts)
-                    ) engine=mito;
-                "#,
-            )
-            .await
-            .unwrap();
-
-        let output_dir = tempfile::tempdir().unwrap();
-        let cli = cli::Command::parse_from([
-            "cli",
-            "export",
-            "--addr",
-            "127.0.0.1:4000",
-            "--output-dir",
-            &*output_dir.path().to_string_lossy(),
-            "--target",
-            "create-table",
-        ]);
-        let mut cli_app = cli.build(LoggingOptions::default()).await?;
-        cli_app.start().await?;
-
-        instance.stop().await?;
-
-        let output_file = output_dir
-            .path()
-            .join("greptime-cli.export.create_table.sql");
-        let res = std::fs::read_to_string(output_file).unwrap();
-        let expect = r#"CREATE TABLE IF NOT EXISTS "a.b.c" (
-  "ts" TIMESTAMP(3) NOT NULL,
-  TIME INDEX ("ts")
-)
-
-ENGINE=mito
-;
-"#;
-        assert_eq!(res.trim(), expect.trim());
-
-        Ok(())
+        )
    }
 }
--- a/src/client/src/error.rs
+++ b/src/client/src/error.rs
@@ -89,8 +89,9 @@ pub enum Error {
        source: common_grpc::error::Error,
    },

-    #[snafu(display("Failed to request RegionServer, code: {}", code))]
+    #[snafu(display("Failed to request RegionServer {}, code: {}", addr, code))]
    RegionServer {
+        addr: String,
        code: Code,
        source: BoxedError,
        #[snafu(implicit)]
--- a/src/client/src/lib.rs
+++ b/src/client/src/lib.rs
@@ -12,8 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+#![feature(assert_matches)]
+
 mod client;
 pub mod client_manager;
+#[cfg(feature = "testing")]
+mod database;
 pub mod error;
 pub mod load_balance;
 mod metrics;
@@ -29,6 +33,8 @@ pub use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
 use snafu::OptionExt;

 pub use self::client::Client;
+#[cfg(feature = "testing")]
+pub use self::database::Database;
 pub use self::error::{Error, Result};
 use crate::error::{IllegalDatabaseResponseSnafu, ServerSnafu};

--- a/src/client/src/region.rs
+++ b/src/client/src/region.rs
@@ -177,7 +177,7 @@ impl RegionRequester {
            .with_label_values(&[request_type.as_str()])
            .start_timer();

-        let mut client = self.client.raw_region_client()?;
+        let (addr, mut client) = self.client.raw_region_client()?;

        let response = client
            .handle(request)
@@ -187,6 +187,7 @@ impl RegionRequester {
                let err: error::Error = e.into();
                // Uses `Error::RegionServer` instead of `Error::Server`
                error::Error::RegionServer {
+                    addr,
                    code,
                    source: BoxedError::new(err),
                    location: location!(),
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -80,6 +80,7 @@ tracing-appender = "0.2"
 tikv-jemallocator = "0.5"

 [dev-dependencies]
+client = { workspace = true, features = ["testing"] }
 common-test-util.workspace = true
 serde.workspace = true
 temp-env = "0.3"
--- a/src/cmd/src/cli.rs
+++ b/src/cmd/src/cli.rs
@@ -22,18 +22,14 @@ mod helper;

 // Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373
 #[allow(unused)]
-// mod repl;
-// TODO(weny): Removes it
-#[allow(deprecated)]
-mod upgrade;
+mod repl;

 use async_trait::async_trait;
 use bench::BenchTableMetadataCommand;
 use clap::Parser;
 use common_telemetry::logging::{LoggingOptions, TracingOptions};
+pub use repl::Repl;
 use tracing_appender::non_blocking::WorkerGuard;
-// pub use repl::Repl;
-use upgrade::UpgradeCommand;

 use self::export::ExportCommand;
 use crate::error::Result;
@@ -116,7 +112,6 @@ impl Command {
 #[derive(Parser)]
 enum SubCommand {
    // Attach(AttachCommand),
-    Upgrade(UpgradeCommand),
    Bench(BenchTableMetadataCommand),
    Export(ExportCommand),
 }
@@ -125,7 +120,6 @@ impl SubCommand {
    async fn build(&self, guard: Vec<WorkerGuard>) -> Result<Instance> {
        match self {
            // SubCommand::Attach(cmd) => cmd.build().await,
-            SubCommand::Upgrade(cmd) => cmd.build(guard).await,
            SubCommand::Bench(cmd) => cmd.build(guard).await,
            SubCommand::Export(cmd) => cmd.build(guard).await,
        }
--- a/src/cmd/src/cli/bench.rs
+++ b/src/cmd/src/cli/bench.rs
@@ -23,13 +23,13 @@ use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::etcd::EtcdStore;
 use common_meta::peer::Peer;
 use common_meta::rpc::router::{Region, RegionRoute};
-use common_meta::table_name::TableName;
 use common_telemetry::info;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use rand::Rng;
 use store_api::storage::RegionNumber;
 use table::metadata::{RawTableInfo, RawTableMeta, TableId, TableIdent, TableType};
+use table::table_name::TableName;
 use tracing_appender::non_blocking::WorkerGuard;

 use self::metadata::TableMetadataBencher;
--- a/src/cmd/src/cli/bench/metadata.rs
+++ b/src/cmd/src/cli/bench/metadata.rs
@@ -16,7 +16,7 @@ use std::time::Instant;

 use common_meta::key::table_route::TableRouteValue;
 use common_meta::key::TableMetadataManagerRef;
-use common_meta::table_name::TableName;
+use table::table_name::TableName;

 use crate::cli::bench::{
    bench_self_recorded, create_region_routes, create_region_wal_options, create_table_info,
--- a/src/cmd/src/cli/export.rs
+++ b/src/cmd/src/cli/export.rs
@@ -434,3 +434,80 @@ fn split_database(database: &str) -> Result<(String, Option<String>)> {
        Ok((catalog.to_string(), Some(schema.to_string())))
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use clap::Parser;
+    use client::{Client, Database};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use common_telemetry::logging::LoggingOptions;
+
+    use crate::error::Result as CmdResult;
+    use crate::options::GlobalOptions;
+    use crate::{cli, standalone, App};
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_export_create_table_with_quoted_names() -> CmdResult<()> {
+        let output_dir = tempfile::tempdir().unwrap();
+
+        let standalone = standalone::Command::parse_from([
+            "standalone",
+            "start",
+            "--data-home",
+            &*output_dir.path().to_string_lossy(),
+        ]);
+
+        let standalone_opts = standalone.load_options(&GlobalOptions::default()).unwrap();
+        let mut instance = standalone.build(standalone_opts).await?;
+        instance.start().await?;
+
+        let client = Client::with_urls(["127.0.0.1:4001"]);
+        let database = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+        database
+            .sql(r#"CREATE DATABASE "cli.export.create_table";"#)
+            .await
+            .unwrap();
+        database
+            .sql(
+                r#"CREATE TABLE "cli.export.create_table"."a.b.c"(
+                        ts TIMESTAMP,
+                        TIME INDEX (ts)
+                    ) engine=mito;
+                "#,
+            )
+            .await
+            .unwrap();
+
+        let output_dir = tempfile::tempdir().unwrap();
+        let cli = cli::Command::parse_from([
+            "cli",
+            "export",
+            "--addr",
+            "127.0.0.1:4000",
+            "--output-dir",
+            &*output_dir.path().to_string_lossy(),
+            "--target",
+            "create-table",
+        ]);
+        let mut cli_app = cli.build(LoggingOptions::default()).await?;
+        cli_app.start().await?;
+
+        instance.stop().await?;
+
+        let output_file = output_dir
+            .path()
+            .join("greptime-cli.export.create_table.sql");
+        let res = std::fs::read_to_string(output_file).unwrap();
+        let expect = r#"CREATE TABLE IF NOT EXISTS "a.b.c" (
+  "ts" TIMESTAMP(3) NOT NULL,
+  TIME INDEX ("ts")
+)
+
+ENGINE=mito
+;
+"#;
+        assert_eq!(res.trim(), expect.trim());
+
+        Ok(())
+    }
+}
--- a/src/cmd/src/cli/repl.rs
+++ b/src/cmd/src/cli/repl.rs
@@ -16,14 +16,18 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;

-use catalog::kvbackend::{
-    CachedMetaKvBackend, CachedMetaKvBackendBuilder, KvBackendCatalogManager,
+use cache::{
+    build_fundamental_cache_registry, with_default_composite_cache_registry, TABLE_CACHE_NAME,
+    TABLE_ROUTE_CACHE_NAME,
 };
-use client::{Client, OutputData, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use catalog::kvbackend::{
+    CachedMetaKvBackend, CachedMetaKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend,
+};
+use client::{Client, Database, OutputData, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_base::Plugins;
 use common_config::Mode;
 use common_error::ext::ErrorExt;
-use common_meta::cache_invalidator::MultiCacheInvalidator;
+use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder};
 use common_query::Output;
 use common_recordbatch::RecordBatches;
 use common_telemetry::debug;
@@ -33,17 +37,18 @@ use query::datafusion::DatafusionQueryEngine;
 use query::logical_optimizer::LogicalOptimizer;
 use query::parser::QueryLanguageParser;
 use query::plan::LogicalPlan;
-use query::query_engine::QueryEngineState;
+use query::query_engine::{DefaultSerializer, QueryEngineState};
 use query::QueryEngine;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 use session::context::QueryContext;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};

 use crate::cli::cmd::ReplCommand;
 use crate::cli::helper::RustylineHelper;
 use crate::cli::AttachCommand;
+use crate::error;
 use crate::error::{
    CollectRecordBatchesSnafu, ParseSqlSnafu, PlanStatementSnafu, PrettyPrintRecordBatchesSnafu,
    ReadlineSnafu, ReplCreationSnafu, RequestDatabaseSnafu, Result, StartMetaClientSnafu,
@@ -180,7 +185,7 @@ impl Repl {
                .context(PlanStatementSnafu)?;

            let plan = DFLogicalSubstraitConvertor {}
-                .encode(&plan)
+                .encode(&plan, DefaultSerializer)
                .context(SubstraitEncodeLogicalPlanSnafu)?;

            self.database.logical_plan(plan.to_vec()).await
@@ -257,19 +262,30 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {

    let cached_meta_backend =
        Arc::new(CachedMetaKvBackendBuilder::new(meta_client.clone()).build());
-    let multi_cache_invalidator = Arc::new(MultiCacheInvalidator::with_invalidators(vec![
-        cached_meta_backend.clone(),
-    ]));
-    let catalog_list = KvBackendCatalogManager::new(
+    let layered_cache_builder = LayeredCacheRegistryBuilder::default().add_cache_registry(
+        CacheRegistryBuilder::default()
+            .add_cache(cached_meta_backend.clone())
+            .build(),
+    );
+    let fundamental_cache_registry =
+        build_fundamental_cache_registry(Arc::new(MetaKvBackend::new(meta_client.clone())));
+    let layered_cache_registry = Arc::new(
+        with_default_composite_cache_registry(
+            layered_cache_builder.add_cache_registry(fundamental_cache_registry),
+        )
+        .context(error::BuildCacheRegistrySnafu)?
+        .build(),
+    );
+
+    let catalog_manager = KvBackendCatalogManager::new(
        Mode::Distributed,
        Some(meta_client.clone()),
        cached_meta_backend.clone(),
-        multi_cache_invalidator,
-    )
-    .await;
+        layered_cache_registry,
+    );
    let plugins: Plugins = Default::default();
    let state = Arc::new(QueryEngineState::new(
-        catalog_list,
+        catalog_manager,
        None,
        None,
        None,
--- a/src/cmd/src/cli/upgrade.rs
+++ b/src/cmd/src/cli/upgrade.rs
@@ -1,584 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use clap::Parser;
-use client::api::v1::meta::TableRouteValue;
-use common_meta::ddl::utils::region_storage_path;
-use common_meta::error as MetaError;
-use common_meta::key::catalog_name::{CatalogNameKey, CatalogNameValue};
-use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue, RegionInfo};
-use common_meta::key::schema_name::{SchemaNameKey, SchemaNameValue};
-use common_meta::key::table_info::{TableInfoKey, TableInfoValue};
-use common_meta::key::table_name::{TableNameKey, TableNameValue};
-use common_meta::key::table_region::{TableRegionKey, TableRegionValue};
-use common_meta::key::table_route::{TableRouteKey, TableRouteValue as NextTableRouteValue};
-use common_meta::key::{MetaKey, RegionDistribution, TableMetaValue};
-use common_meta::kv_backend::etcd::EtcdStore;
-use common_meta::kv_backend::KvBackendRef;
-use common_meta::range_stream::PaginationStream;
-use common_meta::rpc::router::TableRoute;
-use common_meta::rpc::store::{BatchDeleteRequest, BatchPutRequest, PutRequest, RangeRequest};
-use common_meta::rpc::KeyValue;
-use common_meta::util::get_prefix_end_key;
-use common_telemetry::info;
-use etcd_client::Client;
-use futures::TryStreamExt;
-use prost::Message;
-use snafu::ResultExt;
-use tracing_appender::non_blocking::WorkerGuard;
-use v1_helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue};
-
-use crate::cli::{Instance, Tool};
-use crate::error::{self, ConnectEtcdSnafu, Result};
-
-#[derive(Debug, Default, Parser)]
-pub struct UpgradeCommand {
-    #[clap(long)]
-    etcd_addr: String,
-    #[clap(long)]
-    dryrun: bool,
-
-    #[clap(long)]
-    skip_table_global_keys: bool,
-    #[clap(long)]
-    skip_catalog_keys: bool,
-    #[clap(long)]
-    skip_schema_keys: bool,
-    #[clap(long)]
-    skip_table_route_keys: bool,
-}
-
-impl UpgradeCommand {
-    pub async fn build(&self, guard: Vec<WorkerGuard>) -> Result<Instance> {
-        let client = Client::connect([&self.etcd_addr], None)
-            .await
-            .context(ConnectEtcdSnafu {
-                etcd_addr: &self.etcd_addr,
-            })?;
-        let tool = MigrateTableMetadata {
-            etcd_store: EtcdStore::with_etcd_client(client, 128),
-            dryrun: self.dryrun,
-            skip_catalog_keys: self.skip_catalog_keys,
-            skip_table_global_keys: self.skip_table_global_keys,
-            skip_schema_keys: self.skip_schema_keys,
-            skip_table_route_keys: self.skip_table_route_keys,
-        };
-        Ok(Instance::new(Box::new(tool), guard))
-    }
-}
-
-struct MigrateTableMetadata {
-    etcd_store: KvBackendRef,
-    dryrun: bool,
-
-    skip_table_global_keys: bool,
-
-    skip_catalog_keys: bool,
-
-    skip_schema_keys: bool,
-
-    skip_table_route_keys: bool,
-}
-
-#[async_trait]
-impl Tool for MigrateTableMetadata {
-    // migrates database's metadata from 0.3 to 0.4.
-    async fn do_work(&self) -> Result<()> {
-        if !self.skip_table_global_keys {
-            self.migrate_table_global_values().await?;
-        }
-        if !self.skip_catalog_keys {
-            self.migrate_catalog_keys().await?;
-        }
-        if !self.skip_schema_keys {
-            self.migrate_schema_keys().await?;
-        }
-        if !self.skip_table_route_keys {
-            self.migrate_table_route_keys().await?;
-        }
-        Ok(())
-    }
-}
-
-const PAGE_SIZE: usize = 1000;
-
-impl MigrateTableMetadata {
-    async fn migrate_table_route_keys(&self) -> Result<()> {
-        let key = b"__meta_table_route".to_vec();
-        let range_end = get_prefix_end_key(&key);
-        let mut keys = Vec::new();
-        info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
-
-        let mut stream = PaginationStream::new(
-            self.etcd_store.clone(),
-            RangeRequest::new().with_range(key, range_end),
-            PAGE_SIZE,
-            Arc::new(|kv: KeyValue| {
-                let value =
-                    TableRouteValue::decode(&kv.value[..]).context(MetaError::DecodeProtoSnafu)?;
-                Ok((kv.key, value))
-            }),
-        );
-
-        while let Some((key, value)) = stream.try_next().await.context(error::IterStreamSnafu)? {
-            let table_id = self.migrate_table_route_key(value).await?;
-            keys.push(key);
-            keys.push(TableRegionKey::new(table_id).to_bytes())
-        }
-
-        info!("Total migrated TableRouteKeys: {}", keys.len() / 2);
-        self.delete_migrated_keys(keys).await;
-
-        Ok(())
-    }
-
-    async fn migrate_table_route_key(&self, value: TableRouteValue) -> Result<u32> {
-        let table_route = TableRoute::try_from_raw(
-            &value.peers,
-            value.table_route.expect("expected table_route"),
-        )
-        .unwrap();
-
-        let new_table_value = NextTableRouteValue::physical(table_route.region_routes);
-
-        let table_id = table_route.table.id as u32;
-        let new_key = TableRouteKey::new(table_id);
-        info!("Creating '{new_key}'");
-
-        if self.dryrun {
-            info!("Dryrun: do nothing");
-        } else {
-            self.etcd_store
-                .put(
-                    PutRequest::new()
-                        .with_key(new_key.to_bytes())
-                        .with_value(new_table_value.try_as_raw_value().unwrap()),
-                )
-                .await
-                .unwrap();
-        }
-
-        Ok(table_id)
-    }
-
-    async fn migrate_schema_keys(&self) -> Result<()> {
-        // The schema key prefix.
-        let key = b"__s".to_vec();
-        let range_end = get_prefix_end_key(&key);
-
-        let mut keys = Vec::new();
-        info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
-        let mut stream = PaginationStream::new(
-            self.etcd_store.clone(),
-            RangeRequest::new().with_range(key, range_end),
-            PAGE_SIZE,
-            Arc::new(|kv: KeyValue| {
-                let key_str =
-                    std::str::from_utf8(&kv.key).context(MetaError::ConvertRawKeySnafu)?;
-                let key = v1SchemaKey::parse(key_str)
-                    .unwrap_or_else(|e| panic!("schema key is corrupted: {e}, key: {key_str}"));
-
-                Ok(key)
-            }),
-        );
-        while let Some(key) = stream.try_next().await.context(error::IterStreamSnafu)? {
-            let _ = self.migrate_schema_key(&key).await;
-            keys.push(key.to_string().as_bytes().to_vec());
-        }
-        info!("Total migrated SchemaKeys: {}", keys.len());
-        self.delete_migrated_keys(keys).await;
-
-        Ok(())
-    }
-
-    async fn migrate_schema_key(&self, key: &v1SchemaKey) -> Result<()> {
-        let new_key = SchemaNameKey::new(&key.catalog_name, &key.schema_name);
-        let schema_name_value = SchemaNameValue::default();
-
-        info!("Creating '{new_key}'");
-
-        if self.dryrun {
-            info!("Dryrun: do nothing");
-        } else {
-            self.etcd_store
-                .put(
-                    PutRequest::new()
-                        .with_key(new_key.to_bytes())
-                        .with_value(schema_name_value.try_as_raw_value().unwrap()),
-                )
-                .await
-                .unwrap();
-        }
-
-        Ok(())
-    }
-
-    async fn migrate_catalog_keys(&self) -> Result<()> {
-        // The catalog key prefix.
-        let key = b"__c".to_vec();
-        let range_end = get_prefix_end_key(&key);
-
-        let mut keys = Vec::new();
-        info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
-        let mut stream = PaginationStream::new(
-            self.etcd_store.clone(),
-            RangeRequest::new().with_range(key, range_end),
-            PAGE_SIZE,
-            Arc::new(|kv: KeyValue| {
-                let key_str =
-                    std::str::from_utf8(&kv.key).context(MetaError::ConvertRawKeySnafu)?;
-                let key = v1CatalogKey::parse(key_str)
-                    .unwrap_or_else(|e| panic!("catalog key is corrupted: {e}, key: {key_str}"));
-
-                Ok(key)
-            }),
-        );
-        while let Some(key) = stream.try_next().await.context(error::IterStreamSnafu)? {
-            let _ = self.migrate_catalog_key(&key).await;
-            keys.push(key.to_string().as_bytes().to_vec());
-        }
-        info!("Total migrated CatalogKeys: {}", keys.len());
-        self.delete_migrated_keys(keys).await;
-
-        Ok(())
-    }
-
-    async fn migrate_catalog_key(&self, key: &v1CatalogKey) {
-        let new_key = CatalogNameKey::new(&key.catalog_name);
-        let catalog_name_value = CatalogNameValue;
-
-        info!("Creating '{new_key}'");
-
-        if self.dryrun {
-            info!("Dryrun: do nothing");
-        } else {
-            self.etcd_store
-                .put(
-                    PutRequest::new()
-                        .with_key(new_key.to_bytes())
-                        .with_value(catalog_name_value.try_as_raw_value().unwrap()),
-                )
-                .await
-                .unwrap();
-        }
-    }
-
-    async fn migrate_table_global_values(&self) -> Result<()> {
-        let key = b"__tg".to_vec();
-        let range_end = get_prefix_end_key(&key);
-
-        let mut keys = Vec::new();
-
-        info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
-        let mut stream = PaginationStream::new(
-            self.etcd_store.clone(),
-            RangeRequest::new().with_range(key, range_end.clone()),
-            PAGE_SIZE,
-            Arc::new(|kv: KeyValue| {
-                let key = String::from_utf8_lossy(kv.key()).to_string();
-                let value = TableGlobalValue::from_bytes(kv.value())
-                    .unwrap_or_else(|e| panic!("table global value is corrupted: {e}, key: {key}"));
-
-                Ok((key, value))
-            }),
-        );
-        while let Some((key, value)) = stream.try_next().await.context(error::IterStreamSnafu)? {
-            self.create_table_name_key(&value).await;
-
-            self.create_datanode_table_keys(&value).await;
-
-            self.split_table_global_value(&key, value).await;
-
-            keys.push(key.as_bytes().to_vec());
-        }
-
-        info!("Total migrated TableGlobalKeys: {}", keys.len());
-        self.delete_migrated_keys(keys).await;
-
-        Ok(())
-    }
-
-    async fn delete_migrated_keys(&self, keys: Vec<Vec<u8>>) {
-        for keys in keys.chunks(PAGE_SIZE) {
-            info!("Deleting {} keys", keys.len());
-            let req = BatchDeleteRequest {
-                keys: keys.to_vec(),
-                prev_kv: false,
-            };
-            if self.dryrun {
-                info!("Dryrun: do nothing");
-            } else {
-                self.etcd_store.batch_delete(req).await.unwrap();
-            }
-        }
-    }
-
-    async fn split_table_global_value(&self, key: &str, value: TableGlobalValue) {
-        let table_id = value.table_id();
-        let region_distribution: RegionDistribution = value.regions_id_map.into_iter().collect();
-
-        let table_info_key = TableInfoKey::new(table_id);
-        let table_info_value = TableInfoValue::new(value.table_info);
-
-        let table_region_key = TableRegionKey::new(table_id);
-        let table_region_value = TableRegionValue::new(region_distribution);
-
-        info!("Splitting TableGlobalKey '{key}' into '{table_info_key}' and '{table_region_key}'");
-
-        if self.dryrun {
-            info!("Dryrun: do nothing");
-        } else {
-            self.etcd_store
-                .batch_put(
-                    BatchPutRequest::new()
-                        .add_kv(
-                            table_info_key.to_bytes(),
-                            table_info_value.try_as_raw_value().unwrap(),
-                        )
-                        .add_kv(
-                            table_region_key.to_bytes(),
-                            table_region_value.try_as_raw_value().unwrap(),
-                        ),
-                )
-                .await
-                .unwrap();
-        }
-    }
-
-    async fn create_table_name_key(&self, value: &TableGlobalValue) {
-        let table_info = &value.table_info;
-        let table_id = value.table_id();
-
-        let table_name_key = TableNameKey::new(
-            &table_info.catalog_name,
-            &table_info.schema_name,
-            &table_info.name,
-        );
-        let table_name_value = TableNameValue::new(table_id);
-
-        info!("Creating '{table_name_key}' => {table_id}");
-
-        if self.dryrun {
-            info!("Dryrun: do nothing");
-        } else {
-            self.etcd_store
-                .put(
-                    PutRequest::new()
-                        .with_key(table_name_key.to_bytes())
-                        .with_value(table_name_value.try_as_raw_value().unwrap()),
-                )
-                .await
-                .unwrap();
-        }
-    }
-
-    async fn create_datanode_table_keys(&self, value: &TableGlobalValue) {
-        let table_id = value.table_id();
-        let engine = value.table_info.meta.engine.as_str();
-        let region_storage_path = region_storage_path(
-            &value.table_info.catalog_name,
-            &value.table_info.schema_name,
-        );
-        let region_distribution: RegionDistribution =
-            value.regions_id_map.clone().into_iter().collect();
-
-        // TODO(niebayes): properly fetch or construct wal options.
-        let region_wal_options = HashMap::default();
-
-        let datanode_table_kvs = region_distribution
-            .into_iter()
-            .map(|(datanode_id, regions)| {
-                let k = DatanodeTableKey::new(datanode_id, table_id);
-                info!("Creating DatanodeTableKey '{k}' => {regions:?}");
-                (
-                    k,
-                    DatanodeTableValue::new(
-                        table_id,
-                        regions,
-                        RegionInfo {
-                            engine: engine.to_string(),
-                            region_storage_path: region_storage_path.clone(),
-                            region_options: (&value.table_info.meta.options).into(),
-                            region_wal_options: region_wal_options.clone(),
-                        },
-                    ),
-                )
-            })
-            .collect::<Vec<_>>();
-
-        if self.dryrun {
-            info!("Dryrun: do nothing");
-        } else {
-            let mut req = BatchPutRequest::new();
-            for (key, value) in datanode_table_kvs {
-                req = req.add_kv(key.to_bytes(), value.try_as_raw_value().unwrap());
-            }
-            self.etcd_store.batch_put(req).await.unwrap();
-        }
-    }
-}
-
-#[deprecated(since = "0.4.0", note = "Used for migrate old version(v0.3) metadata")]
-mod v1_helper {
-    use std::collections::HashMap;
-    use std::fmt::{Display, Formatter};
-
-    use err::{DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu};
-    use lazy_static::lazy_static;
-    use regex::Regex;
-    use serde::{Deserialize, Serialize};
-    use snafu::{ensure, OptionExt, ResultExt};
-    use table::metadata::{RawTableInfo, TableId};
-
-    pub const CATALOG_KEY_PREFIX: &str = "__c";
-    pub const SCHEMA_KEY_PREFIX: &str = "__s";
-
-    /// The pattern of a valid catalog, schema or table name.
-    const NAME_PATTERN: &str = "[a-zA-Z_:][a-zA-Z0-9_:]*";
-
-    lazy_static! {
-        static ref CATALOG_KEY_PATTERN: Regex =
-            Regex::new(&format!("^{CATALOG_KEY_PREFIX}-({NAME_PATTERN})$")).unwrap();
-    }
-
-    lazy_static! {
-        static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!(
-            "^{SCHEMA_KEY_PREFIX}-({NAME_PATTERN})-({NAME_PATTERN})$"
-        ))
-        .unwrap();
-    }
-
-    /// Table global info contains necessary info for a datanode to create table regions, including
-    /// table id, table meta(schema...), region id allocation across datanodes.
-    #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-    pub struct TableGlobalValue {
-        /// Id of datanode that created the global table info kv. only for debugging.
-        pub node_id: u64,
-        /// Allocation of region ids across all datanodes.
-        pub regions_id_map: HashMap<u64, Vec<u32>>,
-        pub table_info: RawTableInfo,
-    }
-
-    impl TableGlobalValue {
-        pub fn table_id(&self) -> TableId {
-            self.table_info.ident.table_id
-        }
-    }
-
-    pub struct CatalogKey {
-        pub catalog_name: String,
-    }
-
-    impl Display for CatalogKey {
-        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-            f.write_str(CATALOG_KEY_PREFIX)?;
-            f.write_str("-")?;
-            f.write_str(&self.catalog_name)
-        }
-    }
-
-    impl CatalogKey {
-        pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-            let key = s.as_ref();
-            let captures = CATALOG_KEY_PATTERN
-                .captures(key)
-                .context(InvalidCatalogSnafu { key })?;
-            ensure!(captures.len() == 2, InvalidCatalogSnafu { key });
-            Ok(Self {
-                catalog_name: captures[1].to_string(),
-            })
-        }
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    pub struct CatalogValue;
-
-    pub struct SchemaKey {
-        pub catalog_name: String,
-        pub schema_name: String,
-    }
-
-    impl Display for SchemaKey {
-        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-            f.write_str(SCHEMA_KEY_PREFIX)?;
-            f.write_str("-")?;
-            f.write_str(&self.catalog_name)?;
-            f.write_str("-")?;
-            f.write_str(&self.schema_name)
-        }
-    }
-
-    impl SchemaKey {
-        pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-            let key = s.as_ref();
-            let captures = SCHEMA_KEY_PATTERN
-                .captures(key)
-                .context(InvalidCatalogSnafu { key })?;
-            ensure!(captures.len() == 3, InvalidCatalogSnafu { key });
-            Ok(Self {
-                catalog_name: captures[1].to_string(),
-                schema_name: captures[2].to_string(),
-            })
-        }
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    pub struct SchemaValue;
-
-    macro_rules! define_catalog_value {
-    ( $($val_ty: ty), *) => {
-            $(
-                impl $val_ty {
-                    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-                        serde_json::from_str(s.as_ref())
-                            .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() })
-                    }
-
-                    pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self, Error> {
-                         Self::parse(&String::from_utf8_lossy(bytes.as_ref()))
-                    }
-                }
-            )*
-        }
-    }
-
-    define_catalog_value!(TableGlobalValue);
-
-    mod err {
-        use snafu::{Location, Snafu};
-
-        #[derive(Debug, Snafu)]
-        #[snafu(visibility(pub))]
-        pub enum Error {
-            #[snafu(display("Invalid catalog info: {}", key))]
-            InvalidCatalog {
-                key: String,
-                #[snafu(implicit)]
-                location: Location,
-            },
-
-            #[snafu(display("Failed to deserialize catalog entry value: {}", raw))]
-            DeserializeCatalogEntryValue {
-                raw: String,
-                #[snafu(implicit)]
-                location: Location,
-                source: serde_json::error::Error,
-            },
-        }
-    }
-}
--- a/src/cmd/src/error.rs
+++ b/src/cmd/src/error.rs
@@ -163,6 +163,15 @@ pub enum Error {
        location: Location,
    },

+    #[snafu(display("Failed to request database, sql: {sql}"))]
+    RequestDatabase {
+        sql: String,
+        #[snafu(source)]
+        source: client::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
    #[snafu(display("Failed to collect RecordBatches"))]
    CollectRecordBatches {
        #[snafu(implicit)]
@@ -354,6 +363,7 @@ impl ErrorExt for Error {
            Error::ReplCreation { .. } | Error::Readline { .. } | Error::HttpQuerySql { .. } => {
                StatusCode::Internal
            }
+            Error::RequestDatabase { source, .. } => source.status_code(),
            Error::CollectRecordBatches { source, .. }
            | Error::PrettyPrintRecordBatches { source, .. } => source.status_code(),
            Error::StartMetaClient { source, .. } => source.status_code(),
@@ -365,11 +375,11 @@ impl ErrorExt for Error {

            Error::SerdeJson { .. } | Error::FileIo { .. } => StatusCode::Unexpected,

-            Error::CacheRequired { .. } | Error::BuildCacheRegistry { .. } => StatusCode::Internal,
-
            Error::Other { source, .. } => source.status_code(),

            Error::BuildRuntime { source, .. } => source.status_code(),
+
+            Error::CacheRequired { .. } | Error::BuildCacheRegistry { .. } => StatusCode::Internal,
        }
    }

--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -16,10 +16,7 @@ use std::sync::Arc;
 use std::time::Duration;

 use async_trait::async_trait;
-use cache::{
-    build_fundamental_cache_registry, with_default_composite_cache_registry, TABLE_CACHE_NAME,
-    TABLE_ROUTE_CACHE_NAME,
-};
+use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
 use catalog::kvbackend::{CachedMetaKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend};
 use clap::Parser;
 use client::client_manager::DatanodeClients;
@@ -302,25 +299,12 @@ impl StartCommand {
            .build(),
        );

-        let table_cache = layered_cache_registry
-            .get()
-            .context(error::CacheRequiredSnafu {
-                name: TABLE_CACHE_NAME,
-            })?;
-        let table_route_cache =
-            layered_cache_registry
-                .get()
-                .context(error::CacheRequiredSnafu {
-                    name: TABLE_ROUTE_CACHE_NAME,
-                })?;
        let catalog_manager = KvBackendCatalogManager::new(
            opts.mode,
            Some(meta_client.clone()),
            cached_meta_backend.clone(),
-            table_cache,
-            table_route_cache,
-        )
-        .await;
+            layered_cache_registry.clone(),
+        );

        let executor = HandlerGroupExecutor::new(vec![
            Arc::new(ParseMailboxMessageHandler),
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -16,10 +16,7 @@ use std::sync::Arc;
 use std::{fs, path};

 use async_trait::async_trait;
-use cache::{
-    build_fundamental_cache_registry, with_default_composite_cache_registry, TABLE_CACHE_NAME,
-    TABLE_ROUTE_CACHE_NAME,
-};
+use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
 use catalog::kvbackend::KvBackendCatalogManager;
 use clap::Parser;
 use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
@@ -61,14 +58,14 @@ use servers::export_metrics::ExportMetricsOption;
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
-use snafu::{OptionExt, ResultExt};
+use snafu::ResultExt;
 use tracing_appender::non_blocking::WorkerGuard;

 use crate::error::{
-    BuildCacheRegistrySnafu, CacheRequiredSnafu, CreateDirSnafu, IllegalConfigSnafu,
-    InitDdlManagerSnafu, InitMetadataSnafu, InitTimezoneSnafu, LoadLayeredConfigSnafu, Result,
-    ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu,
-    StartProcedureManagerSnafu, StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
+    BuildCacheRegistrySnafu, CreateDirSnafu, IllegalConfigSnafu, InitDdlManagerSnafu,
+    InitMetadataSnafu, InitTimezoneSnafu, LoadLayeredConfigSnafu, Result, ShutdownDatanodeSnafu,
+    ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
+    StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
 };
 use crate::options::GlobalOptions;
 use crate::{log_versions, App};
@@ -421,20 +418,12 @@ impl StartCommand {
            .build(),
        );

-        let table_cache = layered_cache_registry.get().context(CacheRequiredSnafu {
-            name: TABLE_CACHE_NAME,
-        })?;
-        let table_route_cache = layered_cache_registry.get().context(CacheRequiredSnafu {
-            name: TABLE_ROUTE_CACHE_NAME,
-        })?;
        let catalog_manager = KvBackendCatalogManager::new(
            dn_opts.mode,
            None,
            kv_backend.clone(),
-            table_cache,
-            table_route_cache,
-        )
-        .await;
+            layered_cache_registry.clone(),
+        );

        let table_metadata_manager =
            Self::create_table_metadata_manager(kv_backend.clone()).await?;
--- a/src/common/function/src/scalars/math/clamp.rs
+++ b/src/common/function/src/scalars/math/clamp.rs
@@ -143,8 +143,6 @@ fn clamp_impl<T: LogicalPrimitiveType, const CLAMP_MIN: bool, const CLAMP_MAX: b
    min: T::Native,
    max: T::Native,
 ) -> Result<VectorRef> {
-    common_telemetry::info!("[DEBUG] min {min:?}, max {max:?}");
-
    let iter = ArrayIter::new(input);
    let result = iter.map(|x| {
        x.map(|x| {
--- a/src/common/grpc/Cargo.toml
+++ b/src/common/grpc/Cargo.toml
@@ -25,7 +25,7 @@ prost.workspace = true
 snafu.workspace = true
 tokio.workspace = true
 tonic.workspace = true
-tower = "0.4"
+tower.workspace = true

 [dev-dependencies]
 criterion = "0.4"
--- a/src/common/meta/src/cache.rs
+++ b/src/common/meta/src/cache.rs
@@ -24,7 +24,7 @@ pub use registry::{
    LayeredCacheRegistryBuilder, LayeredCacheRegistryRef,
 };
 pub use table::{
-    new_table_info_cache, new_table_name_cache, new_table_route_cache, TableInfoCache,
-    TableInfoCacheRef, TableNameCache, TableNameCacheRef, TableRoute, TableRouteCache,
-    TableRouteCacheRef,
+    new_table_info_cache, new_table_name_cache, new_table_route_cache, new_view_info_cache,
+    TableInfoCache, TableInfoCacheRef, TableNameCache, TableNameCacheRef, TableRoute,
+    TableRouteCache, TableRouteCacheRef, ViewInfoCache, ViewInfoCacheRef,
 };
--- a/src/common/meta/src/cache/flow/table_flownode.rs
+++ b/src/common/meta/src/cache/flow/table_flownode.rs
@@ -145,13 +145,13 @@ mod tests {

    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
    use moka::future::CacheBuilder;
+    use table::table_name::TableName;

    use crate::cache::flow::table_flownode::new_table_flownode_set_cache;
    use crate::instruction::{CacheIdent, CreateFlow, DropFlow};
    use crate::key::flow::flow_info::FlowInfoValue;
    use crate::key::flow::FlowMetadataManager;
    use crate::kv_backend::memory::MemoryKvBackend;
-    use crate::table_name::TableName;

    #[tokio::test]
    async fn test_cache_empty_set() {
--- a/src/common/meta/src/cache/table.rs
+++ b/src/common/meta/src/cache/table.rs
@@ -15,6 +15,9 @@
 mod table_info;
 mod table_name;
 mod table_route;
+mod view_info;
+
 pub use table_info::{new_table_info_cache, TableInfoCache, TableInfoCacheRef};
 pub use table_name::{new_table_name_cache, TableNameCache, TableNameCacheRef};
 pub use table_route::{new_table_route_cache, TableRoute, TableRouteCache, TableRouteCacheRef};
+pub use view_info::{new_view_info_cache, ViewInfoCache, ViewInfoCacheRef};
--- a/src/common/meta/src/cache/table/table_name.rs
+++ b/src/common/meta/src/cache/table/table_name.rs
@@ -18,6 +18,7 @@ use futures::future::BoxFuture;
 use moka::future::Cache;
 use snafu::OptionExt;
 use table::metadata::TableId;
+use table::table_name::TableName;

 use crate::cache::{CacheContainer, Initializer};
 use crate::error;
@@ -25,7 +26,6 @@ use crate::error::Result;
 use crate::instruction::CacheIdent;
 use crate::key::table_name::{TableNameKey, TableNameManager, TableNameManagerRef};
 use crate::kv_backend::KvBackendRef;
-use crate::table_name::TableName;

 /// [TableNameCache] caches the [TableName] to [TableId] mapping.
 pub type TableNameCache = CacheContainer<TableName, TableId, CacheIdent>;
--- a/src/common/meta/src/cache/table/view_info.rs
+++ b/src/common/meta/src/cache/table/view_info.rs
@@ -0,0 +1,143 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use futures::future::BoxFuture;
+use moka::future::Cache;
+use snafu::OptionExt;
+use store_api::storage::TableId;
+
+use crate::cache::{CacheContainer, Initializer};
+use crate::error;
+use crate::error::Result;
+use crate::instruction::CacheIdent;
+use crate::key::view_info::{ViewInfoManager, ViewInfoManagerRef, ViewInfoValue};
+use crate::kv_backend::KvBackendRef;
+
+/// [ViewInfoCache] caches the [TableId] to [ViewInfoValue] mapping.
+pub type ViewInfoCache = CacheContainer<TableId, Arc<ViewInfoValue>, CacheIdent>;
+
+pub type ViewInfoCacheRef = Arc<ViewInfoCache>;
+
+/// Constructs a [ViewInfoCache].
+pub fn new_view_info_cache(
+    name: String,
+    cache: Cache<TableId, Arc<ViewInfoValue>>,
+    kv_backend: KvBackendRef,
+) -> ViewInfoCache {
+    let view_info_manager = Arc::new(ViewInfoManager::new(kv_backend));
+    let init = init_factory(view_info_manager);
+
+    CacheContainer::new(name, cache, Box::new(invalidator), init, Box::new(filter))
+}
+
+fn init_factory(view_info_manager: ViewInfoManagerRef) -> Initializer<TableId, Arc<ViewInfoValue>> {
+    Arc::new(move |view_id| {
+        let view_info_manager = view_info_manager.clone();
+        Box::pin(async move {
+            let view_info = view_info_manager
+                .get(*view_id)
+                .await?
+                .context(error::ValueNotExistSnafu {})?
+                .into_inner();
+
+            Ok(Some(Arc::new(view_info)))
+        })
+    })
+}
+
+fn invalidator<'a>(
+    cache: &'a Cache<TableId, Arc<ViewInfoValue>>,
+    ident: &'a CacheIdent,
+) -> BoxFuture<'a, Result<()>> {
+    Box::pin(async move {
+        if let CacheIdent::TableId(table_id) = ident {
+            cache.invalidate(table_id).await
+        }
+        Ok(())
+    })
+}
+
+fn filter(ident: &CacheIdent) -> bool {
+    matches!(ident, CacheIdent::TableId(_))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashSet;
+    use std::sync::Arc;
+
+    use moka::future::CacheBuilder;
+    use table::table_name::TableName;
+
+    use super::*;
+    use crate::ddl::tests::create_view::test_create_view_task;
+    use crate::key::TableMetadataManager;
+    use crate::kv_backend::memory::MemoryKvBackend;
+
+    #[tokio::test]
+    async fn test_view_info_cache() {
+        let mem_kv = Arc::new(MemoryKvBackend::default());
+        let table_metadata_manager = TableMetadataManager::new(mem_kv.clone());
+        let cache = CacheBuilder::new(128).build();
+        let cache = new_view_info_cache("test".to_string(), cache, mem_kv.clone());
+
+        let result = cache.get(1024).await.unwrap();
+        assert!(result.is_none());
+        let mut task = test_create_view_task("my_view");
+        let table_names = {
+            let mut set = HashSet::new();
+            set.insert(TableName {
+                catalog_name: "greptime".to_string(),
+                schema_name: "public".to_string(),
+                table_name: "a_table".to_string(),
+            });
+            set.insert(TableName {
+                catalog_name: "greptime".to_string(),
+                schema_name: "public".to_string(),
+                table_name: "b_table".to_string(),
+            });
+            set
+        };
+
+        task.view_info.ident.table_id = 1024;
+        table_metadata_manager
+            .create_view_metadata(
+                task.view_info.clone(),
+                task.create_view.logical_plan.clone(),
+                table_names,
+            )
+            .await
+            .unwrap();
+
+        let view_info = cache.get(1024).await.unwrap().unwrap();
+        assert_eq!(view_info.view_info, task.create_view.logical_plan);
+        assert_eq!(
+            view_info.table_names,
+            task.create_view
+                .table_names
+                .iter()
+                .map(|t| t.clone().into())
+                .collect::<HashSet<_>>()
+        );
+
+        assert!(cache.contains_key(&1024));
+        cache
+            .invalidate(&[CacheIdent::TableId(1024)])
+            .await
+            .unwrap();
+        assert!(!cache.contains_key(&1024));
+    }
+}
--- a/src/common/meta/src/ddl.rs
+++ b/src/common/meta/src/ddl.rs
@@ -48,7 +48,7 @@ pub mod table_meta;
 #[cfg(any(test, feature = "testing"))]
 pub mod test_util;
 #[cfg(test)]
-mod tests;
+pub(crate) mod tests;
 pub mod truncate_table;
 pub mod utils;

--- a/src/common/meta/src/ddl/alter_logical_tables/table_cache_keys.rs
+++ b/src/common/meta/src/ddl/alter_logical_tables/table_cache_keys.rs
@@ -13,10 +13,10 @@
 // limitations under the License.

 use table::metadata::RawTableInfo;
+use table::table_name::TableName;

 use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
 use crate::instruction::CacheIdent;
-use crate::table_name::TableName;

 impl AlterLogicalTablesProcedure {
    pub(crate) fn build_table_cache_keys_to_invalidate(&self) -> Vec<CacheIdent> {
--- a/src/common/meta/src/ddl/create_logical_tables/update_metadata.rs
+++ b/src/common/meta/src/ddl/create_logical_tables/update_metadata.rs
@@ -18,13 +18,13 @@ use common_telemetry::{info, warn};
 use itertools::Itertools;
 use snafu::OptionExt;
 use table::metadata::TableId;
+use table::table_name::TableName;

 use crate::cache_invalidator::Context;
 use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure;
 use crate::ddl::physical_table_metadata;
 use crate::error::{Result, TableInfoNotFoundSnafu};
 use crate::instruction::CacheIdent;
-use crate::table_name::TableName;

 impl CreateLogicalTablesProcedure {
    pub(crate) async fn update_physical_table_metadata(&mut self) -> Result<()> {
--- a/src/common/meta/src/ddl/create_view.rs
+++ b/src/common/meta/src/ddl/create_view.rs
@@ -22,9 +22,11 @@ use strum::AsRefStr;
 use table::metadata::{RawTableInfo, TableId, TableType};
 use table::table_reference::TableReference;

+use crate::cache_invalidator::Context;
 use crate::ddl::utils::handle_retry_error;
 use crate::ddl::{DdlContext, TableMetadata, TableMetadataAllocatorContext};
 use crate::error::{self, Result};
+use crate::instruction::CacheIdent;
 use crate::key::table_name::TableNameKey;
 use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock};
 use crate::rpc::ddl::CreateViewTask;
@@ -157,6 +159,25 @@ impl CreateViewProcedure {
        Ok(Status::executing(true))
    }

+    async fn invalidate_view_cache(&self) -> Result<()> {
+        let cache_invalidator = &self.context.cache_invalidator;
+        let ctx = Context {
+            subject: Some("Invalidate view cache by creating view".to_string()),
+        };
+
+        cache_invalidator
+            .invalidate(
+                &ctx,
+                &[
+                    CacheIdent::TableName(self.data.table_ref().into()),
+                    CacheIdent::TableId(self.view_id()),
+                ],
+            )
+            .await?;
+
+        Ok(())
+    }
+
    /// Creates view metadata
    ///
    /// Abort(not-retry):
@@ -175,15 +196,21 @@ impl CreateViewProcedure {
                    view_name: self.data.table_ref().to_string(),
                })?;
            let new_logical_plan = self.data.task.raw_logical_plan().clone();
+            let table_names = self.data.task.table_names();
+
            manager
-                .update_view_info(view_id, &current_view_info, new_logical_plan)
+                .update_view_info(view_id, &current_view_info, new_logical_plan, table_names)
                .await?;

            info!("Updated view metadata for view {view_id}");
        } else {
            let raw_view_info = self.view_info().clone();
            manager
-                .create_view_metadata(raw_view_info, self.data.task.raw_logical_plan())
+                .create_view_metadata(
+                    raw_view_info,
+                    self.data.task.raw_logical_plan().clone(),
+                    self.data.task.table_names(),
+                )
                .await?;

            info!(
@@ -191,6 +218,7 @@ impl CreateViewProcedure {
                ctx.procedure_id
            );
        }
+        self.invalidate_view_cache().await?;

        Ok(Status::done_with_output(view_id))
    }
--- a/src/common/meta/src/ddl/drop_database/cursor.rs
+++ b/src/common/meta/src/ddl/drop_database/cursor.rs
@@ -14,19 +14,23 @@

 use std::any::Any;

+use common_catalog::format_full_table_name;
 use common_procedure::Status;
 use futures::TryStreamExt;
 use serde::{Deserialize, Serialize};
-use table::metadata::TableId;
+use snafu::OptionExt;
+use table::metadata::{TableId, TableType};
+use table::table_name::TableName;

 use super::executor::DropDatabaseExecutor;
 use super::metadata::DropDatabaseRemoveMetadata;
 use super::DropTableTarget;
+use crate::cache_invalidator::Context;
 use crate::ddl::drop_database::{DropDatabaseContext, State};
 use crate::ddl::DdlContext;
-use crate::error::Result;
+use crate::error::{Result, TableInfoNotFoundSnafu};
+use crate::instruction::CacheIdent;
 use crate::key::table_route::TableRouteValue;
-use crate::table_name::TableName;

 #[derive(Debug, Serialize, Deserialize)]
 pub(crate) struct DropDatabaseCursor {
@@ -101,6 +105,40 @@ impl DropDatabaseCursor {
            )),
        }
    }
+
+    async fn handle_view(
+        &self,
+        ddl_ctx: &DdlContext,
+        ctx: &mut DropDatabaseContext,
+        table_name: String,
+        table_id: TableId,
+    ) -> Result<(Box<dyn State>, Status)> {
+        let view_name = TableName::new(&ctx.catalog, &ctx.schema, &table_name);
+        ddl_ctx
+            .table_metadata_manager
+            .destroy_view_info(table_id, &view_name)
+            .await?;
+
+        let cache_invalidator = &ddl_ctx.cache_invalidator;
+        let ctx = Context {
+            subject: Some("Invalidate table cache by dropping table".to_string()),
+        };
+
+        cache_invalidator
+            .invalidate(
+                &ctx,
+                &[
+                    CacheIdent::TableName(view_name),
+                    CacheIdent::TableId(table_id),
+                ],
+            )
+            .await?;
+
+        Ok((
+            Box::new(DropDatabaseCursor::new(self.target)),
+            Status::executing(false),
+        ))
+    }
 }

 #[async_trait::async_trait]
@@ -122,6 +160,20 @@ impl State for DropDatabaseCursor {
        match ctx.tables.as_mut().unwrap().try_next().await? {
            Some((table_name, table_name_value)) => {
                let table_id = table_name_value.table_id();
+
+                let table_info_value = ddl_ctx
+                    .table_metadata_manager
+                    .table_info_manager()
+                    .get(table_id)
+                    .await?
+                    .with_context(|| TableInfoNotFoundSnafu {
+                        table: format_full_table_name(&ctx.catalog, &ctx.schema, &table_name),
+                    })?;
+
+                if table_info_value.table_info.table_type == TableType::View {
+                    return self.handle_view(ddl_ctx, ctx, table_name, table_id).await;
+                }
+
                match ddl_ctx
                    .table_metadata_manager
                    .table_route_manager()
--- a/src/common/meta/src/ddl/drop_database/executor.rs
+++ b/src/common/meta/src/ddl/drop_database/executor.rs
@@ -19,6 +19,7 @@ use common_telemetry::info;
 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use table::metadata::TableId;
+use table::table_name::TableName;

 use super::cursor::DropDatabaseCursor;
 use super::{DropDatabaseContext, DropTableTarget};
@@ -29,7 +30,6 @@ use crate::error::{self, Result};
 use crate::key::table_route::TableRouteValue;
 use crate::region_keeper::OperatingRegionGuard;
 use crate::rpc::router::{operating_leader_regions, RegionRoute};
-use crate::table_name::TableName;

 #[derive(Debug, Serialize, Deserialize)]
 pub(crate) struct DropDatabaseExecutor {
@@ -135,6 +135,7 @@ mod tests {
    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
    use common_error::ext::BoxedError;
    use common_recordbatch::SendableRecordBatchStream;
+    use table::table_name::TableName;

    use crate::ddl::drop_database::cursor::DropDatabaseCursor;
    use crate::ddl::drop_database::executor::DropDatabaseExecutor;
@@ -144,7 +145,6 @@ mod tests {
    use crate::key::datanode_table::DatanodeTableKey;
    use crate::peer::Peer;
    use crate::rpc::router::region_distribution;
-    use crate::table_name::TableName;
    use crate::test_util::{new_ddl_context, MockDatanodeHandler, MockDatanodeManager};

    #[derive(Clone)]
--- a/src/common/meta/src/ddl/drop_table/executor.rs
+++ b/src/common/meta/src/ddl/drop_table/executor.rs
@@ -23,6 +23,7 @@ use futures::future::join_all;
 use snafu::ensure;
 use store_api::storage::RegionId;
 use table::metadata::TableId;
+use table::table_name::TableName;

 use crate::cache_invalidator::Context;
 use crate::ddl::utils::add_peer_context_if_needed;
@@ -32,7 +33,6 @@ use crate::instruction::CacheIdent;
 use crate::key::table_name::TableNameKey;
 use crate::key::table_route::TableRouteValue;
 use crate::rpc::router::{find_leader_regions, find_leaders, RegionRoute};
-use crate::table_name::TableName;

 /// [Control] indicated to the caller whether to go to the next step.
 #[derive(Debug)]
@@ -224,6 +224,7 @@ mod tests {
    use api::v1::{ColumnDataType, SemanticType};
    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
    use table::metadata::RawTableInfo;
+    use table::table_name::TableName;

    use super::*;
    use crate::ddl::test_util::columns::TestColumnDefBuilder;
@@ -231,7 +232,6 @@ mod tests {
        build_raw_table_info_from_expr, TestCreateTableExprBuilder,
    };
    use crate::key::table_route::TableRouteValue;
-    use crate::table_name::TableName;
    use crate::test_util::{new_ddl_context, MockDatanodeManager};

    fn test_create_raw_table_info(name: &str) -> RawTableInfo {
--- a/src/common/meta/src/ddl/tests.rs
+++ b/src/common/meta/src/ddl/tests.rs
@@ -17,7 +17,7 @@ mod alter_table;
 mod create_flow;
 mod create_logical_tables;
 mod create_table;
-mod create_view;
+pub(crate) mod create_view;
 mod drop_database;
 mod drop_flow;
 mod drop_table;
--- a/src/common/meta/src/ddl/tests/create_flow.rs
+++ b/src/common/meta/src/ddl/tests/create_flow.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_procedure_test::execute_procedure_until_done;
 use session::context::QueryContext;
+use table::table_name::TableName;

 use crate::ddl::create_flow::CreateFlowProcedure;
 use crate::ddl::test_util::create_table::test_create_table_task;
@@ -27,7 +28,6 @@ use crate::ddl::DdlContext;
 use crate::key::table_route::TableRouteValue;
 use crate::key::FlowId;
 use crate::rpc::ddl::CreateFlowTask;
-use crate::table_name::TableName;
 use crate::test_util::{new_ddl_context, MockFlownodeManager};
 use crate::{error, ClusterId};

--- a/src/common/meta/src/ddl/tests/create_view.rs
+++ b/src/common/meta/src/ddl/tests/create_view.rs
@@ -13,9 +13,10 @@
 // limitations under the License.

 use std::assert_matches::assert_matches;
+use std::collections::HashSet;
 use std::sync::Arc;

-use api::v1::CreateViewExpr;
+use api::v1::{CreateViewExpr, TableName};
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_procedure::{Context as ProcedureContext, Procedure, ProcedureId, Status};
@@ -31,7 +32,35 @@ use crate::error::Error;
 use crate::rpc::ddl::CreateViewTask;
 use crate::test_util::{new_ddl_context, MockDatanodeManager};

-fn test_create_view_task(name: &str) -> CreateViewTask {
+fn test_table_names() -> HashSet<table::table_name::TableName> {
+    let mut set = HashSet::new();
+    set.insert(table::table_name::TableName {
+        catalog_name: "greptime".to_string(),
+        schema_name: "public".to_string(),
+        table_name: "a_table".to_string(),
+    });
+    set.insert(table::table_name::TableName {
+        catalog_name: "greptime".to_string(),
+        schema_name: "public".to_string(),
+        table_name: "b_table".to_string(),
+    });
+    set
+}
+
+pub(crate) fn test_create_view_task(name: &str) -> CreateViewTask {
+    let table_names = vec![
+        TableName {
+            catalog_name: "greptime".to_string(),
+            schema_name: "public".to_string(),
+            table_name: "a_table".to_string(),
+        },
+        TableName {
+            catalog_name: "greptime".to_string(),
+            schema_name: "public".to_string(),
+            table_name: "b_table".to_string(),
+        },
+    ];
+
    let expr = CreateViewExpr {
        catalog_name: "greptime".to_string(),
        schema_name: "public".to_string(),
@@ -39,6 +68,7 @@ fn test_create_view_task(name: &str) -> CreateViewTask {
        or_replace: false,
        create_if_not_exists: false,
        logical_plan: vec![1, 2, 3],
+        table_names,
    };

    let view_info = RawTableInfo {
@@ -70,7 +100,11 @@ async fn test_on_prepare_view_exists_err() {
    // Puts a value to table name key.
    ddl_context
        .table_metadata_manager
-        .create_view_metadata(task.view_info.clone(), &task.create_view.logical_plan)
+        .create_view_metadata(
+            task.view_info.clone(),
+            task.create_view.logical_plan.clone(),
+            test_table_names(),
+        )
        .await
        .unwrap();
    let mut procedure = CreateViewProcedure::new(cluster_id, task, ddl_context);
@@ -90,7 +124,11 @@ async fn test_on_prepare_with_create_if_view_exists() {
    // Puts a value to table name key.
    ddl_context
        .table_metadata_manager
-        .create_view_metadata(task.view_info.clone(), &task.create_view.logical_plan)
+        .create_view_metadata(
+            task.view_info.clone(),
+            task.create_view.logical_plan.clone(),
+            test_table_names(),
+        )
        .await
        .unwrap();
    let mut procedure = CreateViewProcedure::new(cluster_id, task, ddl_context);
--- a/src/common/meta/src/ddl/tests/drop_flow.rs
+++ b/src/common/meta/src/ddl/tests/drop_flow.rs
@@ -18,6 +18,7 @@ use std::sync::Arc;

 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_procedure_test::execute_procedure_until_done;
+use table::table_name::TableName;

 use crate::ddl::drop_flow::DropFlowProcedure;
 use crate::ddl::test_util::create_table::test_create_table_task;
@@ -26,7 +27,6 @@ use crate::ddl::tests::create_flow::create_test_flow;
 use crate::error;
 use crate::key::table_route::TableRouteValue;
 use crate::rpc::ddl::DropFlowTask;
-use crate::table_name::TableName;
 use crate::test_util::{new_ddl_context, MockFlownodeManager};

 fn test_drop_flow_task(flow_name: &str, flow_id: u32, drop_if_exists: bool) -> DropFlowTask {
--- a/src/common/meta/src/ddl/truncate_table.rs
+++ b/src/common/meta/src/ddl/truncate_table.rs
@@ -28,6 +28,7 @@ use snafu::{ensure, ResultExt};
 use store_api::storage::RegionId;
 use strum::AsRefStr;
 use table::metadata::{RawTableInfo, TableId};
+use table::table_name::TableName;
 use table::table_reference::TableReference;

 use super::utils::handle_retry_error;
@@ -40,7 +41,6 @@ use crate::key::DeserializedValueWithBytes;
 use crate::lock_key::{CatalogLock, SchemaLock, TableLock};
 use crate::rpc::ddl::TruncateTableTask;
 use crate::rpc::router::{find_leader_regions, find_leaders, RegionRoute};
-use crate::table_name::TableName;
 use crate::{metrics, ClusterId};

 pub struct TruncateTableProcedure {
--- a/src/common/meta/src/ddl_manager.rs
+++ b/src/common/meta/src/ddl_manager.rs
@@ -489,8 +489,7 @@ async fn handle_create_table_task(

    Ok(SubmitDdlTaskResponse {
        key: procedure_id.into(),
-        table_id: Some(table_id),
-        ..Default::default()
+        table_ids: vec![table_id],
    })
 }

@@ -534,7 +533,6 @@ async fn handle_create_logical_table_tasks(
    Ok(SubmitDdlTaskResponse {
        key: procedure_id.into(),
        table_ids,
-        ..Default::default()
    })
 }

@@ -690,8 +688,7 @@ async fn handle_create_view_task(

    Ok(SubmitDdlTaskResponse {
        key: procedure_id.into(),
-        table_id: Some(view_id),
-        ..Default::default()
+        table_ids: vec![view_id],
    })
 }

--- a/src/common/meta/src/instruction.rs
+++ b/src/common/meta/src/instruction.rs
@@ -20,11 +20,11 @@ use serde::{Deserialize, Serialize};
 use store_api::storage::{RegionId, RegionNumber};
 use strum::Display;
 use table::metadata::TableId;
+use table::table_name::TableName;

 use crate::flow_name::FlowName;
 use crate::key::schema_name::SchemaName;
 use crate::key::FlowId;
-use crate::table_name::TableName;
 use crate::{ClusterId, DatanodeId, FlownodeId};

 #[derive(Eq, Hash, PartialEq, Clone, Debug, Serialize, Deserialize)]
--- a/src/common/meta/src/key.rs
+++ b/src/common/meta/src/key.rs
@@ -89,9 +89,6 @@ pub mod flow;
 pub mod schema_name;
 pub mod table_info;
 pub mod table_name;
-// TODO(weny): removes it.
-#[allow(deprecated)]
-pub mod table_region;
 pub mod view_info;
 // TODO(weny): removes it.
 #[allow(deprecated)]
@@ -119,6 +116,7 @@ use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::storage::RegionNumber;
 use table::metadata::{RawTableInfo, TableId};
+use table::table_name::TableName;
 use table_info::{TableInfoKey, TableInfoManager, TableInfoValue};
 use table_name::{TableNameKey, TableNameManager, TableNameValue};
 use view_info::{ViewInfoKey, ViewInfoManager, ViewInfoValue};
@@ -138,14 +136,12 @@ use crate::kv_backend::txn::{Txn, TxnOp};
 use crate::kv_backend::KvBackendRef;
 use crate::rpc::router::{region_distribution, RegionRoute, RegionStatus};
 use crate::rpc::store::BatchDeleteRequest;
-use crate::table_name::TableName;
 use crate::DatanodeId;

 pub const NAME_PATTERN: &str = r"[a-zA-Z_:-][a-zA-Z0-9_:\-\.]*";
 pub const MAINTENANCE_KEY: &str = "maintenance";

 const DATANODE_TABLE_KEY_PREFIX: &str = "__dn_table";
-const TABLE_REGION_KEY_PREFIX: &str = "__table_region";
 pub const TABLE_INFO_KEY_PREFIX: &str = "__table_info";
 pub const VIEW_INFO_KEY_PREFIX: &str = "__view_info";
 pub const TABLE_NAME_KEY_PREFIX: &str = "__table_name";
@@ -490,7 +486,8 @@ impl TableMetadataManager {
    pub async fn create_view_metadata(
        &self,
        view_info: RawTableInfo,
-        raw_logical_plan: &Vec<u8>,
+        raw_logical_plan: Vec<u8>,
+        table_names: HashSet<TableName>,
    ) -> Result<()> {
        let view_id = view_info.ident.table_id;

@@ -512,7 +509,7 @@ impl TableMetadataManager {
            .build_create_txn(view_id, &table_info_value)?;

        // Creates view info
-        let view_info_value = ViewInfoValue::new(raw_logical_plan);
+        let view_info_value = ViewInfoValue::new(raw_logical_plan, table_names);
        let (create_view_info_txn, on_create_view_info_failure) = self
            .view_info_manager()
            .build_create_txn(view_id, &view_info_value)?;
@@ -804,6 +801,33 @@ impl TableMetadataManager {
        Ok(())
    }

+    fn view_info_keys(&self, view_id: TableId, view_name: &TableName) -> Result<Vec<Vec<u8>>> {
+        let mut keys = Vec::with_capacity(3);
+        let view_name = TableNameKey::new(
+            &view_name.catalog_name,
+            &view_name.schema_name,
+            &view_name.table_name,
+        );
+        let table_info_key = TableInfoKey::new(view_id);
+        let view_info_key = ViewInfoKey::new(view_id);
+        keys.push(view_name.to_bytes());
+        keys.push(table_info_key.to_bytes());
+        keys.push(view_info_key.to_bytes());
+
+        Ok(keys)
+    }
+
+    /// Deletes metadata for view **permanently**.
+    /// The caller MUST ensure it has the exclusive access to `ViewNameKey`.
+    pub async fn destroy_view_info(&self, view_id: TableId, view_name: &TableName) -> Result<()> {
+        let keys = self.view_info_keys(view_id, view_name)?;
+        let _ = self
+            .kv_backend
+            .batch_delete(BatchDeleteRequest::new().with_keys(keys))
+            .await?;
+        Ok(())
+    }
+
    /// Renames the table name and returns an error if different metadata exists.
    /// The caller MUST ensure it has the exclusive access to old and new `TableNameKey`s,
    /// and the new `TableNameKey` MUST be empty.
@@ -903,8 +927,9 @@ impl TableMetadataManager {
        view_id: TableId,
        current_view_info_value: &DeserializedValueWithBytes<ViewInfoValue>,
        new_view_info: Vec<u8>,
+        table_names: HashSet<TableName>,
    ) -> Result<()> {
-        let new_view_info_value = current_view_info_value.update(new_view_info);
+        let new_view_info_value = current_view_info_value.update(new_view_info, table_names);

        // Updates view info.
        let (update_view_info_txn, on_update_view_info_failure) = self
@@ -1174,7 +1199,7 @@ impl_optional_meta_value! {

 #[cfg(test)]
 mod tests {
-    use std::collections::{BTreeMap, HashMap};
+    use std::collections::{BTreeMap, HashMap, HashSet};
    use std::sync::Arc;

    use bytes::Bytes;
@@ -1183,6 +1208,7 @@ mod tests {
    use futures::TryStreamExt;
    use store_api::storage::RegionId;
    use table::metadata::{RawTableInfo, TableInfo};
+    use table::table_name::TableName;

    use super::datanode_table::DatanodeTableKey;
    use super::test_utils;
@@ -1197,7 +1223,6 @@ mod tests {
    use crate::kv_backend::memory::MemoryKvBackend;
    use crate::peer::Peer;
    use crate::rpc::router::{region_distribution, Region, RegionRoute, RegionStatus};
-    use crate::table_name::TableName;

    #[test]
    fn test_deserialized_value_with_bytes() {
@@ -1250,6 +1275,21 @@ mod tests {
        test_utils::new_test_table_info(10, region_numbers)
    }

+    fn new_test_table_names() -> HashSet<TableName> {
+        let mut set = HashSet::new();
+        set.insert(TableName {
+            catalog_name: "greptime".to_string(),
+            schema_name: "public".to_string(),
+            table_name: "a_table".to_string(),
+        });
+        set.insert(TableName {
+            catalog_name: "greptime".to_string(),
+            schema_name: "public".to_string(),
+            table_name: "b_table".to_string(),
+        });
+        set
+    }
+
    async fn create_physical_table_metadata(
        table_metadata_manager: &TableMetadataManager,
        table_info: RawTableInfo,
@@ -1961,9 +2001,11 @@ mod tests {

        let logical_plan: Vec<u8> = vec![1, 2, 3];

+        let table_names = new_test_table_names();
+
        // Create metadata
        table_metadata_manager
-            .create_view_metadata(view_info.clone(), &logical_plan)
+            .create_view_metadata(view_info.clone(), logical_plan.clone(), table_names.clone())
            .await
            .unwrap();

@@ -1977,6 +2019,7 @@ mod tests {
                .unwrap()
                .into_inner();
            assert_eq!(current_view_info.view_info, logical_plan);
+            assert_eq!(current_view_info.table_names, table_names);
            // assert table info
            let current_table_info = table_metadata_manager
                .table_info_manager()
@@ -1989,16 +2032,43 @@ mod tests {
        }

        let new_logical_plan: Vec<u8> = vec![4, 5, 6];
-        let current_view_info_value =
-            DeserializedValueWithBytes::from_inner(ViewInfoValue::new(&logical_plan));
+        let new_table_names = {
+            let mut set = HashSet::new();
+            set.insert(TableName {
+                catalog_name: "greptime".to_string(),
+                schema_name: "public".to_string(),
+                table_name: "b_table".to_string(),
+            });
+            set.insert(TableName {
+                catalog_name: "greptime".to_string(),
+                schema_name: "public".to_string(),
+                table_name: "c_table".to_string(),
+            });
+            set
+        };
+
+        let current_view_info_value = DeserializedValueWithBytes::from_inner(ViewInfoValue::new(
+            logical_plan.clone(),
+            table_names,
+        ));
        // should be ok.
        table_metadata_manager
-            .update_view_info(view_id, &current_view_info_value, new_logical_plan.clone())
+            .update_view_info(
+                view_id,
+                &current_view_info_value,
+                new_logical_plan.clone(),
+                new_table_names.clone(),
+            )
            .await
            .unwrap();
        // if table info was updated, it should be ok.
        table_metadata_manager
-            .update_view_info(view_id, &current_view_info_value, new_logical_plan.clone())
+            .update_view_info(
+                view_id,
+                &current_view_info_value,
+                new_logical_plan.clone(),
+                new_table_names.clone(),
+            )
            .await
            .unwrap();

@@ -2011,14 +2081,21 @@ mod tests {
            .unwrap()
            .into_inner();
        assert_eq!(updated_view_info.view_info, new_logical_plan);
+        assert_eq!(updated_view_info.table_names, new_table_names);

        let wrong_view_info = logical_plan.clone();
-        let wrong_view_info_value =
-            DeserializedValueWithBytes::from_inner(current_view_info_value.update(wrong_view_info));
+        let wrong_view_info_value = DeserializedValueWithBytes::from_inner(
+            current_view_info_value.update(wrong_view_info, new_table_names.clone()),
+        );
        // if the current_view_info_value is wrong, it should return an error.
        // The ABA problem.
        assert!(table_metadata_manager
-            .update_view_info(view_id, &wrong_view_info_value, new_logical_plan.clone())
+            .update_view_info(
+                view_id,
+                &wrong_view_info_value,
+                new_logical_plan.clone(),
+                new_table_names.clone(),
+            )
            .await
            .is_err());

@@ -2031,5 +2108,6 @@ mod tests {
            .unwrap()
            .into_inner();
        assert_eq!(current_view_info.view_info, new_logical_plan);
+        assert_eq!(current_view_info.table_names, new_table_names);
    }
 }
--- a/src/common/meta/src/key/datanode_table.rs
+++ b/src/common/meta/src/key/datanode_table.rs
@@ -72,12 +72,8 @@ impl DatanodeTableKey {
        }
    }

-    fn prefix(datanode_id: DatanodeId) -> String {
-        format!("{}/{datanode_id}", DATANODE_TABLE_KEY_PREFIX)
-    }
-
-    pub fn range_start_key(datanode_id: DatanodeId) -> String {
-        format!("{}/", Self::prefix(datanode_id))
+    pub fn prefix(datanode_id: DatanodeId) -> String {
+        format!("{}/{datanode_id}/", DATANODE_TABLE_KEY_PREFIX)
    }
 }

@@ -114,7 +110,7 @@ impl<'a> MetaKey<'a, DatanodeTableKey> for DatanodeTableKey {

 impl Display for DatanodeTableKey {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}/{}", Self::prefix(self.datanode_id), self.table_id)
+        write!(f, "{}{}", Self::prefix(self.datanode_id), self.table_id)
    }
 }

@@ -164,7 +160,7 @@ impl DatanodeTableManager {
        &self,
        datanode_id: DatanodeId,
    ) -> BoxStream<'static, Result<DatanodeTableValue>> {
-        let start_key = DatanodeTableKey::range_start_key(datanode_id);
+        let start_key = DatanodeTableKey::prefix(datanode_id);
        let req = RangeRequest::new().with_prefix(start_key.as_bytes());

        let stream = PaginationStream::new(
--- a/src/common/meta/src/key/flow.rs
+++ b/src/common/meta/src/key/flow.rs
@@ -262,12 +262,12 @@ mod tests {

    use futures::TryStreamExt;
    use table::metadata::TableId;
+    use table::table_name::TableName;

    use super::*;
    use crate::key::flow::table_flow::TableFlowKey;
    use crate::key::FlowPartitionId;
    use crate::kv_backend::memory::MemoryKvBackend;
-    use crate::table_name::TableName;
    use crate::FlownodeId;

    #[derive(Debug)]
--- a/src/common/meta/src/key/flow/flow_info.rs
+++ b/src/common/meta/src/key/flow/flow_info.rs
@@ -20,6 +20,7 @@ use regex::Regex;
 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use table::metadata::TableId;
+use table::table_name::TableName;

 use crate::error::{self, Result};
 use crate::key::flow::FlowScoped;
@@ -27,7 +28,6 @@ use crate::key::txn_helper::TxnOpGetResponseSet;
 use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId, MetaKey, TableMetaValue};
 use crate::kv_backend::txn::Txn;
 use crate::kv_backend::KvBackendRef;
-use crate::table_name::TableName;
 use crate::FlownodeId;

 const FLOW_INFO_KEY_PREFIX: &str = "info";
--- a/src/common/meta/src/key/flow/flownode_flow.rs
+++ b/src/common/meta/src/key/flow/flownode_flow.rs
@@ -69,8 +69,7 @@ impl FlownodeFlowKey {

    /// The prefix used to retrieve all [FlownodeFlowKey]s with the specified `flownode_id`.
    pub fn range_start_key(flownode_id: FlownodeId) -> Vec<u8> {
-        let inner =
-            BytesAdapter::from(FlownodeFlowKeyInner::range_start_key(flownode_id).into_bytes());
+        let inner = BytesAdapter::from(FlownodeFlowKeyInner::prefix(flownode_id).into_bytes());

        FlowScoped::new(inner).to_bytes()
    }
@@ -108,13 +107,8 @@ impl FlownodeFlowKeyInner {
        }
    }

-    fn prefix(flownode_id: FlownodeId) -> String {
-        format!("{}/{flownode_id}", FLOWNODE_FLOW_KEY_PREFIX)
-    }
-
-    /// The prefix used to retrieve all [FlownodeFlowKey]s with the specified `flownode_id`.
-    fn range_start_key(flownode_id: FlownodeId) -> String {
-        format!("{}/", Self::prefix(flownode_id))
+    pub fn prefix(flownode_id: FlownodeId) -> String {
+        format!("{}/{flownode_id}/", FLOWNODE_FLOW_KEY_PREFIX)
    }
 }

--- a/src/common/meta/src/key/flow/table_flow.rs
+++ b/src/common/meta/src/key/flow/table_flow.rs
@@ -80,7 +80,7 @@ impl TableFlowKey {

    /// The prefix used to retrieve all [TableFlowKey]s with the specified `table_id`.
    pub fn range_start_key(table_id: TableId) -> Vec<u8> {
-        let inner = BytesAdapter::from(TableFlowKeyInner::range_start_key(table_id).into_bytes());
+        let inner = BytesAdapter::from(TableFlowKeyInner::prefix(table_id).into_bytes());

        FlowScoped::new(inner).to_bytes()
    }
@@ -123,12 +123,7 @@ impl TableFlowKeyInner {
    }

    fn prefix(table_id: TableId) -> String {
-        format!("{}/{table_id}", TABLE_FLOW_KEY_PREFIX)
-    }
-
-    /// The prefix used to retrieve all [TableFlowKey]s with the specified `table_id`.
-    fn range_start_key(table_id: TableId) -> String {
-        format!("{}/", Self::prefix(table_id))
+        format!("{}/{table_id}/", TABLE_FLOW_KEY_PREFIX)
    }
 }

--- a/src/common/meta/src/key/table_info.rs
+++ b/src/common/meta/src/key/table_info.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use table::metadata::{RawTableInfo, TableId};
+use table::table_name::TableName;
 use table::table_reference::TableReference;

 use super::TABLE_INFO_KEY_PATTERN;
@@ -28,7 +29,6 @@ use crate::key::{DeserializedValueWithBytes, MetaKey, TableMetaValue, TABLE_INFO
 use crate::kv_backend::txn::Txn;
 use crate::kv_backend::KvBackendRef;
 use crate::rpc::store::BatchGetRequest;
-use crate::table_name::TableName;

 /// The key stores the metadata of the table.
 ///
--- a/src/common/meta/src/key/table_name.rs
+++ b/src/common/meta/src/key/table_name.rs
@@ -20,6 +20,7 @@ use futures_util::stream::BoxStream;
 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use table::metadata::TableId;
+use table::table_name::TableName;

 use super::{MetaKey, TableMetaValue, TABLE_NAME_KEY_PATTERN, TABLE_NAME_KEY_PREFIX};
 use crate::error::{Error, InvalidTableMetadataSnafu, Result};
@@ -29,7 +30,6 @@ use crate::kv_backend::KvBackendRef;
 use crate::range_stream::{PaginationStream, DEFAULT_PAGE_SIZE};
 use crate::rpc::store::{BatchGetRequest, RangeRequest};
 use crate::rpc::KeyValue;
-use crate::table_name::TableName;

 #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
 pub struct TableNameKey<'a> {
@@ -48,7 +48,7 @@ impl<'a> TableNameKey<'a> {
    }

    pub fn prefix_to_table(catalog: &str, schema: &str) -> String {
-        format!("{}/{}/{}", TABLE_NAME_KEY_PREFIX, catalog, schema)
+        format!("{}/{}/{}/", TABLE_NAME_KEY_PREFIX, catalog, schema)
    }
 }

@@ -56,7 +56,7 @@ impl Display for TableNameKey<'_> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
-            "{}/{}",
+            "{}{}",
            Self::prefix_to_table(self.catalog, self.schema),
            self.table
        )
@@ -268,7 +268,11 @@ impl TableNameManager {
 #[cfg(test)]
 mod tests {

+    use futures::StreamExt;
+
    use super::*;
+    use crate::kv_backend::KvBackend;
+    use crate::rpc::store::PutRequest;

    #[test]
    fn test_strip_table_name() {
@@ -324,4 +328,39 @@ mod tests {
        assert_eq!(value.try_as_raw_value().unwrap(), literal);
        assert_eq!(TableNameValue::try_from_raw_value(literal).unwrap(), value);
    }
+
+    #[tokio::test]
+    async fn test_prefix_scan_tables() {
+        let memory_kv = Arc::new(MemoryKvBackend::<crate::error::Error>::new());
+        memory_kv
+            .put(PutRequest {
+                key: TableNameKey {
+                    catalog: "greptime",
+                    schema: "👉",
+                    table: "t",
+                }
+                .to_bytes(),
+                value: vec![],
+                prev_kv: false,
+            })
+            .await
+            .unwrap();
+        memory_kv
+            .put(PutRequest {
+                key: TableNameKey {
+                    catalog: "greptime",
+                    schema: "👉👈",
+                    table: "t",
+                }
+                .to_bytes(),
+                value: vec![],
+                prev_kv: false,
+            })
+            .await
+            .unwrap();
+
+        let manager = TableNameManager::new(memory_kv);
+        let items = manager.tables("greptime", "👉").collect::<Vec<_>>().await;
+        assert_eq!(items.len(), 1);
+    }
 }
--- a/src/common/meta/src/key/table_region.rs
+++ b/src/common/meta/src/key/table_region.rs
@@ -1,130 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::BTreeMap;
-use std::fmt::Display;
-
-use lazy_static::lazy_static;
-use regex::Regex;
-use serde::{Deserialize, Serialize};
-use snafu::{OptionExt, ResultExt};
-use store_api::storage::RegionNumber;
-use table::metadata::TableId;
-
-use super::{MetaKey, TABLE_REGION_KEY_PREFIX};
-use crate::error::{InvalidTableMetadataSnafu, Result, SerdeJsonSnafu};
-use crate::{impl_table_meta_value, DatanodeId};
-
-pub type RegionDistribution = BTreeMap<DatanodeId, Vec<RegionNumber>>;
-
-#[deprecated(
-    since = "0.4.0",
-    note = "Please use the TableRouteManager's get_region_distribution method instead"
-)]
-#[derive(Debug, PartialEq)]
-pub struct TableRegionKey {
-    table_id: TableId,
-}
-
-lazy_static! {
-    static ref TABLE_REGION_KEY_PATTERN: Regex =
-        Regex::new(&format!("^{TABLE_REGION_KEY_PREFIX}/([0-9]+)$")).unwrap();
-}
-
-impl TableRegionKey {
-    pub fn new(table_id: TableId) -> Self {
-        Self { table_id }
-    }
-}
-
-impl Display for TableRegionKey {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}/{}", TABLE_REGION_KEY_PREFIX, self.table_id)
-    }
-}
-
-impl<'a> MetaKey<'a, TableRegionKey> for TableRegionKey {
-    fn to_bytes(&self) -> Vec<u8> {
-        self.to_string().into_bytes()
-    }
-
-    fn from_bytes(bytes: &'a [u8]) -> Result<TableRegionKey> {
-        let key = std::str::from_utf8(bytes).map_err(|e| {
-            InvalidTableMetadataSnafu {
-                err_msg: format!(
-                    "TableRegionKey '{}' is not a valid UTF8 string: {e}",
-                    String::from_utf8_lossy(bytes)
-                ),
-            }
-            .build()
-        })?;
-        let captures =
-            TABLE_REGION_KEY_PATTERN
-                .captures(key)
-                .context(InvalidTableMetadataSnafu {
-                    err_msg: format!("Invalid TableRegionKey '{key}'"),
-                })?;
-        // Safety: pass the regex check above
-        let table_id = captures[1].parse::<TableId>().unwrap();
-        Ok(TableRegionKey { table_id })
-    }
-}
-
-#[deprecated(
-    since = "0.4.0",
-    note = "Please use the TableRouteManager's get_region_distribution method instead"
-)]
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
-pub struct TableRegionValue {
-    pub region_distribution: RegionDistribution,
-    version: u64,
-}
-
-impl TableRegionValue {
-    pub fn new(region_distribution: RegionDistribution) -> Self {
-        Self {
-            region_distribution,
-            version: 0,
-        }
-    }
-}
-
-impl_table_meta_value! {TableRegionValue}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::key::TableMetaValue;
-
-    #[test]
-    fn test_serialization() {
-        let key = TableRegionKey::new(24);
-        let raw_key = key.to_bytes();
-        assert_eq!(raw_key, b"__table_region/24");
-        let deserialized = TableRegionKey::from_bytes(b"__table_region/24").unwrap();
-        assert_eq!(key, deserialized);
-
-        let value = TableRegionValue {
-            region_distribution: RegionDistribution::from([(1, vec![1, 2, 3]), (2, vec![4, 5, 6])]),
-            version: 0,
-        };
-        let literal = br#"{"region_distribution":{"1":[1,2,3],"2":[4,5,6]},"version":0}"#;
-
-        assert_eq!(value.try_as_raw_value().unwrap(), literal);
-        assert_eq!(
-            TableRegionValue::try_from_raw_value(literal).unwrap(),
-            value,
-        );
-    }
-}
--- a/src/common/meta/src/key/view_info.rs
+++ b/src/common/meta/src/key/view_info.rs
@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
+use std::sync::Arc;

 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use table::metadata::TableId;
+use table::table_name::TableName;

 use super::VIEW_INFO_KEY_PATTERN;
 use crate::error::{InvalidViewInfoSnafu, Result};
@@ -80,21 +82,30 @@ impl<'a> MetaKey<'a, ViewInfoKey> for ViewInfoKey {
 /// The VIEW info value that keeps the metadata.
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct ViewInfoValue {
+    /// The encoded logical plan
    pub view_info: RawViewLogicalPlan,
+    /// The resolved fully table names in logical plan
+    pub table_names: HashSet<TableName>,
    version: u64,
 }

 impl ViewInfoValue {
-    pub fn new(view_info: &RawViewLogicalPlan) -> Self {
+    pub fn new(view_info: RawViewLogicalPlan, table_names: HashSet<TableName>) -> Self {
        Self {
-            view_info: view_info.clone(),
+            view_info,
+            table_names,
            version: 0,
        }
    }

-    pub(crate) fn update(&self, new_view_info: RawViewLogicalPlan) -> Self {
+    pub(crate) fn update(
+        &self,
+        new_view_info: RawViewLogicalPlan,
+        table_names: HashSet<TableName>,
+    ) -> Self {
        Self {
            view_info: new_view_info,
+            table_names,
            version: self.version + 1,
        }
    }
@@ -105,6 +116,8 @@ pub struct ViewInfoManager {
    kv_backend: KvBackendRef,
 }

+pub type ViewInfoManagerRef = Arc<ViewInfoManager>;
+
 impl ViewInfoManager {
    pub fn new(kv_backend: KvBackendRef) -> Self {
        Self { kv_backend }
@@ -254,9 +267,25 @@ mod tests {

    #[test]
    fn test_value_serialization() {
+        let table_names = {
+            let mut set = HashSet::new();
+            set.insert(TableName {
+                catalog_name: "greptime".to_string(),
+                schema_name: "public".to_string(),
+                table_name: "a_table".to_string(),
+            });
+            set.insert(TableName {
+                catalog_name: "greptime".to_string(),
+                schema_name: "public".to_string(),
+                table_name: "b_table".to_string(),
+            });
+            set
+        };
+
        let value = ViewInfoValue {
            view_info: vec![1, 2, 3],
            version: 1,
+            table_names,
        };
        let serialized = value.try_as_raw_value().unwrap();
        let deserialized = ViewInfoValue::try_from_raw_value(&serialized).unwrap();
--- a/src/common/meta/src/lib.rs
+++ b/src/common/meta/src/lib.rs
@@ -40,7 +40,6 @@ pub mod region_keeper;
 pub mod rpc;
 pub mod sequence;
 pub mod state_store;
-pub mod table_name;
 #[cfg(any(test, feature = "testing"))]
 pub mod test_util;
 pub mod util;
--- a/src/common/meta/src/rpc/ddl.rs
+++ b/src/common/meta/src/rpc/ddl.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::result;

 use api::v1::meta::ddl_task_request::Task;
@@ -39,11 +39,11 @@ use serde_with::{serde_as, DefaultOnNull};
 use session::context::QueryContextRef;
 use snafu::{OptionExt, ResultExt};
 use table::metadata::{RawTableInfo, TableId};
+use table::table_name::TableName;
 use table::table_reference::TableReference;

 use crate::error::{self, Result};
 use crate::key::FlowId;
-use crate::table_name::TableName;

 /// DDL tasks
 #[derive(Debug, Clone)]
@@ -274,10 +274,7 @@ impl TryFrom<SubmitDdlTaskRequest> for PbDdlTaskRequest {
 #[derive(Debug, Default)]
 pub struct SubmitDdlTaskResponse {
    pub key: Vec<u8>,
-    // For create physical table
-    // TODO(jeremy): remove it?
-    pub table_id: Option<TableId>,
-    // For create multi logical tables
+    // `table_id`s for `CREATE TABLE` or `CREATE LOGICAL TABLES` task.
    pub table_ids: Vec<TableId>,
 }

@@ -285,11 +282,9 @@ impl TryFrom<PbDdlTaskResponse> for SubmitDdlTaskResponse {
    type Error = error::Error;

    fn try_from(resp: PbDdlTaskResponse) -> Result<Self> {
-        let table_id = resp.table_id.map(|t| t.id);
        let table_ids = resp.table_ids.into_iter().map(|t| t.id).collect();
        Ok(Self {
            key: resp.pid.map(|pid| pid.key).unwrap_or_default(),
-            table_id,
            table_ids,
        })
    }
@@ -299,9 +294,6 @@ impl From<SubmitDdlTaskResponse> for PbDdlTaskResponse {
    fn from(val: SubmitDdlTaskResponse) -> Self {
        Self {
            pid: Some(ProcedureId { key: val.key }),
-            table_id: val
-                .table_id
-                .map(|table_id| api::v1::TableId { id: table_id }),
            table_ids: val
                .table_ids
                .into_iter()
@@ -332,6 +324,14 @@ impl CreateViewTask {
    pub fn raw_logical_plan(&self) -> &Vec<u8> {
        &self.create_view.logical_plan
    }
+
+    pub fn table_names(&self) -> HashSet<TableName> {
+        self.create_view
+            .table_names
+            .iter()
+            .map(|t| t.clone().into())
+            .collect()
+    }
 }

 impl TryFrom<PbCreateViewTask> for CreateViewTask {
--- a/src/common/meta/src/rpc/router.rs
+++ b/src/common/meta/src/rpc/router.rs
@@ -25,11 +25,11 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use snafu::OptionExt;
 use store_api::storage::{RegionId, RegionNumber};
 use strum::AsRefStr;
+use table::table_name::TableName;

 use crate::error::{self, Result};
 use crate::key::RegionDistribution;
 use crate::peer::Peer;
-use crate::table_name::TableName;
 use crate::DatanodeId;

 pub fn region_distribution(region_routes: &[RegionRoute]) -> RegionDistribution {
--- a/src/common/query/Cargo.toml
+++ b/src/common/query/Cargo.toml
@@ -4,12 +4,16 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true

+[features]
+testing = []
+
 [lints]
 workspace = true

 [dependencies]
 api.workspace = true
 async-trait.workspace = true
+bytes.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 common-recordbatch.workspace = true
--- a/src/common/query/src/error.rs
+++ b/src/common/query/src/error.rs
@@ -206,6 +206,13 @@ pub enum Error {
        location: Location,
    },

+    #[snafu(display("Failed to decode logical plan: {source}"))]
+    DecodePlan {
+        #[snafu(implicit)]
+        location: Location,
+        source: BoxedError,
+    },
+
    #[snafu(display("Failed to do table mutation"))]
    TableMutation {
        source: BoxedError,
@@ -282,11 +289,12 @@ impl ErrorExt for Error {
            | Error::InvalidFuncArgs { .. } => StatusCode::InvalidArguments,

            Error::ConvertDfRecordBatchStream { source, .. } => source.status_code(),
-            Error::ExecutePhysicalPlan { source, .. } => source.status_code(),
-            Error::Execute { source, .. } => source.status_code(),
-            Error::ProcedureService { source, .. } | Error::TableMutation { source, .. } => {
-                source.status_code()
-            }
+
+            Error::DecodePlan { source, .. }
+            | Error::Execute { source, .. }
+            | Error::ExecutePhysicalPlan { source, .. }
+            | Error::ProcedureService { source, .. }
+            | Error::TableMutation { source, .. } => source.status_code(),

            Error::PermissionDenied { .. } => StatusCode::PermissionDenied,
        }
--- a/src/common/query/src/lib.rs
+++ b/src/common/query/src/lib.rs
@@ -18,7 +18,8 @@ mod function;
 pub mod logical_plan;
 pub mod prelude;
 mod signature;
-
+#[cfg(any(test, feature = "testing"))]
+pub mod test_util;
 use std::fmt::{Debug, Display, Formatter};
 use std::sync::Arc;

--- a/src/common/query/src/logical_plan.rs
+++ b/src/common/query/src/logical_plan.rs
@@ -19,12 +19,15 @@ mod udf;

 use std::sync::Arc;

+use datafusion::catalog::CatalogProviderList;
+use datafusion::logical_expr::LogicalPlan;
 use datatypes::prelude::ConcreteDataType;
 pub use expr::build_filter_from_timestamp;

 pub use self::accumulator::{Accumulator, AggregateFunctionCreator, AggregateFunctionCreatorRef};
 pub use self::udaf::AggregateFunction;
 pub use self::udf::ScalarUdf;
+use crate::error::Result;
 use crate::function::{ReturnTypeFunction, ScalarFunctionImplementation};
 use crate::logical_plan::accumulator::*;
 use crate::signature::{Signature, Volatility};
@@ -68,6 +71,25 @@ pub fn create_aggregate_function(
    )
 }

+/// The datafusion `[LogicalPlan]` decoder.
+#[async_trait::async_trait]
+pub trait SubstraitPlanDecoder {
+    /// Decode the [`LogicalPlan`] from bytes with the [`CatalogProviderList`].
+    /// When `optimize` is true, it will do the optimization for decoded plan.
+    ///
+    /// TODO(dennis): It's not a good design for an API to do many things.
+    /// The `optimize` was introduced because of `query` and `catalog` cyclic dependency issue
+    /// I am happy to refactor it if we have a better solution.
+    async fn decode(
+        &self,
+        message: bytes::Bytes,
+        catalog_list: Arc<dyn CatalogProviderList>,
+        optimize: bool,
+    ) -> Result<LogicalPlan>;
+}
+
+pub type SubstraitPlanDecoderRef = Arc<dyn SubstraitPlanDecoder + Send + Sync>;
+
 #[cfg(test)]
 mod tests {
    use std::sync::Arc;
--- a/src/mito2/src/wal/wal_entry_reader.rs
+++ b/src/mito2/src/wal/wal_entry_reader.rs
@@ -12,13 +12,31 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use store_api::storage::RegionId;
+use std::sync::Arc;
+
+use datafusion::catalog::CatalogProviderList;
+use datafusion::logical_expr::LogicalPlan;

 use crate::error::Result;
-use crate::wal::raw_entry_reader::LogStoreNamespace;
-use crate::wal::{EntryId, WalEntryStream};
+use crate::logical_plan::SubstraitPlanDecoder;

-/// [OneshotWalEntryReader] provides the ability to read and decode entries from the underlying store.
-pub(crate) trait OneshotWalEntryReader: Send + Sync {
-    fn read(self, ctx: LogStoreNamespace, start_id: EntryId) -> Result<WalEntryStream>;
+/// Dummy `[SubstraitPlanDecoder]` for test.
+pub struct DummyDecoder;
+
+impl DummyDecoder {
+    pub fn arc() -> Arc<Self> {
+        Arc::new(DummyDecoder)
+    }
+}
+
+#[async_trait::async_trait]
+impl SubstraitPlanDecoder for DummyDecoder {
+    async fn decode(
+        &self,
+        _message: bytes::Bytes,
+        _catalog_list: Arc<dyn CatalogProviderList>,
+        _optimize: bool,
+    ) -> Result<LogicalPlan> {
+        unreachable!()
+    }
 }
--- a/src/common/substrait/Cargo.toml
+++ b/src/common/substrait/Cargo.toml
@@ -10,19 +10,15 @@ workspace = true
 [dependencies]
 async-trait.workspace = true
 bytes.workspace = true
-catalog.workspace = true
 common-error.workspace = true
-common-function.workspace = true
 common-macro.workspace = true
 common-telemetry.workspace = true
 datafusion.workspace = true
 datafusion-common.workspace = true
 datafusion-expr.workspace = true
 datafusion-substrait.workspace = true
-datatypes.workspace = true
 promql.workspace = true
 prost.workspace = true
-session.workspace = true
 snafu.workspace = true

 [dependencies.substrait_proto]
--- a/src/common/substrait/src/df_substrait.rs
+++ b/src/common/substrait/src/df_substrait.rs
@@ -16,26 +16,19 @@ use std::sync::Arc;

 use async_trait::async_trait;
 use bytes::{Buf, Bytes, BytesMut};
-use common_function::function_registry::FUNCTION_REGISTRY;
-use common_function::scalars::udf::create_udf;
 use datafusion::catalog::CatalogProviderList;
 use datafusion::execution::context::SessionState;
 use datafusion::execution::runtime_env::RuntimeEnv;
-use datafusion::execution::FunctionRegistry;
 use datafusion::prelude::{SessionConfig, SessionContext};
 use datafusion_expr::LogicalPlan;
 use datafusion_substrait::logical_plan::consumer::from_substrait_plan;
 use datafusion_substrait::logical_plan::producer::to_substrait_plan;
 use datafusion_substrait::substrait::proto::Plan;
 use prost::Message;
-use session::context::QueryContextRef;
 use snafu::ResultExt;

-use crate::error::{
-    DFInternalSnafu, DecodeDfPlanSnafu, DecodeRelSnafu, EncodeDfPlanSnafu, EncodeRelSnafu, Error,
-};
-use crate::extension_serializer::ExtensionSerializer;
-use crate::SubstraitPlan;
+use crate::error::{DecodeDfPlanSnafu, DecodeRelSnafu, EncodeDfPlanSnafu, EncodeRelSnafu, Error};
+use crate::{SerializerRegistry, SubstraitPlan};

 pub struct DFLogicalSubstraitConvertor;

@@ -49,15 +42,8 @@ impl SubstraitPlan for DFLogicalSubstraitConvertor {
        &self,
        message: B,
        catalog_list: Arc<dyn CatalogProviderList>,
-        mut state: SessionState,
-        query_ctx: QueryContextRef,
+        state: SessionState,
    ) -> Result<Self::Plan, Self::Error> {
-        // substrait decoder will look up the UDFs in SessionState, so we need to register them
-        for func in FUNCTION_REGISTRY.functions() {
-            let udf = Arc::new(create_udf(func, query_ctx.clone(), Default::default()).into());
-            state.register_udf(udf).context(DFInternalSnafu)?;
-        }
-
        let mut context = SessionContext::new_with_state(state);
        context.register_catalog_list(catalog_list);
        let plan = Plan::decode(message).context(DecodeRelSnafu)?;
@@ -67,10 +53,13 @@ impl SubstraitPlan for DFLogicalSubstraitConvertor {
        Ok(df_plan)
    }

-    fn encode(&self, plan: &Self::Plan) -> Result<Bytes, Self::Error> {
+    fn encode(
+        &self,
+        plan: &Self::Plan,
+        serializer: impl SerializerRegistry + 'static,
+    ) -> Result<Bytes, Self::Error> {
        let mut buf = BytesMut::new();
-
-        let substrait_plan = self.to_sub_plan(plan)?;
+        let substrait_plan = self.to_sub_plan(plan, serializer)?;
        substrait_plan.encode(&mut buf).context(EncodeRelSnafu)?;

        Ok(buf.freeze())
@@ -78,10 +67,14 @@ impl SubstraitPlan for DFLogicalSubstraitConvertor {
 }

 impl DFLogicalSubstraitConvertor {
-    pub fn to_sub_plan(&self, plan: &LogicalPlan) -> Result<Box<Plan>, Error> {
+    pub fn to_sub_plan(
+        &self,
+        plan: &LogicalPlan,
+        serializer: impl SerializerRegistry + 'static,
+    ) -> Result<Box<Plan>, Error> {
        let session_state =
            SessionState::new_with_config_rt(SessionConfig::new(), Arc::new(RuntimeEnv::default()))
-                .with_serializer_registry(Arc::new(ExtensionSerializer));
+                .with_serializer_registry(Arc::new(serializer));
        let context = SessionContext::new_with_state(session_state);

        to_substrait_plan(plan, &context).context(EncodeDfPlanSnafu)
--- a/src/common/substrait/src/error.rs
+++ b/src/common/substrait/src/error.rs
@@ -18,7 +18,6 @@ use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
-use datatypes::prelude::ConcreteDataType;
 use prost::{DecodeError, EncodeError};
 use snafu::{Location, Snafu};

@@ -26,34 +25,6 @@ use snafu::{Location, Snafu};
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Unsupported physical plan: {}", name))]
-    UnsupportedPlan {
-        name: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Unsupported expr: {}", name))]
-    UnsupportedExpr {
-        name: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Unsupported concrete type: {:?}", ty))]
-    UnsupportedConcreteType {
-        ty: ConcreteDataType,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Unsupported substrait type: {}", ty))]
-    UnsupportedSubstraitType {
-        ty: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
    #[snafu(display("Failed to decode substrait relation"))]
    DecodeRel {
        #[snafu(source)]
@@ -70,33 +41,6 @@ pub enum Error {
        location: Location,
    },

-    #[snafu(display("Input plan is empty"))]
-    EmptyPlan {
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Input expression is empty"))]
-    EmptyExpr {
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Missing required field in protobuf, field: {}, plan: {}", field, plan))]
-    MissingField {
-        field: String,
-        plan: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Invalid parameters: {}", reason))]
-    InvalidParameters {
-        reason: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
    #[snafu(display("Internal error from DataFusion"))]
    DFInternal {
        #[snafu(source)]
@@ -118,35 +62,6 @@ pub enum Error {
        location: Location,
    },

-    #[snafu(display(
-        "Schema from Substrait proto doesn't match with the schema in storage.
-        Substrait schema: {:?}
-        Storage schema: {:?}",
-        substrait_schema,
-        storage_schema
-    ))]
-    SchemaNotMatch {
-        substrait_schema: datafusion::arrow::datatypes::SchemaRef,
-        storage_schema: datafusion::arrow::datatypes::SchemaRef,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to convert DataFusion schema"))]
-    ConvertDfSchema {
-        #[snafu(implicit)]
-        location: Location,
-        source: datatypes::error::Error,
-    },
-
-    #[snafu(display("Unable to resolve table: {table_name}, error: "))]
-    ResolveTable {
-        table_name: String,
-        #[snafu(implicit)]
-        location: Location,
-        source: catalog::error::Error,
-    },
-
    #[snafu(display("Failed to encode DataFusion plan"))]
    EncodeDfPlan {
        #[snafu(source)]
@@ -169,24 +84,13 @@ pub type Result<T> = std::result::Result<T, Error>;
 impl ErrorExt for Error {
    fn status_code(&self) -> StatusCode {
        match self {
-            Error::UnsupportedConcreteType { .. }
-            | Error::UnsupportedPlan { .. }
-            | Error::UnsupportedExpr { .. }
-            | Error::UnsupportedSubstraitType { .. } => StatusCode::Unsupported,
-            Error::UnknownPlan { .. }
-            | Error::EncodeRel { .. }
-            | Error::DecodeRel { .. }
-            | Error::EmptyPlan { .. }
-            | Error::EmptyExpr { .. }
-            | Error::MissingField { .. }
-            | Error::InvalidParameters { .. }
-            | Error::SchemaNotMatch { .. } => StatusCode::InvalidArguments,
+            Error::UnknownPlan { .. } | Error::EncodeRel { .. } | Error::DecodeRel { .. } => {
+                StatusCode::InvalidArguments
+            }
            Error::DFInternal { .. }
            | Error::Internal { .. }
            | Error::EncodeDfPlan { .. }
            | Error::DecodeDfPlan { .. } => StatusCode::Internal,
-            Error::ConvertDfSchema { source, .. } => source.status_code(),
-            Error::ResolveTable { source, .. } => source.status_code(),
        }
    }

--- a/src/common/substrait/src/extension_serializer.rs
+++ b/src/common/substrait/src/extension_serializer.rs
@@ -67,7 +67,6 @@ impl SerializerRegistry for ExtensionSerializer {
            name if name == EmptyMetric::name() => Err(DataFusionError::Substrait(
                "EmptyMetric should not be serialized".to_string(),
            )),
-            "MergeScan" => Ok(vec![]),
            other => Err(DataFusionError::NotImplemented(format!(
                "Serizlize logical plan for {}",
                other
--- a/src/common/substrait/src/lib.rs
+++ b/src/common/substrait/src/lib.rs
@@ -23,11 +23,11 @@ use async_trait::async_trait;
 use bytes::{Buf, Bytes};
 use datafusion::catalog::CatalogProviderList;
 use datafusion::execution::context::SessionState;
+pub use datafusion::execution::registry::SerializerRegistry;
 /// Re-export the Substrait module of datafusion,
 /// note this is a different version of the `substrait_proto` crate
 pub use datafusion_substrait::substrait as substrait_proto_df;
 pub use datafusion_substrait::{logical_plan as df_logical_plan, variation_const};
-use session::context::QueryContextRef;
 pub use substrait_proto;

 pub use crate::df_substrait::DFLogicalSubstraitConvertor;
@@ -42,8 +42,11 @@ pub trait SubstraitPlan {
        message: B,
        catalog_list: Arc<dyn CatalogProviderList>,
        state: SessionState,
-        query_ctx: QueryContextRef,
    ) -> Result<Self::Plan, Self::Error>;

-    fn encode(&self, plan: &Self::Plan) -> Result<Bytes, Self::Error>;
+    fn encode(
+        &self,
+        plan: &Self::Plan,
+        serializer: impl SerializerRegistry + 'static,
+    ) -> Result<Bytes, Self::Error>;
 }
--- a/src/common/telemetry/src/logging.rs
+++ b/src/common/telemetry/src/logging.rs
@@ -94,7 +94,7 @@ pub fn init_default_ut_logging() {
            env::var("UNITTEST_LOG_DIR").unwrap_or_else(|_| "/tmp/__unittest_logs".to_string());

        let level = env::var("UNITTEST_LOG_LEVEL").unwrap_or_else(|_|
-            "debug,hyper=warn,tower=warn,datafusion=warn,reqwest=warn,sqlparser=warn,h2=info,opendal=info".to_string()
+            "debug,hyper=warn,tower=warn,datafusion=warn,reqwest=warn,sqlparser=warn,h2=info,opendal=info,rskafka=info".to_string()
        );
        let opts = LoggingOptions {
            dir: dir.clone(),
--- a/src/datanode/Cargo.toml
+++ b/src/datanode/Cargo.toml
@@ -57,7 +57,6 @@ servers.workspace = true
 session.workspace = true
 snafu.workspace = true
 store-api.workspace = true
-substrait.workspace = true
 table.workspace = true
 tokio.workspace = true
 toml.workspace = true
--- a/src/datanode/src/error.rs
+++ b/src/datanode/src/error.rs
@@ -64,11 +64,18 @@ pub enum Error {
        source: query::error::Error,
    },

+    #[snafu(display("Failed to create plan decoder"))]
+    NewPlanDecoder {
+        #[snafu(implicit)]
+        location: Location,
+        source: query::error::Error,
+    },
+
    #[snafu(display("Failed to decode logical plan"))]
    DecodeLogicalPlan {
        #[snafu(implicit)]
        location: Location,
-        source: substrait::error::Error,
+        source: common_query::error::Error,
    },

    #[snafu(display("Incorrect internal state: {}", state))]
@@ -388,7 +395,9 @@ impl ErrorExt for Error {
    fn status_code(&self) -> StatusCode {
        use Error::*;
        match self {
-            ExecuteLogicalPlan { source, .. } => source.status_code(),
+            NewPlanDecoder { source, .. } | ExecuteLogicalPlan { source, .. } => {
+                source.status_code()
+            }

            BuildRegionRequests { source, .. } => source.status_code(),
            HandleHeartbeatResponse { source, .. } | GetMetadata { source, .. } => {
--- a/src/datanode/src/event_listener.rs
+++ b/src/datanode/src/event_listener.rs
@@ -41,19 +41,13 @@ pub struct RegionServerEventSender(pub(crate) UnboundedSender<RegionServerEvent>
 impl RegionServerEventListener for RegionServerEventSender {
    fn on_region_registered(&self, region_id: RegionId) {
        if let Err(e) = self.0.send(RegionServerEvent::Registered(region_id)) {
-            error!(
-                "Failed to send registering region: {region_id} event, source: {}",
-                e
-            );
+            error!(e; "Failed to send registering region: {region_id} event");
        }
    }

    fn on_region_deregistered(&self, region_id: RegionId) {
        if let Err(e) = self.0.send(RegionServerEvent::Deregistered(region_id)) {
-            error!(
-                "Failed to send deregistering region: {region_id} event, source: {}",
-                e
-            );
+            error!(e; "Failed to send deregistering region: {region_id} event");
        }
    }
 }
--- a/src/datanode/src/heartbeat.rs
+++ b/src/datanode/src/heartbeat.rs
@@ -213,6 +213,7 @@ impl HeartbeatTask {
        let epoch = self.region_alive_keeper.epoch();

        self.region_alive_keeper.start(Some(event_receiver)).await?;
+        let mut last_sent = Instant::now();

        common_runtime::spawn_bg(async move {
            let sleep = tokio::time::sleep(Duration::from_millis(0));
@@ -271,6 +272,10 @@ impl HeartbeatTask {
                    }
                };
                if let Some(req) = req {
+                    metrics::LAST_SENT_HEARTBEAT_ELAPSED
+                        .set(last_sent.elapsed().as_millis() as i64);
+                    // Resets the timer.
+                    last_sent = Instant::now();
                    debug!("Sending heartbeat request: {:?}", req);
                    if let Err(e) = tx.send(req).await {
                        error!(e; "Failed to send heartbeat to metasrv");
--- a/src/datanode/src/metrics.rs
+++ b/src/datanode/src/metrics.rs
@@ -35,6 +35,12 @@ lazy_static! {
        "last received heartbeat lease elapsed",
    )
    .unwrap();
+    /// The elapsed time since the last sent heartbeat.
+    pub static ref LAST_SENT_HEARTBEAT_ELAPSED: IntGauge = register_int_gauge!(
+        "greptime_last_sent_heartbeat_lease_elapsed",
+        "last sent heartbeat lease elapsed",
+    )
+    .unwrap();
    pub static ref LEASE_EXPIRED_REGION: IntGaugeVec = register_int_gauge_vec!(
        "greptime_lease_expired_region",
        "lease expired region",
--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -51,13 +51,13 @@ use store_api::metric_engine_consts::{
 use store_api::region_engine::{RegionEngineRef, RegionRole, SetReadonlyResponse};
 use store_api::region_request::{AffectedRows, RegionCloseRequest, RegionRequest};
 use store_api::storage::RegionId;
-use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 use tonic::{Request, Response, Result as TonicResult};

 use crate::error::{
    self, BuildRegionRequestsSnafu, DecodeLogicalPlanSnafu, ExecuteLogicalPlanSnafu,
-    FindLogicalRegionsSnafu, HandleRegionRequestSnafu, RegionEngineNotFoundSnafu,
-    RegionNotFoundSnafu, Result, StopRegionEngineSnafu, UnexpectedSnafu, UnsupportedOutputSnafu,
+    FindLogicalRegionsSnafu, HandleRegionRequestSnafu, NewPlanDecoderSnafu,
+    RegionEngineNotFoundSnafu, RegionNotFoundSnafu, Result, StopRegionEngineSnafu, UnexpectedSnafu,
+    UnsupportedOutputSnafu,
 };
 use crate::event_listener::RegionServerEventListenerRef;

@@ -189,7 +189,7 @@ impl RegionServer {

    pub async fn region_disk_usage(&self, region_id: RegionId) -> Option<i64> {
        match self.inner.region_map.get(&region_id) {
-            Some(e) => e.region_disk_usage(region_id).await,
+            Some(e) => e.region_disk_usage(region_id),
            None => None,
        }
    }
@@ -409,9 +409,7 @@ impl RegionServerInner {
        let engine = match region_change {
            RegionChange::Register(attribute) => match current_region_status {
                Some(status) => match status.clone() {
-                    RegionEngineWithStatus::Registering(_) => {
-                        return Ok(CurrentEngine::EarlyReturn(0))
-                    }
+                    RegionEngineWithStatus::Registering(engine) => engine,
                    RegionEngineWithStatus::Deregistering(_) => {
                        return error::RegionBusySnafu { region_id }.fail()
                    }
@@ -655,14 +653,13 @@ impl RegionServerInner {

        let catalog_list = Arc::new(DummyCatalogList::with_table_provider(table_provider));
        let query_engine_ctx = self.query_engine.engine_context(ctx.clone());
+        let plan_decoder = query_engine_ctx
+            .new_plan_decoder()
+            .context(NewPlanDecoderSnafu)?;
+
        // decode substrait plan to logical plan and execute it
-        let logical_plan = DFLogicalSubstraitConvertor
-            .decode(
-                Bytes::from(plan),
-                catalog_list,
-                query_engine_ctx.state().clone(),
-                ctx.clone(),
-            )
+        let logical_plan = plan_decoder
+            .decode(Bytes::from(plan), catalog_list, false)
            .await
            .context(DecodeLogicalPlanSnafu)?;

@@ -781,34 +778,32 @@ mod tests {
        let mut mock_region_server = mock_region_server();
        let (engine, _receiver) = MockRegionEngine::new(MITO_ENGINE_NAME);
        let engine_name = engine.name();
-
        mock_region_server.register_engine(engine.clone());
-
        let region_id = RegionId::new(1, 1);
        let builder = CreateRequestBuilder::new();
        let create_req = builder.build();
-
        // Tries to create/open a registering region.
        mock_region_server.inner.region_map.insert(
            region_id,
            RegionEngineWithStatus::Registering(engine.clone()),
        );
-
        let response = mock_region_server
            .handle_request(region_id, RegionRequest::Create(create_req))
            .await
            .unwrap();
        assert_eq!(response.affected_rows, 0);
-
        let status = mock_region_server
            .inner
            .region_map
            .get(&region_id)
            .unwrap()
            .clone();
+        assert!(matches!(status, RegionEngineWithStatus::Ready(_)));

-        assert!(matches!(status, RegionEngineWithStatus::Registering(_)));
-
+        mock_region_server.inner.region_map.insert(
+            region_id,
+            RegionEngineWithStatus::Registering(engine.clone()),
+        );
        let response = mock_region_server
            .handle_request(
                region_id,
@@ -822,14 +817,13 @@ mod tests {
            .await
            .unwrap();
        assert_eq!(response.affected_rows, 0);
-
        let status = mock_region_server
            .inner
            .region_map
            .get(&region_id)
            .unwrap()
            .clone();
-        assert!(matches!(status, RegionEngineWithStatus::Registering(_)));
+        assert!(matches!(status, RegionEngineWithStatus::Ready(_)));
    }

    #[tokio::test]
@@ -1020,7 +1014,7 @@ mod tests {
                region_change: RegionChange::Register(RegionAttribute::Mito),
                assert: Box::new(|result| {
                    let current_engine = result.unwrap();
-                    assert_matches!(current_engine, CurrentEngine::EarlyReturn(_));
+                    assert_matches!(current_engine, CurrentEngine::Engine(_));
                }),
            },
            CurrentEngineTest {
--- a/src/datanode/src/tests.rs
+++ b/src/datanode/src/tests.rs
@@ -200,7 +200,7 @@ impl RegionEngine for MockRegionEngine {
        unimplemented!()
    }

-    async fn region_disk_usage(&self, _region_id: RegionId) -> Option<i64> {
+    fn region_disk_usage(&self, _region_id: RegionId) -> Option<i64> {
        unimplemented!()
    }

--- a/src/file-engine/src/engine.rs
+++ b/src/file-engine/src/engine.rs
@@ -107,7 +107,7 @@ impl RegionEngine for FileRegionEngine {
        self.inner.stop().await.map_err(BoxedError::new)
    }

-    async fn region_disk_usage(&self, _: RegionId) -> Option<i64> {
+    fn region_disk_usage(&self, _: RegionId) -> Option<i64> {
        None
    }

@@ -229,8 +229,9 @@ impl EngineInner {
        let res = FileRegion::create(region_id, request, &self.object_store).await;
        let region = res.inspect_err(|err| {
            error!(
-                "Failed to create region, region_id: {}, err: {}",
-                region_id, err
+                err;
+                "Failed to create region, region_id: {}",
+                region_id
            );
        })?;
        self.regions.write().unwrap().insert(region_id, region);
@@ -259,8 +260,9 @@ impl EngineInner {
        let res = FileRegion::open(region_id, request, &self.object_store).await;
        let region = res.inspect_err(|err| {
            error!(
-                "Failed to open region, region_id: {}, err: {}",
-                region_id, err
+                err;
+                "Failed to open region, region_id: {}",
+                region_id
            );
        })?;
        self.regions.write().unwrap().insert(region_id, region);
@@ -302,8 +304,9 @@ impl EngineInner {
            let res = FileRegion::drop(&region, &self.object_store).await;
            res.inspect_err(|err| {
                error!(
-                    "Failed to drop region, region_id: {}, err: {}",
-                    region_id, err
+                    err;
+                    "Failed to drop region, region_id: {}",
+                    region_id
                );
            })?;
        }
--- a/src/flow/src/adapter.rs
+++ b/src/flow/src/adapter.rs
@@ -35,12 +35,12 @@ use itertools::Itertools;
 use query::{QueryEngine, QueryEngineFactory};
 use serde::{Deserialize, Serialize};
 use session::context::QueryContext;
-use snafu::{OptionExt, ResultExt};
+use snafu::{ensure, OptionExt, ResultExt};
 use store_api::storage::{ConcreteDataType, RegionId};
 use table::metadata::TableId;
 use tokio::sync::{oneshot, watch, Mutex, RwLock};

-use crate::adapter::error::{ExternalSnafu, TableNotFoundSnafu, UnexpectedSnafu};
+use crate::adapter::error::{ExternalSnafu, InternalSnafu, TableNotFoundSnafu, UnexpectedSnafu};
 pub(crate) use crate::adapter::node_context::FlownodeContext;
 use crate::adapter::table_source::TableSource;
 use crate::adapter::util::column_schemas_to_proto;
@@ -66,6 +66,11 @@ use error::Error;

 pub const PER_REQ_MAX_ROW_CNT: usize = 8192;

+// TODO: replace this with `GREPTIME_TIMESTAMP` before v0.9
+pub const AUTO_CREATED_PLACEHOLDER_TS_COL: &str = "__ts_placeholder";
+
+pub const UPDATE_AT_TS_COL: &str = "update_at";
+
 // TODO: refactor common types for flow to a separate module
 /// FlowId is a unique identifier for a flow task
 pub type FlowId = u64;
@@ -154,7 +159,7 @@ pub struct FlownodeManager {
    table_info_source: TableSource,
    frontend_invoker: RwLock<Option<Box<dyn FrontendInvoker + Send + Sync>>>,
    /// contains mapping from table name to global id, and table schema
-    node_context: Mutex<FlownodeContext>,
+    node_context: RwLock<FlownodeContext>,
    flow_err_collectors: RwLock<BTreeMap<FlowId, ErrCollector>>,
    src_send_buf_lens: RwLock<BTreeMap<TableId, watch::Receiver<usize>>>,
    tick_manager: FlowTickManager,
@@ -189,7 +194,7 @@ impl FlownodeManager {
            query_engine,
            table_info_source: srv_map,
            frontend_invoker: RwLock::new(None),
-            node_context: Mutex::new(node_context),
+            node_context: RwLock::new(node_context),
            flow_err_collectors: Default::default(),
            src_send_buf_lens: Default::default(),
            tick_manager,
@@ -279,15 +284,21 @@ impl FlownodeManager {
                    .map(|i| meta.schema.column_schemas[i].name.clone())
                    .collect_vec();
                let schema = meta.schema.column_schemas;
-                let is_auto_create = schema
-                    .last()
-                    .map(|s| s.name == "__ts_placeholder")
-                    .unwrap_or(false);
+                // check if the last column is the auto created timestamp column, hence the table is auto created from
+                // flow's plan type
+                let is_auto_create = {
+                    let correct_name = schema
+                        .last()
+                        .map(|s| s.name == AUTO_CREATED_PLACEHOLDER_TS_COL)
+                        .unwrap_or(false);
+                    let correct_time_index = meta.schema.timestamp_index == Some(schema.len() - 1);
+                    correct_name && correct_time_index
+                };
                (primary_keys, schema, is_auto_create)
            } else {
                // TODO(discord9): condiser remove buggy auto create by schema

-                let node_ctx = self.node_context.lock().await;
+                let node_ctx = self.node_context.read().await;
                let gid: GlobalId = node_ctx
                    .table_repr
                    .get_by_name(&table_name)
@@ -302,6 +313,7 @@ impl FlownodeManager {
                    .clone();
                // TODO(discord9): use default key from schema
                let primary_keys = schema
+                    .typ()
                    .keys
                    .first()
                    .map(|v| {
@@ -312,24 +324,31 @@ impl FlownodeManager {
                    })
                    .unwrap_or_default();
                let update_at = ColumnSchema::new(
-                    "update_at",
+                    UPDATE_AT_TS_COL,
                    ConcreteDataType::timestamp_millisecond_datatype(),
                    true,
                );
                // TODO(discord9): bugged so we can't infer time index from flow plan, so we have to manually set one
                let ts_col = ColumnSchema::new(
-                    "__ts_placeholder",
+                    AUTO_CREATED_PLACEHOLDER_TS_COL,
                    ConcreteDataType::timestamp_millisecond_datatype(),
                    true,
                )
                .with_time_index(true);

                let wout_ts = schema
+                    .typ()
                    .column_types
+                    .clone()
                    .into_iter()
                    .enumerate()
                    .map(|(idx, typ)| {
-                        ColumnSchema::new(format!("Col_{idx}"), typ.scalar_type, typ.nullable)
+                        let name = schema
+                            .names
+                            .get(idx)
+                            .cloned()
+                            .unwrap_or(format!("Col_{}", idx));
+                        ColumnSchema::new(name, typ.scalar_type, typ.nullable)
                    })
                    .collect_vec();

@@ -339,7 +358,7 @@ impl FlownodeManager {

                (primary_keys, with_ts, true)
            };
-
+            let schema_len = schema.len();
            let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;

            debug!(
@@ -348,16 +367,7 @@ impl FlownodeManager {
                table_name.join("."),
                reqs
            );
-            let now = SystemTime::now();
-            let now = now
-                .duration_since(SystemTime::UNIX_EPOCH)
-                .map(|s| s.as_millis() as repr::Timestamp)
-                .unwrap_or_else(|_| {
-                    -(SystemTime::UNIX_EPOCH
-                        .duration_since(now)
-                        .unwrap()
-                        .as_millis() as repr::Timestamp)
-                });
+            let now = self.tick_manager.tick();
            for req in reqs {
                match req {
                    DiffRequest::Insert(insert) => {
@@ -370,13 +380,23 @@ impl FlownodeManager {
                                ))]);
                                // ts col, if auto create
                                if is_auto_create {
+                                    ensure!(
+                                        row.len() == schema_len - 1,
+                                        InternalSnafu {
+                                            reason: format!(
+                                                "Row len mismatch, expect {} got {}",
+                                                schema_len - 1,
+                                                row.len()
+                                            )
+                                        }
+                                    );
                                    row.extend([Value::from(
                                        common_time::Timestamp::new_millisecond(0),
                                    )]);
                                }
-                                row.into()
+                                Ok(row.into())
                            })
-                            .collect::<Vec<_>>();
+                            .collect::<Result<Vec<_>, Error>>()?;
                        let table_name = table_name.last().unwrap().clone();
                        let req = RowInsertRequest {
                            table_name,
@@ -442,7 +462,7 @@ impl FlownodeManager {
        let mut output = BTreeMap::new();
        for (name, sink_recv) in self
            .node_context
-            .lock()
+            .write()
            .await
            .sink_receiver
            .iter_mut()
@@ -490,9 +510,12 @@ impl FlownodeManager {
        debug!("Starting to run");
        loop {
            // TODO(discord9): only run when new inputs arrive or scheduled to
-            self.run_available().await.unwrap();
+            debug!("call run_available in run every second");
+            self.run_available(true).await.unwrap();
+            debug!("call send_writeback_requests in run every second");
            // TODO(discord9): error handling
            self.send_writeback_requests().await.unwrap();
+            debug!("call log_all_errors in run every second");
            self.log_all_errors().await;
            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
        }
@@ -501,29 +524,44 @@ impl FlownodeManager {
    /// Run all available subgraph in the flow node
    /// This will try to run all dataflow in this node
    ///
-    /// However this is not blocking and can sometimes return while actual computation is still running in worker thread
+    /// set `blocking` to true to wait until lock is acquired
+    /// and false to return immediately if lock is not acquired
    /// TODO(discord9): add flag for subgraph that have input since last run
-    pub async fn run_available(&self) -> Result<(), Error> {
-        let now = self.tick_manager.tick();
-
+    pub async fn run_available(&self, blocking: bool) -> Result<(), Error> {
        loop {
+            let now = self.tick_manager.tick();
            for worker in self.worker_handles.iter() {
                // TODO(discord9): consider how to handle error in individual worker
-                worker.lock().await.run_available(now).await.unwrap();
+                if blocking {
+                    worker.lock().await.run_available(now).await?;
+                } else if let Ok(worker) = worker.try_lock() {
+                    worker.run_available(now).await?;
+                } else {
+                    return Ok(());
+                }
            }
            // first check how many inputs were sent
-            let send_cnt = match self.node_context.lock().await.flush_all_sender() {
-                Ok(cnt) => cnt,
+            let (flush_res, buf_len) = if blocking {
+                let ctx = self.node_context.read().await;
+                (ctx.flush_all_sender().await, ctx.get_send_buf_size().await)
+            } else {
+                match self.node_context.try_read() {
+                    Ok(ctx) => (ctx.flush_all_sender().await, ctx.get_send_buf_size().await),
+                    Err(_) => return Ok(()),
+                }
+            };
+            match flush_res {
+                Ok(_) => (),
                Err(err) => {
                    common_telemetry::error!("Flush send buf errors: {:?}", err);
                    break;
                }
            };
-            // if no inputs
-            if send_cnt == 0 {
+            // if no thing in send buf then break
+            if buf_len == 0 {
                break;
            } else {
-                debug!("FlownodeManager::run_available: send_cnt={}", send_cnt);
+                debug!("Send buf len = {}", buf_len);
            }
        }

@@ -542,7 +580,9 @@ impl FlownodeManager {
            rows.len()
        );
        let table_id = region_id.table_id();
-        self.node_context.lock().await.send(table_id, rows)?;
+        self.node_context.read().await.send(table_id, rows).await?;
+        // TODO(discord9): put it in a background task?
+        // self.run_available(false).await?;
        Ok(())
    }
 }
@@ -588,7 +628,7 @@ impl FlownodeManager {
            }
        }

-        let mut node_ctx = self.node_context.lock().await;
+        let mut node_ctx = self.node_context.write().await;
        // assign global id to source and sink table
        for source in source_table_ids {
            node_ctx
@@ -653,21 +693,22 @@ impl FlownodeManager {
 ///
 /// TODO(discord9): better way to do it, and not expose flow tick even to other flow to avoid
 /// TSO coord mess
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct FlowTickManager {
+    /// The starting instant of the flow, used with `start_timestamp` to calculate the current timestamp
    start: Instant,
-}
-
-impl std::fmt::Debug for FlowTickManager {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("FlowTickManager").finish()
-    }
+    /// The timestamp when the flow started
+    start_timestamp: repr::Timestamp,
 }

 impl FlowTickManager {
    pub fn new() -> Self {
        FlowTickManager {
            start: Instant::now(),
+            start_timestamp: SystemTime::now()
+                .duration_since(SystemTime::UNIX_EPOCH)
+                .unwrap()
+                .as_millis() as repr::Timestamp,
        }
    }

@@ -677,6 +718,6 @@ impl FlowTickManager {
    pub fn tick(&self) -> repr::Timestamp {
        let current = Instant::now();
        let since_the_epoch = current - self.start;
-        since_the_epoch.as_millis() as repr::Timestamp
+        since_the_epoch.as_millis() as repr::Timestamp + self.start_timestamp
    }
 }
--- a/src/flow/src/adapter/flownode_impl.rs
+++ b/src/flow/src/adapter/flownode_impl.rs
@@ -14,13 +14,17 @@

 //! impl `FlowNode` trait for FlowNodeManager so standalone can call them

+use std::collections::HashMap;
+
 use api::v1::flow::{flow_request, CreateRequest, DropRequest, FlowRequest, FlowResponse};
 use api::v1::region::InsertRequests;
 use common_error::ext::BoxedError;
 use common_meta::error::{ExternalSnafu, Result, UnexpectedSnafu};
 use common_meta::node_manager::Flownode;
+use common_telemetry::debug;
 use itertools::Itertools;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::RegionId;

 use crate::adapter::FlownodeManager;
 use crate::repr::{self, DiffRow};
@@ -101,12 +105,57 @@ impl Flownode for FlownodeManager {
    async fn handle_inserts(&self, request: InsertRequests) -> Result<FlowResponse> {
        for write_request in request.requests {
            let region_id = write_request.region_id;
-            let rows_proto = write_request.rows.map(|r| r.rows).unwrap_or(vec![]);
+            let table_id = RegionId::from(region_id).table_id();
+
+            let (insert_schema, rows_proto) = write_request
+                .rows
+                .map(|r| (r.schema, r.rows))
+                .unwrap_or_default();
+
            // TODO(discord9): reconsider time assignment mechanism
            let now = self.tick_manager.tick();
+
+            let fetch_order = {
+                let ctx = self.node_context.read().await;
+                let table_col_names = ctx
+                    .table_repr
+                    .get_by_table_id(&table_id)
+                    .map(|r| r.1)
+                    .and_then(|id| ctx.schema.get(&id))
+                    .map(|desc| &desc.names)
+                    .context(UnexpectedSnafu {
+                        err_msg: format!("Table not found: {}", table_id),
+                    })?;
+                let name_to_col = HashMap::<_, _>::from_iter(
+                    insert_schema
+                        .iter()
+                        .enumerate()
+                        .map(|(i, name)| (&name.column_name, i)),
+                );
+                let fetch_order: Vec<usize> = table_col_names
+                    .iter()
+                    .map(|names| {
+                        name_to_col.get(names).copied().context(UnexpectedSnafu {
+                            err_msg: format!("Column not found: {}", names),
+                        })
+                    })
+                    .try_collect()?;
+                if !fetch_order.iter().enumerate().all(|(i, &v)| i == v) {
+                    debug!("Reordering columns: {:?}", fetch_order)
+                }
+                fetch_order
+            };
+
            let rows: Vec<DiffRow> = rows_proto
                .into_iter()
-                .map(repr::Row::from)
+                .map(|r| {
+                    let r = repr::Row::from(r);
+                    let reordered = fetch_order
+                        .iter()
+                        .map(|&i| r.inner[i].clone())
+                        .collect_vec();
+                    repr::Row::new(reordered)
+                })
                .map(|r| (r, now, 1))
                .collect_vec();
            self.handle_write_request(region_id.into(), rows)
--- a/src/flow/src/adapter/node_context.rs
+++ b/src/flow/src/adapter/node_context.rs
@@ -21,13 +21,13 @@ use common_telemetry::debug;
 use session::context::QueryContext;
 use snafu::{OptionExt, ResultExt};
 use table::metadata::TableId;
-use tokio::sync::{broadcast, mpsc};
+use tokio::sync::{broadcast, mpsc, RwLock};

 use crate::adapter::error::{Error, EvalSnafu, TableNotFoundSnafu};
 use crate::adapter::{FlowId, TableName, TableSource};
 use crate::expr::error::InternalSnafu;
 use crate::expr::GlobalId;
-use crate::repr::{DiffRow, RelationType, BROADCAST_CAP};
+use crate::repr::{DiffRow, RelationDesc, RelationType, BROADCAST_CAP};

 /// A context that holds the information of the dataflow
 #[derive(Default, Debug)]
@@ -51,10 +51,8 @@ pub struct FlownodeContext {
            mpsc::UnboundedReceiver<DiffRow>,
        ),
    >,
-    /// store source in buffer for each source table, in case broadcast channel is full
-    pub send_buffer: BTreeMap<TableId, VecDeque<DiffRow>>,
    /// the schema of the table, query from metasrv or inferred from TypedPlan
-    pub schema: HashMap<GlobalId, RelationType>,
+    pub schema: HashMap<GlobalId, RelationDesc>,
    /// All the tables that have been registered in the worker
    pub table_repr: IdToNameMap,
    pub query_context: Option<Arc<QueryContext>>,
@@ -67,18 +65,20 @@ pub struct FlownodeContext {
 #[derive(Debug)]
 pub struct SourceSender {
    sender: broadcast::Sender<DiffRow>,
-    send_buf: VecDeque<DiffRow>,
+    send_buf: RwLock<VecDeque<DiffRow>>,
 }

 impl Default for SourceSender {
    fn default() -> Self {
        Self {
-            sender: broadcast::Sender::new(BROADCAST_CAP),
+            // TODO(discord9): found a better way then increase this to prevent lagging and hence missing input data
+            sender: broadcast::Sender::new(BROADCAST_CAP * 2),
            send_buf: Default::default(),
        }
    }
 }

+// TODO: make all send operation immut
 impl SourceSender {
    pub fn get_receiver(&self) -> broadcast::Receiver<DiffRow> {
        self.sender.subscribe()
@@ -86,15 +86,16 @@ impl SourceSender {

    /// send as many as possible rows from send buf
    /// until send buf is empty or broadchannel is full
-    pub fn try_send_all(&mut self) -> Result<usize, Error> {
+    pub async fn try_send_all(&self) -> Result<usize, Error> {
        let mut row_cnt = 0;
        loop {
+            let mut send_buf = self.send_buf.write().await;
            // if inner sender channel is empty or send buf is empty, there
            // is nothing to do for now, just break
-            if self.sender.len() >= BROADCAST_CAP || self.send_buf.is_empty() {
+            if self.sender.len() >= BROADCAST_CAP || send_buf.is_empty() {
                break;
            }
-            if let Some(row) = self.send_buf.pop_front() {
+            if let Some(row) = send_buf.pop_front() {
                self.sender
                    .send(row)
                    .map_err(|err| {
@@ -109,16 +110,20 @@ impl SourceSender {
        }
        if row_cnt > 0 {
            debug!("Send {} rows", row_cnt);
+            debug!(
+                "Remaining Send buf.len() = {}",
+                self.send_buf.read().await.len()
+            );
        }

        Ok(row_cnt)
    }

    /// return number of rows it actual send(including what's in the buffer)
-    pub fn send_rows(&mut self, rows: Vec<DiffRow>) -> Result<usize, Error> {
-        self.send_buf.extend(rows);
+    pub async fn send_rows(&self, rows: Vec<DiffRow>) -> Result<usize, Error> {
+        self.send_buf.write().await.extend(rows);

-        let row_cnt = self.try_send_all()?;
+        let row_cnt = self.try_send_all().await?;

        Ok(row_cnt)
    }
@@ -128,23 +133,35 @@ impl FlownodeContext {
    /// return number of rows it actual send(including what's in the buffer)
    ///
    /// TODO(discord9): make this concurrent
-    pub fn send(&mut self, table_id: TableId, rows: Vec<DiffRow>) -> Result<usize, Error> {
+    pub async fn send(&self, table_id: TableId, rows: Vec<DiffRow>) -> Result<usize, Error> {
        let sender = self
            .source_sender
-            .get_mut(&table_id)
+            .get(&table_id)
            .with_context(|| TableNotFoundSnafu {
                name: table_id.to_string(),
            })?;
        // debug!("FlownodeContext::send: trying to send {} rows", rows.len());
-        sender.send_rows(rows)
+        sender.send_rows(rows).await
    }

    /// flush all sender's buf
-    pub fn flush_all_sender(&mut self) -> Result<usize, Error> {
-        self.source_sender
-            .iter_mut()
-            .map(|(_table_id, src_sender)| src_sender.try_send_all())
-            .try_fold(0, |acc, x| x.map(|x| x + acc))
+    ///
+    /// return numbers being sent
+    pub async fn flush_all_sender(&self) -> Result<usize, Error> {
+        let mut sum = 0;
+        for sender in self.source_sender.values() {
+            sender.try_send_all().await.inspect(|x| sum += x)?;
+        }
+        Ok(sum)
+    }
+
+    /// Return the sum number of rows in all send buf
+    pub async fn get_send_buf_size(&self) -> usize {
+        let mut sum = 0;
+        for sender in self.source_sender.values() {
+            sum += sender.send_buf.read().await.len();
+        }
+        sum
    }
 }

@@ -226,7 +243,7 @@ impl FlownodeContext {
    /// Retrieves a GlobalId and table schema representing a table previously registered by calling the [register_table] function.
    ///
    /// Returns an error if no table has been registered with the provided names
-    pub fn table(&self, name: &TableName) -> Result<(GlobalId, RelationType), Error> {
+    pub fn table(&self, name: &TableName) -> Result<(GlobalId, RelationDesc), Error> {
        let id = self
            .table_repr
            .get_by_name(name)
@@ -297,7 +314,7 @@ impl FlownodeContext {
            .get_by_name(table_name)
            .map(|(_, gid)| gid)
            .unwrap();
-        self.schema.insert(gid, schema);
+        self.schema.insert(gid, schema.into_unnamed());
        Ok(())
    }

--- a/src/flow/src/adapter/table_source.rs
+++ b/src/flow/src/adapter/table_source.rs
@@ -17,7 +17,6 @@
 use common_error::ext::BoxedError;
 use common_meta::key::table_info::{TableInfoManager, TableInfoValue};
 use common_meta::key::table_name::{TableNameKey, TableNameManager};
-use itertools::Itertools;
 use snafu::{OptionExt, ResultExt};
 use table::metadata::TableId;

@@ -25,7 +24,7 @@ use crate::adapter::error::{
    Error, ExternalSnafu, TableNotFoundMetaSnafu, TableNotFoundSnafu, UnexpectedSnafu,
 };
 use crate::adapter::TableName;
-use crate::repr::{self, ColumnType, RelationType};
+use crate::repr::{self, ColumnType, RelationDesc, RelationType};

 /// mapping of table name <-> table id should be query from tableinfo manager
 pub struct TableSource {
@@ -107,7 +106,7 @@ impl TableSource {
    pub async fn get_table_name_schema(
        &self,
        table_id: &TableId,
-    ) -> Result<(TableName, RelationType), Error> {
+    ) -> Result<(TableName, RelationDesc), Error> {
        let table_info_value = self
            .get_table_info_value(table_id)
            .await?
@@ -123,14 +122,20 @@ impl TableSource {
        ];

        let raw_schema = table_info_value.table_info.meta.schema;
-        let column_types = raw_schema
+        let (column_types, col_names): (Vec<_>, Vec<_>) = raw_schema
            .column_schemas
+            .clone()
            .into_iter()
-            .map(|col| ColumnType {
-                nullable: col.is_nullable(),
-                scalar_type: col.data_type,
+            .map(|col| {
+                (
+                    ColumnType {
+                        nullable: col.is_nullable(),
+                        scalar_type: col.data_type,
+                    },
+                    col.name,
+                )
            })
-            .collect_vec();
+            .unzip();

        let key = table_info_value.table_info.meta.primary_key_indices;
        let keys = vec![repr::Key::from(key)];
@@ -138,10 +143,13 @@ impl TableSource {
        let time_index = raw_schema.timestamp_index;
        Ok((
            table_name,
-            RelationType {
-                column_types,
-                keys,
-                time_index,
+            RelationDesc {
+                typ: RelationType {
+                    column_types,
+                    keys,
+                    time_index,
+                },
+                names: col_names,
            },
        ))
    }
--- a/src/flow/src/adapter/worker.rs
+++ b/src/flow/src/adapter/worker.rs
@@ -285,8 +285,8 @@ impl<'s> Worker<'s> {
                Ok(Some((id, resp))) => {
                    if let Err(err) = self.itc_server.blocking_lock().resp(id, resp) {
                        common_telemetry::error!(
-                            "Worker's itc server has been closed unexpectedly, shutting down worker: {}",
-                            err
+                            err;
+                            "Worker's itc server has been closed unexpectedly, shutting down worker"
                        );
                        break;
                    };
--- a/src/flow/src/compute/render/map.rs
+++ b/src/flow/src/compute/render/map.rs
@@ -124,9 +124,13 @@ fn mfp_subgraph(
    // 1. Read all updates that were emitted between the last time this arrangement had updates and the current time.
    // 2. Output the updates.
    // 3. Truncate all updates within that range.
-    let from = arrange.read().last_compaction_time().map(|n| n + 1);
+    let from = arrange.read().last_compaction_time();
    let from = from.unwrap_or(repr::Timestamp::MIN);
-    let output_kv = arrange.read().get_updates_in_range(from..=now);
+    let range = (
+        std::ops::Bound::Excluded(from),
+        std::ops::Bound::Included(now),
+    );
+    let output_kv = arrange.read().get_updates_in_range(range);
    // the output is expected to be key -> empty val
    let output = output_kv
        .into_iter()
--- a/src/flow/src/compute/render/reduce.rs
+++ b/src/flow/src/compute/render/reduce.rs
@@ -26,7 +26,7 @@ use crate::adapter::error::{Error, PlanSnafu};
 use crate::compute::render::{Context, SubgraphArg};
 use crate::compute::state::Scheduler;
 use crate::compute::types::{Arranged, Collection, CollectionBundle, ErrCollector, Toff};
-use crate::expr::error::{DataTypeSnafu, InternalSnafu};
+use crate::expr::error::{DataAlreadyExpiredSnafu, DataTypeSnafu, InternalSnafu};
 use crate::expr::{AggregateExpr, EvalError, ScalarExpr};
 use crate::plan::{AccumulablePlan, AggrWithIndex, KeyValPlan, Plan, ReducePlan, TypedPlan};
 use crate::repr::{self, DiffRow, KeyValDiffRow, RelationType, Row};
@@ -301,9 +301,13 @@ fn update_reduce_distinct_arrange(
    // Deal with output:

    // 1. Read all updates that were emitted between the last time this arrangement had updates and the current time.
-    let from = arrange.read().last_compaction_time().map(|n| n + 1);
+    let from = arrange.read().last_compaction_time();
    let from = from.unwrap_or(repr::Timestamp::MIN);
-    let output_kv = arrange.read().get_updates_in_range(from..=now);
+    let range = (
+        std::ops::Bound::Excluded(from),
+        std::ops::Bound::Included(now),
+    );
+    let output_kv = arrange.read().get_updates_in_range(range);

    // 2. Truncate all updates stored in arrangement within that range.
    let run_compaction = || {
@@ -397,6 +401,29 @@ fn reduce_accum_subgraph(
    // TODO(discord9): consider key-based lock
    let mut arrange = arrange.write();
    for (key, value_diffs) in key_to_vals {
+        if let Some(expire_man) = &arrange.get_expire_state() {
+            let mut is_expired = false;
+            err_collector.run(|| {
+                if let Some(expired) = expire_man.get_expire_duration(now, &key)? {
+                    is_expired = true;
+                    // expired data is ignored in computation, and a simple warning is logged
+                    common_telemetry::warn!(
+                        "Data already expired: {}",
+                        DataAlreadyExpiredSnafu {
+                            expired_by: expired,
+                        }
+                        .build()
+                    );
+                    Ok(())
+                } else {
+                    Ok(())
+                }
+            });
+            if is_expired {
+                // errors already collected, we can just continue to next key
+                continue;
+            }
+        }
        let col_diffs = {
            let row_len = value_diffs[0].0.len();
            let res = err_collector.run(|| get_col_diffs(value_diffs, row_len));
--- a/src/flow/src/compute/render/src_sink.rs
+++ b/src/flow/src/compute/render/src_sink.rs
@@ -20,12 +20,14 @@ use common_telemetry::{debug, info};
 use hydroflow::scheduled::graph_ext::GraphExt;
 use itertools::Itertools;
 use snafu::OptionExt;
+use tokio::sync::broadcast::error::TryRecvError;
 use tokio::sync::{broadcast, mpsc};

 use crate::adapter::error::{Error, PlanSnafu};
 use crate::compute::render::Context;
 use crate::compute::types::{Arranged, Collection, CollectionBundle, Toff};
-use crate::expr::GlobalId;
+use crate::expr::error::InternalSnafu;
+use crate::expr::{EvalError, GlobalId};
 use crate::repr::{DiffRow, Row, BROADCAST_CAP};

 #[allow(clippy::mutable_key_type)]
@@ -55,18 +57,43 @@ impl<'referred, 'df> Context<'referred, 'df> {
            .df
            .add_subgraph_source("source", send_port, move |_ctx, send| {
                let now = *now.borrow();
-                let arr = arrange_handler_inner.write().get_updates_in_range(..=now);
-                err_collector.run(|| arrange_handler_inner.write().compact_to(now));
+                // write lock to prevent unexpected mutation
+                let mut arranged = arrange_handler_inner.write();
+                let arr = arranged.get_updates_in_range(..=now);
+                err_collector.run(|| arranged.compact_to(now));

+                debug!("Call source");
                let prev_avail = arr.into_iter().map(|((k, _), t, d)| (k, t, d));
                let mut to_send = Vec::new();
                let mut to_arrange = Vec::new();
                // TODO(discord9): handling tokio broadcast error
-                while let Ok((r, t, d)) = src_recv.try_recv() {
-                    if t <= now {
-                        to_send.push((r, t, d));
-                    } else {
-                        to_arrange.push(((r, Row::empty()), t, d));
+                loop {
+                    match src_recv.try_recv() {
+                        Ok((r, t, d)) => {
+                            if t <= now {
+                                to_send.push((r, t, d));
+                            } else {
+                                to_arrange.push(((r, Row::empty()), t, d));
+                            }
+                        }
+                        Err(TryRecvError::Empty) => {
+                            break;
+                        }
+                        Err(TryRecvError::Lagged(lag_offset)) => {
+                            common_telemetry::error!("Flow missing {} rows behind", lag_offset);
+                            break;
+                        }
+                        Err(err) => {
+                            err_collector.run(|| -> Result<(), EvalError> {
+                                InternalSnafu {
+                                    reason: format!(
+                                        "Error receiving from broadcast channel: {}",
+                                        err
+                                    ),
+                                }
+                                .fail()
+                            });
+                        }
                    }
                }
                let all = prev_avail.chain(to_send).collect_vec();
@@ -77,10 +104,10 @@ impl<'referred, 'df> Context<'referred, 'df> {
                        to_arrange.len()
                    );
                }
-                err_collector.run(|| arrange_handler_inner.write().apply_updates(now, to_arrange));
+                err_collector.run(|| arranged.apply_updates(now, to_arrange));
                send.give(all);
-                // always schedule source to run at next tick
-                inner_schd.schedule_at(now + 1);
+                // always schedule source to run at now so we can repeatedly run source if needed
+                inner_schd.schedule_at(now);
            });
        schd.set_cur_subgraph(sub);
        let arranged = Arranged::new(arrange_handler);
--- a/src/flow/src/expr/error.rs
+++ b/src/flow/src/expr/error.rs
@@ -100,4 +100,11 @@ pub enum EvalError {
        #[snafu(implicit)]
        location: Location,
    },
+
+    #[snafu(display("Incoming data already expired by {} ms", expired_by))]
+    DataAlreadyExpired {
+        expired_by: i64,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
--- a/src/flow/src/expr/func.rs
+++ b/src/flow/src/expr/func.rs
@@ -76,6 +76,13 @@ impl UnmaterializableFunc {
        }
    }

+    pub fn is_valid_func_name(name: &str) -> bool {
+        matches!(
+            name.to_lowercase().as_str(),
+            "now" | "current_schema" | "tumble"
+        )
+    }
+
    /// Create a UnmaterializableFunc from a string of the function name
    pub fn from_str_args(name: &str, args: Vec<TypedExpr>) -> Result<Self, Error> {
        match name.to_lowercase().as_str() {
@@ -183,6 +190,13 @@ impl UnaryFunc {
        }
    }

+    pub fn is_valid_func_name(name: &str) -> bool {
+        matches!(
+            name.to_lowercase().as_str(),
+            "not" | "is_null" | "is_true" | "is_false" | "step_timestamp" | "cast"
+        )
+    }
+
    /// Create a UnaryFunc from a string of the function name and given argument type(optional)
    pub fn from_str_and_type(
        name: &str,
@@ -278,9 +292,9 @@ impl UnaryFunc {
                start_time,
            } => {
                let ts = get_ts_as_millisecond(arg)?;
-                let start_time = start_time.map(|t| t.val()).unwrap_or(0);
+                let start_time = start_time.map(|t| t.val());
                let window_size = (window_size.to_nanosecond() / 1_000_000) as repr::Duration; // nanosecond to millisecond
-                let window_start = start_time + (ts - start_time) / window_size * window_size;
+                let window_start = get_window_start(ts, window_size, start_time);

                let ret = Timestamp::new_millisecond(window_start);
                Ok(Value::from(ret))
@@ -290,9 +304,9 @@ impl UnaryFunc {
                start_time,
            } => {
                let ts = get_ts_as_millisecond(arg)?;
-                let start_time = start_time.map(|t| t.val()).unwrap_or(0);
+                let start_time = start_time.map(|t| t.val());
                let window_size = (window_size.to_nanosecond() / 1_000_000) as repr::Duration; // nanosecond to millisecond
-                let window_start = start_time + (ts - start_time) / window_size * window_size;
+                let window_start = get_window_start(ts, window_size, start_time);

                let window_end = window_start + window_size;
                let ret = Timestamp::new_millisecond(window_end);
@@ -302,6 +316,35 @@ impl UnaryFunc {
    }
 }

+fn get_window_start(
+    ts: repr::Timestamp,
+    window_size: repr::Duration,
+    start_time: Option<repr::Timestamp>,
+) -> repr::Timestamp {
+    let start_time = start_time.unwrap_or(0);
+    // left close right open
+    if ts >= start_time {
+        start_time + (ts - start_time) / window_size * window_size
+    } else {
+        start_time + (ts - start_time) / window_size * window_size
+            - if ((start_time - ts) % window_size) != 0 {
+                window_size
+            } else {
+                0
+            }
+    }
+}
+
+#[test]
+fn test_get_window_start() {
+    assert_eq!(get_window_start(1, 3, None), 0);
+    assert_eq!(get_window_start(3, 3, None), 3);
+    assert_eq!(get_window_start(0, 3, None), 0);
+
+    assert_eq!(get_window_start(-1, 3, None), -3);
+    assert_eq!(get_window_start(-3, 3, None), -3);
+}
+
 fn get_ts_as_millisecond(arg: Value) -> Result<repr::Timestamp, EvalError> {
    let ts = if let Some(ts) = arg.as_timestamp() {
        ts.convert_to(TimeUnit::Millisecond)
@@ -550,6 +593,27 @@ impl BinaryFunc {
        Ok(ret)
    }

+    pub fn is_valid_func_name(name: &str) -> bool {
+        matches!(
+            name.to_lowercase().as_str(),
+            "eq" | "equal"
+                | "not_eq"
+                | "not_equal"
+                | "lt"
+                | "lte"
+                | "gt"
+                | "gte"
+                | "add"
+                | "sub"
+                | "subtract"
+                | "mul"
+                | "multiply"
+                | "div"
+                | "divide"
+                | "mod"
+        )
+    }
+
    /// choose the appropriate specialization based on the input types
    /// return a specialization of the binary function and it's actual input and output type(so no null type present)
    ///
@@ -741,6 +805,10 @@ impl VariadicFunc {
        }
    }

+    pub fn is_valid_func_name(name: &str) -> bool {
+        matches!(name.to_lowercase().as_str(), "and" | "or")
+    }
+
    /// Create a VariadicFunc from a string of the function name and given argument types(optional)
    pub fn from_str_and_types(
        name: &str,
--- a/src/flow/src/expr/scalar.rs
+++ b/src/flow/src/expr/scalar.rs
@@ -45,6 +45,8 @@ impl TypedExpr {

 impl TypedExpr {
    /// expand multi-value expression to multiple expressions with new indices
+    ///
+    /// Currently it just mean expand `TumbleWindow` to `TumbleWindowFloor` and `TumbleWindowCeiling`
    pub fn expand_multi_value(
        input_typ: &RelationType,
        exprs: &[TypedExpr],
--- a/src/flow/src/repr/relation.rs
+++ b/src/flow/src/repr/relation.rs
@@ -262,6 +262,19 @@ impl RelationType {

        true
    }
+
+    /// Return relation describe with column names
+    pub fn into_named(self, names: Vec<ColumnName>) -> RelationDesc {
+        RelationDesc { typ: self, names }
+    }
+
+    /// Return relation describe without column names
+    pub fn into_unnamed(self) -> RelationDesc {
+        RelationDesc {
+            typ: self,
+            names: vec![],
+        }
+    }
 }

 /// The type of a `Value`
@@ -325,8 +338,8 @@ fn return_true() -> bool {
 /// Individual column names are optional.
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
 pub struct RelationDesc {
-    typ: RelationType,
-    names: Vec<ColumnName>,
+    pub typ: RelationType,
+    pub names: Vec<ColumnName>,
 }

 impl RelationDesc {
--- a/src/flow/src/transform.rs
+++ b/src/flow/src/transform.rs
@@ -23,6 +23,7 @@ use literal::{from_substrait_literal, from_substrait_type};
 use prost::Message;
 use query::parser::QueryLanguageParser;
 use query::plan::LogicalPlan;
+use query::query_engine::DefaultSerializer;
 use query::QueryEngine;
 use session::context::QueryContext;
 use snafu::{OptionExt, ResultExt};
@@ -121,7 +122,7 @@ pub async fn sql_to_flow_plan(
        .context(ExternalSnafu)?;
    let LogicalPlan::DfPlan(plan) = plan;
    let sub_plan = DFLogicalSubstraitConvertor {}
-        .to_sub_plan(&plan)
+        .to_sub_plan(&plan, DefaultSerializer)
        .map_err(BoxedError::new)
        .context(ExternalSnafu)?;

@@ -211,7 +212,7 @@ mod test {
            let schema = RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]);

            tri_map.insert(Some(name.clone()), Some(1024), gid);
-            schemas.insert(gid, schema);
+            schemas.insert(gid, schema.into_unnamed());
        }

        {
@@ -225,7 +226,7 @@ mod test {
                ColumnType::new(CDT::uint32_datatype(), false),
                ColumnType::new(CDT::datetime_datatype(), false),
            ]);
-            schemas.insert(gid, schema);
+            schemas.insert(gid, schema.into_unnamed());
            tri_map.insert(Some(name.clone()), Some(1025), gid);
        }

@@ -294,7 +295,9 @@ mod test {
        let LogicalPlan::DfPlan(plan) = plan;

        // encode then decode so to rely on the impl of conversion from logical plan to substrait plan
-        let bytes = DFLogicalSubstraitConvertor {}.encode(&plan).unwrap();
+        let bytes = DFLogicalSubstraitConvertor {}
+            .encode(&plan, DefaultSerializer)
+            .unwrap();

        proto::Plan::decode(bytes).unwrap()
    }
--- a/src/flow/src/transform/aggr.rs
+++ b/src/flow/src/transform/aggr.rs
@@ -435,6 +435,236 @@ mod test {
    use crate::repr::{self, ColumnType, RelationType};
    use crate::transform::test::{create_test_ctx, create_test_query_engine, sql_to_substrait};

+    /// TODO(discord9): add more illegal sql tests
+    #[tokio::test]
+    async fn test_tumble_composite() {
+        let engine = create_test_query_engine();
+        let sql =
+            "SELECT number, avg(number) FROM numbers_with_ts GROUP BY tumble(ts, '1 hour'), number";
+        let plan = sql_to_substrait(engine.clone(), sql).await;
+
+        let mut ctx = create_test_ctx();
+        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).unwrap();
+
+        let aggr_exprs = vec![
+            AggregateExpr {
+                func: AggregateFunc::SumUInt32,
+                expr: ScalarExpr::Column(0),
+                distinct: false,
+            },
+            AggregateExpr {
+                func: AggregateFunc::Count,
+                expr: ScalarExpr::Column(0),
+                distinct: false,
+            },
+        ];
+        let avg_expr = ScalarExpr::If {
+            cond: Box::new(ScalarExpr::Column(4).call_binary(
+                ScalarExpr::Literal(Value::from(0i64), CDT::int64_datatype()),
+                BinaryFunc::NotEq,
+            )),
+            then: Box::new(ScalarExpr::Column(3).call_binary(
+                ScalarExpr::Column(4).call_unary(UnaryFunc::Cast(CDT::uint64_datatype())),
+                BinaryFunc::DivUInt64,
+            )),
+            els: Box::new(ScalarExpr::Literal(Value::Null, CDT::uint64_datatype())),
+        };
+        let expected = TypedPlan {
+            // TODO(discord9): mfp indirectly ref to key columns
+            /*
+            .with_key(vec![1])
+            .with_time_index(Some(0)),*/
+            plan: Plan::Mfp {
+                input: Box::new(
+                    Plan::Reduce {
+                        input: Box::new(
+                            Plan::Get {
+                                id: crate::expr::Id::Global(GlobalId::User(1)),
+                            }
+                            .with_types(RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
+                        ),
+                        key_val_plan: KeyValPlan {
+                            key_plan: MapFilterProject::new(2)
+                                .map(vec![
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowFloor {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: None,
+                                        },
+                                    ),
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowCeiling {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: None,
+                                        },
+                                    ),
+                                    ScalarExpr::Column(0),
+                                ])
+                                .unwrap()
+                                .project(vec![2, 3, 4])
+                                .unwrap()
+                                .into_safe(),
+                            val_plan: MapFilterProject::new(2)
+                                .project(vec![0, 1])
+                                .unwrap()
+                                .into_safe(),
+                        },
+                        reduce_plan: ReducePlan::Accumulable(AccumulablePlan {
+                            full_aggrs: aggr_exprs.clone(),
+                            simple_aggrs: vec![
+                                AggrWithIndex::new(aggr_exprs[0].clone(), 0, 0),
+                                AggrWithIndex::new(aggr_exprs[1].clone(), 0, 1),
+                            ],
+                            distinct_aggrs: vec![],
+                        }),
+                    }
+                    .with_types(
+                        RelationType::new(vec![
+                            // keys
+                            ColumnType::new(CDT::datetime_datatype(), false), // window start(time index)
+                            ColumnType::new(CDT::datetime_datatype(), false), // window end(pk)
+                            ColumnType::new(CDT::uint32_datatype(), false),   // number(pk)
+                            // values
+                            ColumnType::new(CDT::uint64_datatype(), true), // avg.sum(number)
+                            ColumnType::new(CDT::int64_datatype(), true),  // avg.count(number)
+                        ])
+                        .with_key(vec![1, 2])
+                        .with_time_index(Some(0)),
+                    ),
+                ),
+                mfp: MapFilterProject::new(5)
+                    .map(vec![
+                        avg_expr,
+                        ScalarExpr::Column(2), // number(pk)
+                        ScalarExpr::Column(5), // avg.sum(number)
+                        ScalarExpr::Column(0), // window start
+                        ScalarExpr::Column(1), // window end
+                    ])
+                    .unwrap()
+                    .project(vec![6, 7, 8, 9])
+                    .unwrap(),
+            },
+            typ: RelationType::new(vec![
+                ColumnType::new(CDT::uint32_datatype(), false), // number
+                ColumnType::new(CDT::uint64_datatype(), true),  // avg(number)
+                ColumnType::new(CDT::datetime_datatype(), false), // window start
+                ColumnType::new(CDT::datetime_datatype(), false), // window end
+            ]),
+        };
+        assert_eq!(flow_plan, expected);
+    }
+
+    #[tokio::test]
+    async fn test_tumble_parse_optional() {
+        let engine = create_test_query_engine();
+        let sql = "SELECT sum(number) FROM numbers_with_ts GROUP BY tumble(ts, '1 hour')";
+        let plan = sql_to_substrait(engine.clone(), sql).await;
+
+        let mut ctx = create_test_ctx();
+        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).unwrap();
+
+        let aggr_expr = AggregateExpr {
+            func: AggregateFunc::SumUInt32,
+            expr: ScalarExpr::Column(0),
+            distinct: false,
+        };
+        let expected = TypedPlan {
+            typ: RelationType::new(vec![
+                ColumnType::new(CDT::uint64_datatype(), true), // sum(number)
+                ColumnType::new(CDT::datetime_datatype(), false), // window start
+                ColumnType::new(CDT::datetime_datatype(), false), // window end
+            ]),
+            // TODO(discord9): mfp indirectly ref to key columns
+            /*
+            .with_key(vec![1])
+            .with_time_index(Some(0)),*/
+            plan: Plan::Mfp {
+                input: Box::new(
+                    Plan::Reduce {
+                        input: Box::new(
+                            Plan::Get {
+                                id: crate::expr::Id::Global(GlobalId::User(1)),
+                            }
+                            .with_types(RelationType::new(vec![
+                                ColumnType::new(ConcreteDataType::uint32_datatype(), false),
+                                ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                            ])),
+                        ),
+                        key_val_plan: KeyValPlan {
+                            key_plan: MapFilterProject::new(2)
+                                .map(vec![
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowFloor {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: None,
+                                        },
+                                    ),
+                                    ScalarExpr::Column(1).call_unary(
+                                        UnaryFunc::TumbleWindowCeiling {
+                                            window_size: Interval::from_month_day_nano(
+                                                0,
+                                                0,
+                                                3_600_000_000_000,
+                                            ),
+                                            start_time: None,
+                                        },
+                                    ),
+                                ])
+                                .unwrap()
+                                .project(vec![2, 3])
+                                .unwrap()
+                                .into_safe(),
+                            val_plan: MapFilterProject::new(2)
+                                .project(vec![0, 1])
+                                .unwrap()
+                                .into_safe(),
+                        },
+                        reduce_plan: ReducePlan::Accumulable(AccumulablePlan {
+                            full_aggrs: vec![aggr_expr.clone()],
+                            simple_aggrs: vec![AggrWithIndex::new(aggr_expr.clone(), 0, 0)],
+                            distinct_aggrs: vec![],
+                        }),
+                    }
+                    .with_types(
+                        RelationType::new(vec![
+                            ColumnType::new(CDT::datetime_datatype(), false), // window start
+                            ColumnType::new(CDT::datetime_datatype(), false), // window end
+                            ColumnType::new(CDT::uint64_datatype(), true),    //sum(number)
+                        ])
+                        .with_key(vec![1])
+                        .with_time_index(Some(0)),
+                    ),
+                ),
+                mfp: MapFilterProject::new(3)
+                    .map(vec![
+                        ScalarExpr::Column(2),
+                        ScalarExpr::Column(3),
+                        ScalarExpr::Column(0),
+                        ScalarExpr::Column(1),
+                    ])
+                    .unwrap()
+                    .project(vec![4, 5, 6])
+                    .unwrap(),
+            },
+        };
+        assert_eq!(flow_plan, expected);
+    }
+
    #[tokio::test]
    async fn test_tumble_parse() {
        let engine = create_test_query_engine();
--- a/src/flow/src/transform/expr.rs
+++ b/src/flow/src/transform/expr.rs
@@ -101,8 +101,7 @@ impl TypedExpr {
            .unzip();

        match arg_len {
-            // because variadic function can also have 1 arguments, we need to check if it's a variadic function first
-            1 if VariadicFunc::from_str_and_types(fn_name, &arg_types).is_err() => {
+            1 if UnaryFunc::is_valid_func_name(fn_name) => {
                let func = UnaryFunc::from_str_and_type(fn_name, None)?;
                let arg = arg_exprs[0].clone();
                let ret_type = ColumnType::new_nullable(func.signature().output.clone());
@@ -124,8 +123,7 @@ impl TypedExpr {

                Ok(TypedExpr::new(arg.call_unary(func), ret_type))
            }
-            // because variadic function can also have 2 arguments, we need to check if it's a variadic function first
-            2 if VariadicFunc::from_str_and_types(fn_name, &arg_types).is_err() => {
+            2 if BinaryFunc::is_valid_func_name(fn_name) => {
                let (func, signature) =
                    BinaryFunc::from_str_expr_and_type(fn_name, &arg_exprs, &arg_types[0..2])?;

@@ -167,7 +165,8 @@ impl TypedExpr {
                Ok(TypedExpr::new(ret_expr, ret_type))
            }
            _var => {
-                if let Ok(func) = VariadicFunc::from_str_and_types(fn_name, &arg_types) {
+                if VariadicFunc::is_valid_func_name(fn_name) {
+                    let func = VariadicFunc::from_str_and_types(fn_name, &arg_types)?;
                    let ret_type = ColumnType::new_nullable(func.signature().output.clone());
                    let mut expr = ScalarExpr::CallVariadic {
                        func,
@@ -175,9 +174,8 @@ impl TypedExpr {
                    };
                    expr.optimize();
                    Ok(TypedExpr::new(expr, ret_type))
-                } else if let Ok(func) =
-                    UnmaterializableFunc::from_str_args(fn_name, arg_typed_exprs)
-                {
+                } else if UnmaterializableFunc::is_valid_func_name(fn_name) {
+                    let func = UnmaterializableFunc::from_str_args(fn_name, arg_typed_exprs)?;
                    let ret_type = ColumnType::new_nullable(func.signature().output.clone());
                    Ok(TypedExpr::new(
                        ScalarExpr::CallUnmaterializable(func),
@@ -324,8 +322,12 @@ impl TypedExpr {

 #[cfg(test)]
 mod test {
+    use std::collections::HashMap;
+
+    use common_time::{DateTime, Interval};
    use datatypes::prelude::ConcreteDataType;
    use datatypes::value::Value;
+    use pretty_assertions::assert_eq;

    use super::*;
    use crate::expr::{GlobalId, MapFilterProject};
@@ -510,4 +512,162 @@ mod test {

        assert_eq!(flow_plan.unwrap(), expected);
    }
+
+    #[test]
+    fn test_func_sig() {
+        fn lit(v: impl ToString) -> substrait_proto::proto::FunctionArgument {
+            use substrait_proto::proto::expression;
+            let expr = Expression {
+                rex_type: Some(expression::RexType::Literal(expression::Literal {
+                    nullable: false,
+                    type_variation_reference: 0,
+                    literal_type: Some(expression::literal::LiteralType::String(v.to_string())),
+                })),
+            };
+            substrait_proto::proto::FunctionArgument {
+                arg_type: Some(substrait_proto::proto::function_argument::ArgType::Value(
+                    expr,
+                )),
+            }
+        }
+        fn col(i: usize) -> substrait_proto::proto::FunctionArgument {
+            use substrait_proto::proto::expression;
+            let expr = Expression {
+                rex_type: Some(expression::RexType::Selection(Box::new(
+                    expression::FieldReference {
+                        reference_type: Some(
+                            expression::field_reference::ReferenceType::DirectReference(
+                                expression::ReferenceSegment {
+                                    reference_type: Some(
+                                        expression::reference_segment::ReferenceType::StructField(
+                                            Box::new(expression::reference_segment::StructField {
+                                                field: i as i32,
+                                                child: None,
+                                            }),
+                                        ),
+                                    ),
+                                },
+                            ),
+                        ),
+                        root_type: None,
+                    },
+                ))),
+            };
+            substrait_proto::proto::FunctionArgument {
+                arg_type: Some(substrait_proto::proto::function_argument::ArgType::Value(
+                    expr,
+                )),
+            }
+        }
+
+        let f = substrait_proto::proto::expression::ScalarFunction {
+            function_reference: 0,
+            arguments: vec![col(0)],
+            options: vec![],
+            output_type: None,
+            ..Default::default()
+        };
+        let input_schema = RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]);
+        let extensions = FunctionExtensions {
+            anchor_to_name: HashMap::from([(0, "is_null".to_string())]),
+        };
+        let res = TypedExpr::from_substrait_scalar_func(&f, &input_schema, &extensions).unwrap();
+
+        assert_eq!(
+            res,
+            TypedExpr {
+                expr: ScalarExpr::Column(0).call_unary(UnaryFunc::IsNull),
+                typ: ColumnType {
+                    scalar_type: CDT::boolean_datatype(),
+                    nullable: true,
+                },
+            }
+        );
+
+        let f = substrait_proto::proto::expression::ScalarFunction {
+            function_reference: 0,
+            arguments: vec![col(0), col(1)],
+            options: vec![],
+            output_type: None,
+            ..Default::default()
+        };
+        let input_schema = RelationType::new(vec![
+            ColumnType::new(CDT::uint32_datatype(), false),
+            ColumnType::new(CDT::uint32_datatype(), false),
+        ]);
+        let extensions = FunctionExtensions {
+            anchor_to_name: HashMap::from([(0, "add".to_string())]),
+        };
+        let res = TypedExpr::from_substrait_scalar_func(&f, &input_schema, &extensions).unwrap();
+
+        assert_eq!(
+            res,
+            TypedExpr {
+                expr: ScalarExpr::Column(0)
+                    .call_binary(ScalarExpr::Column(1), BinaryFunc::AddUInt32,),
+                typ: ColumnType {
+                    scalar_type: CDT::uint32_datatype(),
+                    nullable: true,
+                },
+            }
+        );
+
+        let f = substrait_proto::proto::expression::ScalarFunction {
+            function_reference: 0,
+            arguments: vec![col(0), lit("1 second"), lit("2021-07-01 00:00:00")],
+            options: vec![],
+            output_type: None,
+            ..Default::default()
+        };
+        let input_schema = RelationType::new(vec![
+            ColumnType::new(CDT::timestamp_nanosecond_datatype(), false),
+            ColumnType::new(CDT::string_datatype(), false),
+        ]);
+        let extensions = FunctionExtensions {
+            anchor_to_name: HashMap::from([(0, "tumble".to_string())]),
+        };
+        let res = TypedExpr::from_substrait_scalar_func(&f, &input_schema, &extensions).unwrap();
+
+        assert_eq!(
+            res,
+            ScalarExpr::CallUnmaterializable(UnmaterializableFunc::TumbleWindow {
+                ts: Box::new(
+                    ScalarExpr::Column(0)
+                        .with_type(ColumnType::new(CDT::timestamp_nanosecond_datatype(), false))
+                ),
+                window_size: Interval::from_month_day_nano(0, 0, 1_000_000_000),
+                start_time: Some(DateTime::new(1625097600000))
+            })
+            .with_type(ColumnType::new(CDT::timestamp_millisecond_datatype(), true)),
+        );
+
+        let f = substrait_proto::proto::expression::ScalarFunction {
+            function_reference: 0,
+            arguments: vec![col(0), lit("1 second")],
+            options: vec![],
+            output_type: None,
+            ..Default::default()
+        };
+        let input_schema = RelationType::new(vec![
+            ColumnType::new(CDT::timestamp_nanosecond_datatype(), false),
+            ColumnType::new(CDT::string_datatype(), false),
+        ]);
+        let extensions = FunctionExtensions {
+            anchor_to_name: HashMap::from([(0, "tumble".to_string())]),
+        };
+        let res = TypedExpr::from_substrait_scalar_func(&f, &input_schema, &extensions).unwrap();
+
+        assert_eq!(
+            res,
+            ScalarExpr::CallUnmaterializable(UnmaterializableFunc::TumbleWindow {
+                ts: Box::new(
+                    ScalarExpr::Column(0)
+                        .with_type(ColumnType::new(CDT::timestamp_nanosecond_datatype(), false))
+                ),
+                window_size: Interval::from_month_day_nano(0, 0, 1_000_000_000),
+                start_time: None
+            })
+            .with_type(ColumnType::new(CDT::timestamp_millisecond_datatype(), true)),
+        )
+    }
 }
--- a/src/flow/src/transform/plan.rs
+++ b/src/flow/src/transform/plan.rs
@@ -269,7 +269,7 @@ impl TypedPlan {
                        id: crate::expr::Id::Global(table.0),
                    };
                    let get_table = TypedPlan {
-                        typ: table.1,
+                        typ: table.1.typ().clone(),
                        plan: get_table,
                    };

--- a/src/flow/src/utils.rs
+++ b/src/flow/src/utils.rs
@@ -18,6 +18,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ops::Bound;
 use std::sync::Arc;

+use common_telemetry::debug;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 use smallvec::{smallvec, SmallVec};
@@ -86,7 +87,7 @@ impl KeyExpiryManager {
    ///
    /// - If given key is expired by now (that is less than `now - expiry_duration`), return the amount of time it's expired.
    /// - If it's not expired, return None
-    pub fn update_event_ts(
+    pub fn get_expire_duration_and_update_event_ts(
        &mut self,
        now: Timestamp,
        row: &Row,
@@ -95,6 +96,33 @@ impl KeyExpiryManager {
            return Ok(None);
        };

+        self.event_ts_to_key
+            .entry(event_ts)
+            .or_default()
+            .insert(row.clone());
+
+        if let Some(expire_time) = self.compute_expiration_timestamp(now) {
+            if expire_time > event_ts {
+                // return how much time it's expired
+                return Ok(Some(expire_time - event_ts));
+            }
+        }
+
+        Ok(None)
+    }
+
+    /// Get the expire duration of a key, if it's expired by now.
+    ///
+    /// Return None if the key is not expired
+    pub fn get_expire_duration(
+        &self,
+        now: Timestamp,
+        row: &Row,
+    ) -> Result<Option<Duration>, EvalError> {
+        let Some(event_ts) = self.extract_event_ts(row)? else {
+            return Ok(None);
+        };
+
        if let Some(expire_time) = self.compute_expiration_timestamp(now) {
            if expire_time > event_ts {
                // return how much time it's expired
@@ -102,10 +130,6 @@ impl KeyExpiryManager {
            }
        }

-        self.event_ts_to_key
-            .entry(event_ts)
-            .or_default()
-            .insert(row.clone());
        Ok(None)
    }

@@ -189,6 +213,10 @@ impl Arrangement {
        }
    }

+    pub fn get_expire_state(&self) -> Option<&KeyExpiryManager> {
+        self.expire_state.as_ref()
+    }
+
    pub fn set_expire_state(&mut self, expire_state: KeyExpiryManager) {
        self.expire_state = Some(expire_state);
    }
@@ -208,8 +236,12 @@ impl Arrangement {
        for ((key, val), update_ts, diff) in updates {
            // check if the key is expired
            if let Some(s) = &mut self.expire_state {
-                if let Some(expired_by) = s.update_event_ts(now, &key)? {
+                if let Some(expired_by) = s.get_expire_duration_and_update_event_ts(now, &key)? {
                    max_expired_by = max_expired_by.max(Some(expired_by));
+                    debug!(
+                        "Expired key: {:?}, expired by: {:?} with time being now={}",
+                        key, expired_by, now
+                    );
                    continue;
                }
            }
@@ -335,7 +367,9 @@ impl Arrangement {
            for (key, updates) in batch {
                // check if the key is expired
                if let Some(s) = &mut self.expire_state {
-                    if let Some(expired_by) = s.update_event_ts(now, &key)? {
+                    if let Some(expired_by) =
+                        s.get_expire_duration_and_update_event_ts(now, &key)?
+                    {
                        max_expired_by = max_expired_by.max(Some(expired_by));
                        continue;
                    }
@@ -540,6 +574,10 @@ impl ArrangeHandler {
    pub fn set_full_arrangement(&self, full: bool) {
        self.write().full_arrangement = full;
    }
+
+    pub fn is_full_arrangement(&self) -> bool {
+        self.read().full_arrangement
+    }
 }

 #[cfg(test)]
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -68,5 +68,5 @@ datanode.workspace = true
 futures = "0.3"
 meta-srv = { workspace = true, features = ["mock"] }
 strfmt = "0.2"
-tower = "0.4"
+tower.workspace = true
 uuid.workspace = true
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -554,6 +554,7 @@ pub fn check_permission(
        Statement::ShowIndex(stmt) => {
            validate_db_permission!(stmt, query_ctx);
        }
+        Statement::ShowStatus(_stmt) => {}
        Statement::DescribeTable(stmt) => {
            validate_param(stmt.name(), query_ctx)?;
        }
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Ruihang Xia	1bfba48755	Revert "build(deps): upgrade opendal to 0.46 (#4037 )" This reverts commit `f9db5ff0d6`.	2024-06-03 20:28:59 +08:00
Ruihang Xia	457998f0fe	Merge branch 'main' into avoid-query-meta Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-05-31 18:16:36 +08:00
Ruihang Xia	b02c256157	perf: use memory state to check if a logical region exists Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-05-31 18:16:08 +08:00
Ruihang Xia	45fee948e9	fix: display error in correct format (#4082 ) * fix: display error in correct format Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * add address to RegionServer error Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-05-31 09:25:14 +00:00
discord9	ea49f8a5c4	feat(flow): make write path faster with shared lock (#4073 ) * feat(WIP): make write faster * feat: read lock on fast path * chore: per review	2024-05-31 06:50:22 +00:00
Jeremyhi	43afea1a9d	refactor!: remove the tableid in ddl response since tableids is enough (#4080 ) * refactor: remove the tableid in ddl response since tableids is enough * chore: upgrade proto	2024-05-31 06:41:53 +00:00
Weny Xu	fcfcf86385	feat: implement `WalEntryDistributor`, `WalEntryReciver` (#4031 ) * feat: implement the `WalEntryDistributor` and `WalEntryReceiver` * test: add tests for `WalEntryDistributor` * refactor: use bounded channel * chore: apply suggestions from CR	2024-05-31 03:03:38 +00:00
Jeremyhi	26b112ab57	refactor: remove upgrade cli tool (#4077 )	2024-05-31 00:40:27 +00:00
dennis zhuang	24612f62dd	feat: querying from view works (#3952 ) * feat: querying from view works * feat: use MemoryCatalogProviderList instead of DummyCatalogList * refactor: revert src/query/src/dummy_catalog.rs * chore: clean code * fix: make clippy happy * fix: toml format * fix: sqlness * fix: forgot files * fix: make sqlness happy * test: table source, serializer and decoder * fix: fail to decode plan because of invalid table names * test: adds more sqlness test for view * chore: remove unused errors * fix: comments * fix: typo * fix: invalidate view info cache after creating view successfully * chore: apply suggestion Co-authored-by: Ruihang Xia <waynestxia@gmail.com> * chore: apply suggestion Co-authored-by: Ruihang Xia <waynestxia@gmail.com> * fix: compile error after rebeasing * chore: style Co-authored-by: Ruihang Xia <waynestxia@gmail.com> * fix: don't export table_name in common-meta * chore: change ViewInfo::new signature * docs: leave a TODO for optimize param --------- Co-authored-by: Ruihang Xia <waynestxia@gmail.com>	2024-05-30 21:45:56 +00:00
Ruihang Xia	85a231850d	fix: add tailing separator to prefix (#4078 ) * fix: add tailing separator to prefix Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * project select result Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-05-30 15:29:26 +00:00
Ruihang Xia	f024054ed3	ci: cargo gc fuzz test runner (#4074 ) * ci: cargo gc fuzz test runner Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * change profile to dev Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-05-30 10:41:29 +00:00
discord9	05751084e7	chore: bump to v0.8.1 (#4055 )	2024-05-30 07:59:52 +00:00
Jeremyhi	8b6596faa0	feat: avoid some cloning when mirror requests to flownode (#4068 ) * feat: some refactor mirror requests to flownode * feat: use spawn_bg to avoid impact front-ground write * feat: add mirror row count metric	2024-05-30 07:29:13 +00:00
Weny Xu	eab309ff7e	fix: avoid acquiring lock during reading stats (#4070 ) * fix: avoid acquiring lock during reading stats * chore: apply suggestions from CR * chore: apply suggestions from CR	2024-05-30 07:08:04 +00:00
discord9	7de336f087	fix(flow): correctness bugs (#4018 ) * fix: optional args of tumble * fix(WIP): choose * feat: rename default ts to GREPTIME_TIMESTAMP * fix: default timestamp name * fix: reorder write requests * fix: expire state * fix: test of tumble * fix: send buf clear * fix: ts<start time correct window * fix: window_start when ts<start_time * mend * fix: range begin>range end * refactor: per reviews * feat!: ts placeholder rename to __ts_placeholder * refactor: better condition * tests(WIP): func sig choose * tests(WIP): tumble func * feat: make run_available optional blocking * tests: tumble transform * chore: clippy * fix?: lagged missing data * fix: flow source break on empty chnl	2024-05-30 03:49:11 +00:00
Weny Xu	6e9a9dc333	refactor(log_store): remove associated type `Namespace` and `Entry` in `LogStore` (#4038 ) * refactor(log_store): remove associated type `Namespace` in `LogStore` * fix(test): filter entries * refactor: ignore incomplete parts * refactor: simplify `RawEntryReaderFilter` * chore: avoid cloning * test: add tests for `maybe_emit_entry` * refactor: remove `Namespace` trait and rename `LogStoreNamespace` to `Namespace` * chore: apply suggestions from CR * refactor: refine `entry` method signature * feat: ingore any potential incomplete parts * refactor: rename `namespace` to `provider` * chore: add debug assertion * refactor: associated type `Entry` in `LogStore` * refactor: renamse `namespace` to `provider` * refactor: remove unwrap * refactor: let `remaining_entries` return a optional vector * test: add basic tests for kafka logstore * refactor: move `append` method under `cfg(test)` * refactor: rename `RawEntry` to `Entry` * refactor: rename `CorruptedLogEntry` to `CorruptedEntry` * test: add tests for handling corrupted raw entry stream * refactor: rename `ns` to `provider` * refactor: remove `entry_stream.rs` file * chore: remove unused code * chore: update comments * chore: apply suggestions from CR * chore: update comments * chore: apply suggestions from CR * chore: remove Deref * chore: add comments * fix: ignores tail corrupted data * chore: add comments * fix: add `MIN_BATCH_SIZE` limit	2024-05-29 13:44:01 +00:00
Yingwen	848bd7e553	feat: Implements row group level parallel unordered scanner (#3992 ) * feat: unordered scanner * feat: support compat * chore: update debug print fix: missing ranges in scan parts * fix: ensure chunk size > 0 * fix: parallel is disabled if there is only one file and memtable * chore: reader metrics * chore: remove todo * refactor: add ScanPartBuilder trait * chore: pass file meta to the part builder * chore: make part builder private * docs: update comment * chore: remove meta() * refactor: only prune file ranges in ScanInput replaces ScanPartBuilder with FileRangeCollector which only collect file ranges * chore: address typo * fix: panic when no partition * feat: Postpone part distribution * chore: handle empty partition in mito * style: fix clippy	2024-05-29 11:06:08 +00:00
LYZJU2019	f0effd2680	feat: Implement SHOW STATUS (#4050 ) * show status returning empty contents * return an empty set instead of affected rows * chore: Update src/query/src/sql.rs --------- Co-authored-by: Yingwen <realevenyag@gmail.com>	2024-05-29 04:49:05 +00:00
dennis zhuang	aafb468547	fix: set local or session time_zone not work (#4064 ) * fix: set local or session time_zone not work * chore: supports PostgreSQL-specific setting time zone	2024-05-29 00:06:13 +00:00
Weny Xu	4aa756c896	feat: open region in background (#4052 ) * feat: open region in background * feat: trace opening regions * feat: wait for the opening region * feat: let engine to handle the future open request * fix: fix `test_region_registering`	2024-05-28 13:58:15 +00:00
Weny Xu	d3860671a8	chore: add `LAST_SENT_HEARTBEAT_ELAPSED` metric (#4062 )	2024-05-28 08:05:37 +00:00
tison	9dd6e033a7	refactor: move Database to client crate behind testing feature (#4059 ) * refactor: move Database to client crate behind testing feature Signed-off-by: tison <wander4096@gmail.com> * partial move Signed-off-by: tison <wander4096@gmail.com> * catch up more Signed-off-by: tison <wander4096@gmail.com> * fix imports Signed-off-by: tison <wander4096@gmail.com> * finish Signed-off-by: tison <wander4096@gmail.com> * tidy Signed-off-by: tison <wander4096@gmail.com> --------- Signed-off-by: tison <wander4096@gmail.com>	2024-05-28 03:21:43 +00:00
Weny Xu	097f62f459	refactor(fuzz-tests): generate ts value separately (#4056 ) refactor: generate ts value separately	2024-05-27 14:07:32 +00:00
Weny Xu	048368fd87	feat: invoke `flush_table` and `compact_table` in fuzz tests (#4045 ) * feat: invoke `flush_table` and `compact_table` in fuzz tests * feat: support to flush and compact physical metric table * fix: avoid to create tables with the same name * feat: validate values after flushing or compacting table	2024-05-27 09:26:50 +00:00
tison	f9db5ff0d6	build(deps): upgrade opendal to 0.46 (#4037 ) * build(deps): upgrade opendal to 0.46 Signed-off-by: tison <wander4096@gmail.com> * migrate writes Signed-off-by: tison <wander4096@gmail.com> * migrate reads Signed-off-by: tison <wander4096@gmail.com> * fixup object safety Signed-off-by: tison <wander4096@gmail.com> * fixup names Signed-off-by: tison <wander4096@gmail.com> * fixup compilation Signed-off-by: tison <wander4096@gmail.com> * fixup compilation Signed-off-by: tison <wander4096@gmail.com> * a few Buffer to Vec Signed-off-by: tison <wander4096@gmail.com> * Make greptime buildable with opendal 0.46 (#5) Signed-off-by: Xuanwo <github@xuanwo.io> * fixup toml check Signed-off-by: tison <wander4096@gmail.com> * test_orc_opener Signed-off-by: tison <wander4096@gmail.com> * Fix lru cache (#6) Signed-off-by: Xuanwo <github@xuanwo.io> * clippy Signed-off-by: tison <wander4096@gmail.com> * improve comments Signed-off-by: tison <wander4096@gmail.com> * address comments Signed-off-by: tison <wander4096@gmail.com> * reduce buf copy Signed-off-by: tison <wander4096@gmail.com> * upgrade to reqwest 0.12 Signed-off-by: tison <wander4096@gmail.com> --------- Signed-off-by: tison <wander4096@gmail.com> Signed-off-by: Xuanwo <github@xuanwo.io> Co-authored-by: Xuanwo <github@xuanwo.io>	2024-05-27 09:12:23 +00:00
Weny Xu	20ce7d428d	fix(metric-engine): missing catchup implementation (#4048 ) * fix(metric-engine): missing catchup implementation * fix: should be `metadata_region_id`	2024-05-27 07:56:46 +00:00
Weny Xu	75bddc0bf5	fix(fuzz-tests): avoid to drop in-use database (#4049 ) * fix(fuzz-tests): avoid to drop in-use database * fix: correct datahome path * fix: correct `schema_name` * chore: apply suggestions from CR	2024-05-27 07:44:59 +00:00
tison	c78043d526	build(deps): merge tower deps to workspace (#4036 ) Signed-off-by: tison <wander4096@gmail.com>	2024-05-27 07:15:30 +00:00