feat: support windowed sort with where condition

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

.github/actions/build-windows-artifacts/action.yml

@@ -40,7 +40,7 @@ runs:
 
     - name: Install PyArrow Package
       shell: pwsh
-      run: pip install pyarrow
+      run: pip install pyarrow numpy
 
     - name: Install WSL distribution
       uses: Vampire/setup-wsl@v2

.github/actions/setup-etcd-cluster/action.yml

@@ -18,7 +18,7 @@ runs:
         --set replicaCount=${{ inputs.etcd-replicas }} \
         --set resources.requests.cpu=50m \
         --set resources.requests.memory=128Mi \
-        --set resources.limits.cpu=1000m \
+        --set resources.limits.cpu=1500m \
         --set resources.limits.memory=2Gi \
         --set auth.rbac.create=false \
         --set auth.rbac.token.enabled=false \

.github/workflows/develop.yml

@@ -436,7 +436,7 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        target: ["fuzz_migrate_mito_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
+        target: ["fuzz_migrate_mito_regions", "fuzz_migrate_metric_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
         mode:
           - name: "Remote WAL"
             minio: true
@@ -449,6 +449,12 @@ jobs:
               minio: true
               kafka: false
               values: "with-minio.yaml"
+          - target: "fuzz_migrate_metric_regions"
+            mode:
+              name: "Local WAL"
+              minio: true
+              kafka: false
+              values: "with-minio.yaml"
     steps:
       - name: Remove unused software
         run: |
@@ -688,7 +694,7 @@ jobs:
         with:
           python-version: '3.10'
       - name: Install PyArrow Package
-        run: pip install pyarrow
+        run: pip install pyarrow numpy
       - name: Setup etcd server
         working-directory: tests-integration/fixtures/etcd
         run: docker compose -f docker-compose-standalone.yml up -d --wait

.github/workflows/nightly-ci.yml

@@ -92,7 +92,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install PyArrow Package
-        run: pip install pyarrow
+        run: pip install pyarrow numpy
       - name: Install WSL distribution
         uses: Vampire/setup-wsl@v2
         with:

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "Inflector"
@@ -200,12 +200,6 @@ version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
 
-[[package]]
-name = "anymap"
-version = "1.0.0-beta.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f1f8f5a6f3d50d89e3797d7593a50f96bb2aaa20ca0cc7be1fb673232c91d72"
-
 [[package]]
 name = "anymap2"
 version = "0.13.0"
@@ -214,7 +208,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"
 
 [[package]]
 name = "api"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-base",
  "common-decimal",
@@ -775,7 +769,7 @@ dependencies = [
 
 [[package]]
 name = "auth"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -1385,7 +1379,7 @@ dependencies = [
 
 [[package]]
 name = "cache"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "catalog",
  "common-error",
@@ -1393,7 +1387,7 @@ dependencies = [
  "common-meta",
  "moka",
  "snafu 0.8.5",
- "substrait 0.9.4",
+ "substrait 0.9.5",
 ]
 
 [[package]]
@@ -1420,7 +1414,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "catalog"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow",
@@ -1759,7 +1753,7 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
 
 [[package]]
 name = "client"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -1789,12 +1783,11 @@ dependencies = [
  "serde_json",
  "snafu 0.8.5",
  "substrait 0.37.3",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "tokio",
  "tokio-stream",
  "tonic 0.11.0",
  "tracing",
- "tracing-subscriber",
 ]
 
 [[package]]
@@ -1808,6 +1801,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "clocksource"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "129026dd5a8a9592d96916258f3a5379589e513ea5e86aeb0bd2530286e44e9e"
+dependencies = [
+ "libc",
+ "time",
+ "winapi",
+]
+
 [[package]]
 name = "cmake"
 version = "0.1.51"
@@ -1819,7 +1823,7 @@ dependencies = [
 
 [[package]]
 name = "cmd"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "auth",
@@ -1876,7 +1880,7 @@ dependencies = [
  "similar-asserts",
  "snafu 0.8.5",
  "store-api",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "temp-env",
  "tempfile",
@@ -1922,7 +1926,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"
 
 [[package]]
 name = "common-base"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "anymap2",
  "async-trait",
@@ -1940,7 +1944,7 @@ dependencies = [
 
 [[package]]
 name = "common-catalog"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "chrono",
  "common-error",
@@ -1951,7 +1955,7 @@ dependencies = [
 
 [[package]]
 name = "common-config"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-base",
  "common-error",
@@ -1974,7 +1978,7 @@ dependencies = [
 
 [[package]]
 name = "common-datasource"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arrow",
  "arrow-schema",
@@ -2011,7 +2015,7 @@ dependencies = [
 
 [[package]]
 name = "common-decimal"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "bigdecimal 0.4.5",
  "common-error",
@@ -2024,7 +2028,7 @@ dependencies = [
 
 [[package]]
 name = "common-error"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "snafu 0.8.5",
  "strum 0.25.0",
@@ -2033,7 +2037,7 @@ dependencies = [
 
 [[package]]
 name = "common-frontend"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -2048,7 +2052,7 @@ dependencies = [
 
 [[package]]
 name = "common-function"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -2088,7 +2092,7 @@ dependencies = [
 
 [[package]]
 name = "common-greptimedb-telemetry"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "common-runtime",
@@ -2105,7 +2109,7 @@ dependencies = [
 
 [[package]]
 name = "common-grpc"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -2131,7 +2135,7 @@ dependencies = [
 
 [[package]]
 name = "common-grpc-expr"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "common-base",
@@ -2144,12 +2148,13 @@ dependencies = [
  "paste",
  "prost 0.12.6",
  "snafu 0.8.5",
+ "store-api",
  "table",
 ]
 
 [[package]]
 name = "common-macro"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arc-swap",
  "common-query",
@@ -2163,7 +2168,7 @@ dependencies = [
 
 [[package]]
 name = "common-mem-prof"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-error",
  "common-macro",
@@ -2176,7 +2181,7 @@ dependencies = [
 
 [[package]]
 name = "common-meta"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "anymap2",
  "api",
@@ -2233,11 +2238,11 @@ dependencies = [
 
 [[package]]
 name = "common-plugins"
-version = "0.9.4"
+version = "0.9.5"
 
 [[package]]
 name = "common-pprof"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-error",
  "common-macro",
@@ -2249,7 +2254,7 @@ dependencies = [
 
 [[package]]
 name = "common-procedure"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -2276,7 +2281,7 @@ dependencies = [
 
 [[package]]
 name = "common-procedure-test"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "common-procedure",
@@ -2284,7 +2289,7 @@ dependencies = [
 
 [[package]]
 name = "common-query"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -2310,7 +2315,7 @@ dependencies = [
 
 [[package]]
 name = "common-recordbatch"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arc-swap",
  "common-error",
@@ -2329,19 +2334,27 @@ dependencies = [
 
 [[package]]
 name = "common-runtime"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
+ "clap 4.5.19",
  "common-error",
  "common-macro",
  "common-telemetry",
+ "futures",
  "lazy_static",
  "num_cpus",
  "once_cell",
+ "parking_lot 0.12.3",
  "paste",
+ "pin-project",
  "prometheus",
+ "rand",
+ "ratelimit",
  "serde",
+ "serde_json",
  "snafu 0.8.5",
+ "tempfile",
  "tokio",
  "tokio-metrics",
  "tokio-metrics-collector",
@@ -2351,7 +2364,7 @@ dependencies = [
 
 [[package]]
 name = "common-telemetry"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "atty",
  "backtrace",
@@ -2379,7 +2392,7 @@ dependencies = [
 
 [[package]]
 name = "common-test-util"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "client",
  "common-query",
@@ -2391,7 +2404,7 @@ dependencies = [
 
 [[package]]
 name = "common-time"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arrow",
  "chrono",
@@ -2407,7 +2420,7 @@ dependencies = [
 
 [[package]]
 name = "common-version"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "build-data",
  "const_format",
@@ -2418,7 +2431,7 @@ dependencies = [
 
 [[package]]
 name = "common-wal"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-base",
  "common-error",
@@ -3227,7 +3240,7 @@ dependencies = [
 
 [[package]]
 name = "datanode"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -3277,7 +3290,7 @@ dependencies = [
  "session",
  "snafu 0.8.5",
  "store-api",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tokio",
  "toml 0.8.19",
@@ -3286,7 +3299,7 @@ dependencies = [
 
 [[package]]
 name = "datatypes"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3892,7 +3905,7 @@ dependencies = [
 
 [[package]]
 name = "file-engine"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -4003,7 +4016,7 @@ dependencies = [
 
 [[package]]
 name = "flow"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow",
@@ -4060,7 +4073,7 @@ dependencies = [
  "snafu 0.8.5",
  "store-api",
  "strum 0.25.0",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tokio",
  "tonic 0.11.0",
@@ -4122,7 +4135,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
 
 [[package]]
 name = "frontend"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -4518,7 +4531,7 @@ dependencies = [
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9#0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=255f87a3318ace3f88a67f76995a0e14910983f4#255f87a3318ace3f88a67f76995a0e14910983f4"
 dependencies = [
  "prost 0.12.6",
  "serde",
@@ -5170,7 +5183,7 @@ dependencies = [
 
 [[package]]
 name = "index"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "asynchronous-codec",
@@ -5250,7 +5263,7 @@ dependencies = [
 [[package]]
 name = "influxdb_line_protocol"
 version = "0.1.0"
-source = "git+https://github.com/evenyag/influxdb_iox?branch=feat/line-protocol#10ef0d0b02705ac7518717390939fa3a9bcfcacc"
+source = "git+https://github.com/evenyag/influxdb_iox?branch=feat%2Fline-protocol#10ef0d0b02705ac7518717390939fa3a9bcfcacc"
 dependencies = [
  "bytes",
  "nom",
@@ -5511,7 +5524,7 @@ dependencies = [
 [[package]]
 name = "jsonb"
 version = "0.4.1"
-source = "git+https://github.com/datafuselabs/jsonb.git?rev=46ad50fc71cf75afbf98eec455f7892a6387c1fc#46ad50fc71cf75afbf98eec455f7892a6387c1fc"
+source = "git+https://github.com/databendlabs/jsonb.git?rev=46ad50fc71cf75afbf98eec455f7892a6387c1fc#46ad50fc71cf75afbf98eec455f7892a6387c1fc"
 dependencies = [
  "byteorder",
  "fast-float",
@@ -6001,7 +6014,7 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
 [[package]]
 name = "log-store"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -6321,7 +6334,7 @@ dependencies = [
 
 [[package]]
 name = "meta-client"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -6347,7 +6360,7 @@ dependencies = [
 
 [[package]]
 name = "meta-srv"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -6408,9 +6421,9 @@ dependencies = [
 [[package]]
 name = "meter-core"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd#80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
+source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac#a10facb353b41460eeb98578868ebf19c2084fac"
 dependencies = [
- "anymap",
+ "anymap2",
  "once_cell",
  "parking_lot 0.12.3",
 ]
@@ -6418,14 +6431,14 @@ dependencies = [
 [[package]]
 name = "meter-macros"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd#80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
+source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac#a10facb353b41460eeb98578868ebf19c2084fac"
 dependencies = [
  "meter-core",
 ]
 
 [[package]]
 name = "metric-engine"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -6528,7 +6541,7 @@ dependencies = [
 
 [[package]]
 name = "mito2"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -7264,7 +7277,7 @@ dependencies = [
 
 [[package]]
 name = "object-store"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "anyhow",
  "bytes",
@@ -7549,12 +7562,13 @@ dependencies = [
  "ordered-float 4.3.0",
  "percent-encoding",
  "rand",
+ "serde_json",
  "thiserror",
 ]
 
 [[package]]
 name = "operator"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -7599,7 +7613,7 @@ dependencies = [
  "sql",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "store-api",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tokio",
  "tokio-util",
@@ -7849,7 +7863,7 @@ dependencies = [
 
 [[package]]
 name = "partition"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -8150,7 +8164,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pipeline"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -8312,13 +8326,14 @@ dependencies = [
 
 [[package]]
 name = "plugins"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "auth",
  "common-base",
  "datanode",
  "frontend",
  "meta-srv",
+ "serde",
  "snafu 0.8.5",
 ]
 
@@ -8586,7 +8601,7 @@ dependencies = [
 
 [[package]]
 name = "promql"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "ahash 0.8.11",
  "async-trait",
@@ -8612,9 +8627,9 @@ dependencies = [
 
 [[package]]
 name = "promql-parser"
-version = "0.4.1"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c1ad4a4cfa84ec4aa5831c82e57af0a3faf3f0af83bee13fa1390b2d0a32dc9"
+checksum = "7fe99e6f80a79abccf1e8fb48dd63473a36057e600cc6ea36147c8318698ae6f"
 dependencies = [
  "cfgrammar",
  "chrono",
@@ -8622,6 +8637,8 @@ dependencies = [
  "lrlex",
  "lrpar",
  "regex",
+ "serde",
+ "serde_json",
 ]
 
 [[package]]
@@ -8822,7 +8839,7 @@ dependencies = [
 
 [[package]]
 name = "puffin"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-compression 0.4.13",
  "async-trait",
@@ -8944,7 +8961,7 @@ dependencies = [
 
 [[package]]
 name = "query"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -8979,6 +8996,7 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-sql",
  "datatypes",
+ "fastrand",
  "format_num",
  "futures",
  "futures-util",
@@ -8993,12 +9011,15 @@ dependencies = [
  "object-store",
  "once_cell",
  "paste",
+ "pretty_assertions",
  "prometheus",
  "promql",
  "promql-parser",
  "prost 0.12.6",
  "rand",
  "regex",
+ "serde",
+ "serde_json",
  "session",
  "snafu 0.8.5",
  "sql",
@@ -9007,10 +9028,11 @@ dependencies = [
  "stats-cli",
  "store-api",
  "streaming-stats",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tokio",
  "tokio-stream",
+ "uuid",
 ]
 
 [[package]]
@@ -9190,6 +9212,17 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "ratelimit"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c1bb13e2dcfa2232ac6887157aad8d9b3fe4ca57f7c8d4938ff5ea9be742300"
+dependencies = [
+ "clocksource",
+ "parking_lot 0.12.3",
+ "thiserror",
+]
+
 [[package]]
 name = "raw-cpuid"
 version = "11.2.0"
@@ -10441,7 +10474,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "script"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -10735,8 +10768,9 @@ dependencies = [
 
 [[package]]
 name = "servers"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
+ "ahash 0.8.11",
  "aide",
  "api",
  "arrow",
@@ -10845,7 +10879,7 @@ dependencies = [
 
 [[package]]
 name = "session"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -10906,9 +10940,9 @@ dependencies = [
 
 [[package]]
 name = "shadow-rs"
-version = "0.31.1"
+version = "0.35.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02c282402d25101f9c893e9cd7e4cae535fe7db18b81291de973026c219ddf1e"
+checksum = "2311e39772c00391875f40e34d43efef247b23930143a70ca5fbec9505937420"
 dependencies = [
  "const_format",
  "git2",
@@ -11166,7 +11200,7 @@ dependencies = [
 
 [[package]]
 name = "sql"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "chrono",
@@ -11227,7 +11261,7 @@ dependencies = [
 
 [[package]]
 name = "sqlness-runner"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "clap 4.5.19",
@@ -11447,7 +11481,7 @@ dependencies = [
 
 [[package]]
 name = "store-api"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -11465,6 +11499,7 @@ dependencies = [
  "datatypes",
  "derive_builder 0.12.0",
  "futures",
+ "humantime",
  "serde",
  "serde_json",
  "snafu 0.8.5",
@@ -11616,7 +11651,7 @@ dependencies = [
 
 [[package]]
 name = "substrait"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "bytes",
@@ -11815,7 +11850,7 @@ dependencies = [
 
 [[package]]
 name = "table"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -12081,7 +12116,7 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
 
 [[package]]
 name = "tests-fuzz"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arbitrary",
  "async-trait",
@@ -12123,7 +12158,7 @@ dependencies = [
 
 [[package]]
 name = "tests-integration"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -12185,7 +12220,7 @@ dependencies = [
  "sql",
  "sqlx",
  "store-api",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tempfile",
  "time",

Cargo.toml

@@ -65,7 +65,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.9.4"
+version = "0.9.5"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -121,13 +121,13 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "255f87a3318ace3f88a67f76995a0e14910983f4" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
-jsonb = { git = "https://github.com/datafuselabs/jsonb.git", rev = "46ad50fc71cf75afbf98eec455f7892a6387c1fc", default-features = false }
+jsonb = { git = "https://github.com/databendlabs/jsonb.git", rev = "46ad50fc71cf75afbf98eec455f7892a6387c1fc", default-features = false }
 lazy_static = "1.4"
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd" }
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "a10facb353b41460eeb98578868ebf19c2084fac" }
 mockall = "0.11.4"
 moka = "0.12"
 notify = "6.1"
@@ -138,15 +138,18 @@ opentelemetry-proto = { version = "0.5", features = [
     "metrics",
     "trace",
     "with-serde",
+    "logs",
 ] }
+parking_lot = "0.12"
 parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
 paste = "1.0"
 pin-project = "1.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-promql-parser = { version = "0.4.1" }
+promql-parser = { version = "0.4.3", features = ["ser"] }
 prost = "0.12"
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.8"
+ratelimit = "0.9"
 regex = "1.8"
 regex-automata = { version = "0.4" }
 reqwest = { version = "0.12", default-features = false, features = [
@@ -166,7 +169,7 @@ schemars = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
 serde_with = "3"
-shadow-rs = "0.31"
+shadow-rs = "0.35"
 similar-asserts = "1.6.0"
 smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
@@ -177,13 +180,16 @@ sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "5
 ] }
 strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
-tokio = { version = "1.36", features = ["full"] }
+tokio = { version = "1.40", features = ["full"] }
 tokio-postgres = "0.7"
 tokio-stream = { version = "0.1" }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
 tonic = { version = "0.11", features = ["tls", "gzip", "zstd"] }
 tower = { version = "0.4" }
+tracing-appender = "0.2"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
+typetag = "0.2"
 uuid = { version = "1.7", features = ["serde", "v4", "fast-rng"] }
 zstd = "0.13"
 
@@ -258,7 +264,7 @@ tokio-rustls = { git = "https://github.com/GreptimeTeam/tokio-rustls" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
-rev = "80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
+rev = "a10facb353b41460eeb98578868ebf19c2084fac"
 
 [profile.release]
 debug = 1

Makefile

@@ -8,7 +8,7 @@ CARGO_BUILD_OPTS := --locked
 IMAGE_REGISTRY ?= docker.io
 IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
-DEV_BUILDER_IMAGE_TAG ?= 2024-06-06-5674c14f-20240920110415
+DEV_BUILDER_IMAGE_TAG ?= 2024-10-19-a5c00e85-20241024184445
 BUILDX_MULTI_PLATFORM_BUILD ?= false
 BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu

config/config.md

@@ -83,6 +83,7 @@
 | `wal.backoff_max` | String | `10s` | The maximum backoff delay.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.backoff_base` | Integer | `2` | The exponential backoff rate, i.e. next backoff = base * current backoff.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.backoff_deadline` | String | `5mins` | The deadline of retries.<br/>**It's only used when the provider is `kafka`**. |
+| `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system<br/>can still successfully replay memtable data without throwing an<br/>out-of-range error.<br/>However, enabling this option might lead to unexpected data loss,<br/>as the system will skip over missing entries instead of treating<br/>them as critical errors. |
 | `metadata_store` | -- | -- | Metadata storage options. |
 | `metadata_store.file_size` | String | `256MB` | Kv file size in bytes. |
 | `metadata_store.purge_threshold` | String | `4GB` | Kv purge threshold. |
@@ -115,7 +116,9 @@
 | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. |
 | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
 | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
-| `region_engine.mito.max_background_jobs` | Integer | `4` | Max number of running background jobs |
+| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
+| `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). |
+| `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). |
 | `region_engine.mito.auto_flush_interval` | String | `1h` | Interval to auto flush a region if it has not flushed yet. |
 | `region_engine.mito.global_write_buffer_size` | String | Auto | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. |
 | `region_engine.mito.global_write_buffer_reject_size` | String | Auto | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`. |
@@ -409,6 +412,7 @@
 | `wal.backoff_deadline` | String | `5mins` | The deadline of retries.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.create_index` | Bool | `true` | Whether to enable WAL index creation.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.dump_index_interval` | String | `60s` | The interval for dumping WAL indexes.<br/>**It's only used when the provider is `kafka`**. |
+| `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system<br/>can still successfully replay memtable data without throwing an<br/>out-of-range error.<br/>However, enabling this option might lead to unexpected data loss,<br/>as the system will skip over missing entries instead of treating<br/>them as critical errors. |
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
@@ -435,7 +439,9 @@
 | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. |
 | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
 | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
-| `region_engine.mito.max_background_jobs` | Integer | `4` | Max number of running background jobs |
+| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
+| `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). |
+| `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). |
 | `region_engine.mito.auto_flush_interval` | String | `1h` | Interval to auto flush a region if it has not flushed yet. |
 | `region_engine.mito.global_write_buffer_size` | String | Auto | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. |
 | `region_engine.mito.global_write_buffer_reject_size` | String | Auto | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` |

config/datanode.example.toml

@@ -213,6 +213,17 @@ create_index = true
 ## **It's only used when the provider is `kafka`**.
 dump_index_interval = "60s"
 
+## Ignore missing entries during read WAL.
+## **It's only used when the provider is `kafka`**.
+##
+## This option ensures that when Kafka messages are deleted, the system
+## can still successfully replay memtable data without throwing an
+## out-of-range error.
+## However, enabling this option might lead to unexpected data loss,
+## as the system will skip over missing entries instead of treating
+## them as critical errors.
+overwrite_entry_start_id = false
+
 # The Kafka SASL configuration.
 # **It's only used when the provider is `kafka`**.
 # Available SASL mechanisms:
@@ -405,8 +416,17 @@ manifest_checkpoint_distance = 10
 ## Whether to compress manifest and checkpoint file by gzip (default false).
 compress_manifest = false
 
-## Max number of running background jobs
-max_background_jobs = 4
+## Max number of running background flush jobs (default: 1/2 of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_flushes = 4
+
+## Max number of running background compaction jobs (default: 1/4 of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_compactions = 2
+
+## Max number of running background purge jobs (default: number of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_purges = 8
 
 ## Interval to auto flush a region if it has not flushed yet.
 auto_flush_interval = "1h"
@@ -626,7 +646,7 @@ url = ""
 headers = { }
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"

config/flownode.example.toml

@@ -101,8 +101,8 @@ threshold = "10s"
 sample_ratio = 1.0
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"
 

config/frontend.example.toml

@@ -231,7 +231,7 @@ url = ""
 headers = { }
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"

config/metasrv.example.toml

@@ -218,7 +218,7 @@ url = ""
 headers = { }
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"

config/standalone.example.toml

@@ -237,6 +237,17 @@ backoff_base = 2
 ## **It's only used when the provider is `kafka`**.
 backoff_deadline = "5mins"
 
+## Ignore missing entries during read WAL.
+## **It's only used when the provider is `kafka`**.
+##
+## This option ensures that when Kafka messages are deleted, the system
+## can still successfully replay memtable data without throwing an
+## out-of-range error.
+## However, enabling this option might lead to unexpected data loss,
+## as the system will skip over missing entries instead of treating
+## them as critical errors.
+overwrite_entry_start_id = false
+
 # The Kafka SASL configuration.
 # **It's only used when the provider is `kafka`**.
 # Available SASL mechanisms:
@@ -443,8 +454,17 @@ manifest_checkpoint_distance = 10
 ## Whether to compress manifest and checkpoint file by gzip (default false).
 compress_manifest = false
 
-## Max number of running background jobs
-max_background_jobs = 4
+## Max number of running background flush jobs (default: 1/2 of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_flushes = 4
+
+## Max number of running background compaction jobs (default: 1/4 of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_compactions = 2
+
+## Max number of running background purge jobs (default: number of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_purges = 8
 
 ## Interval to auto flush a region if it has not flushed yet.
 auto_flush_interval = "1h"
@@ -670,7 +690,7 @@ url = ""
 headers = { }
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"

docs/benchmarks/log/README.md

@@ -48,4 +48,4 @@ Please refer to [SQL query](./query.sql) for GreptimeDB and Clickhouse, and [que
 
 ## Addition
 - You can tune GreptimeDB's configuration to get better performance.
-- You can setup GreptimeDB to use S3 as storage, see [here](https://docs.greptime.com/user-guide/operations/configuration/#storage-options).
+- You can setup GreptimeDB to use S3 as storage, see [here](https://docs.greptime.com/user-guide/deployments/configuration#storage-options).

docs/how-to/how-to-change-log-level-on-the-fly.md

@@ -0,0 +1,16 @@
+# Change Log Level on the Fly
+
+## HTTP API
+
+example:
+```bash
+curl --data "trace;flow=debug" 127.0.0.1:4000/debug/log_level
+```
+And database will reply with something like:
+```bash
+Log Level changed from Some("info") to "trace;flow=debug"%
+```
+
+The data is a string in the format of `global_level;module1=level1;module2=level2;...` that follow the same rule of `RUST_LOG`. 
+
+The module is the module name of the log, and the level is the log level. The log level can be one of the following: `trace`, `debug`, `info`, `warn`, `error`, `off`(case insensitive).

grafana/greptimedb.json

@@ -409,7 +409,39 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "fieldMinMax": false,
           "mappings": [],
@@ -438,18 +470,16 @@
       },
       "id": 27,
       "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
-        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
       "pluginVersion": "10.2.3",
       "targets": [
@@ -467,7 +497,7 @@
         }
       ],
       "title": "CPU",
-      "type": "stat"
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -477,7 +507,39 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "decimals": 0,
           "fieldMinMax": false,
@@ -503,18 +565,16 @@
       },
       "id": 28,
       "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
-        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
       "pluginVersion": "10.2.3",
       "targets": [
@@ -532,7 +592,7 @@
         }
       ],
       "title": "Memory",
-      "type": "stat"
+      "type": "timeseries"
     },
     {
       "collapsed": false,
@@ -3335,6 +3395,6 @@
   "timezone": "",
   "title": "GreptimeDB",
   "uid": "e7097237-669b-4f8d-b751-13067afbfb68",
-  "version": 15,
+  "version": 16,
   "weekStart": ""
 }

rust-toolchain.toml

@@ -1,3 +1,2 @@
 [toolchain]
-channel = "nightly-2024-06-06"
-
+channel = "nightly-2024-10-19"

src/api/src/helper.rs

@@ -116,6 +116,7 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
                     ConcreteDataType::binary_datatype()
                 }
             }
+            ColumnDataType::Json => ConcreteDataType::json_datatype(),
             ColumnDataType::String => ConcreteDataType::string_datatype(),
             ColumnDataType::Date => ConcreteDataType::date_datatype(),
             ColumnDataType::Datetime => ConcreteDataType::datetime_datatype(),
@@ -417,6 +418,10 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
             decimal128_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
+        ColumnDataType::Json => Values {
+            string_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
     }
 }
 

src/auth/src/user_provider/static_user_provider.rs

@@ -33,7 +33,7 @@ impl StaticUserProvider {
             value: value.to_string(),
             msg: "StaticUserProviderOption must be in format `<option>:<value>`",
         })?;
-        return match mode {
+        match mode {
             "file" => {
                 let users = load_credential_from_file(content)?
                     .context(InvalidConfigSnafu {
@@ -58,7 +58,7 @@ impl StaticUserProvider {
                 msg: "StaticUserProviderOption must be in format `file:<path>` or `cmd:<values>`",
             }
                 .fail(),
-        };
+        }
     }
 }
 

src/catalog/src/system_schema/information_schema/region_statistics.rs

@@ -39,9 +39,12 @@ use crate::CatalogManager;
 const REGION_ID: &str = "region_id";
 const TABLE_ID: &str = "table_id";
 const REGION_NUMBER: &str = "region_number";
+const REGION_ROWS: &str = "region_rows";
+const DISK_SIZE: &str = "disk_size";
 const MEMTABLE_SIZE: &str = "memtable_size";
 const MANIFEST_SIZE: &str = "manifest_size";
 const SST_SIZE: &str = "sst_size";
+const INDEX_SIZE: &str = "index_size";
 const ENGINE: &str = "engine";
 const REGION_ROLE: &str = "region_role";
 
@@ -52,9 +55,12 @@ const INIT_CAPACITY: usize = 42;
 /// - `region_id`: The region id.
 /// - `table_id`: The table id.
 /// - `region_number`: The region number.
+/// - `region_rows`: The number of rows in region.
 /// - `memtable_size`: The memtable size in bytes.
+/// - `disk_size`: The approximate disk size in bytes.
 /// - `manifest_size`: The manifest size in bytes.
-/// - `sst_size`: The sst size in bytes.
+/// - `sst_size`: The sst data files size in bytes.
+/// - `index_size`: The sst index files size in bytes.
 /// - `engine`: The engine type.
 /// - `region_role`: The region role.
 ///
@@ -76,9 +82,12 @@ impl InformationSchemaRegionStatistics {
             ColumnSchema::new(REGION_ID, ConcreteDataType::uint64_datatype(), false),
             ColumnSchema::new(TABLE_ID, ConcreteDataType::uint32_datatype(), false),
             ColumnSchema::new(REGION_NUMBER, ConcreteDataType::uint32_datatype(), false),
+            ColumnSchema::new(REGION_ROWS, ConcreteDataType::uint64_datatype(), true),
+            ColumnSchema::new(DISK_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(MEMTABLE_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(MANIFEST_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(SST_SIZE, ConcreteDataType::uint64_datatype(), true),
+            ColumnSchema::new(INDEX_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(ENGINE, ConcreteDataType::string_datatype(), true),
             ColumnSchema::new(REGION_ROLE, ConcreteDataType::string_datatype(), true),
         ]))
@@ -135,9 +144,12 @@ struct InformationSchemaRegionStatisticsBuilder {
     region_ids: UInt64VectorBuilder,
     table_ids: UInt32VectorBuilder,
     region_numbers: UInt32VectorBuilder,
+    region_rows: UInt64VectorBuilder,
+    disk_sizes: UInt64VectorBuilder,
     memtable_sizes: UInt64VectorBuilder,
     manifest_sizes: UInt64VectorBuilder,
     sst_sizes: UInt64VectorBuilder,
+    index_sizes: UInt64VectorBuilder,
     engines: StringVectorBuilder,
     region_roles: StringVectorBuilder,
 }
@@ -150,9 +162,12 @@ impl InformationSchemaRegionStatisticsBuilder {
             region_ids: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             table_ids: UInt32VectorBuilder::with_capacity(INIT_CAPACITY),
             region_numbers: UInt32VectorBuilder::with_capacity(INIT_CAPACITY),
+            region_rows: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
+            disk_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             memtable_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             manifest_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             sst_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
+            index_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             engines: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             region_roles: StringVectorBuilder::with_capacity(INIT_CAPACITY),
         }
@@ -177,9 +192,12 @@ impl InformationSchemaRegionStatisticsBuilder {
             (REGION_ID, &Value::from(region_stat.id.as_u64())),
             (TABLE_ID, &Value::from(region_stat.id.table_id())),
             (REGION_NUMBER, &Value::from(region_stat.id.region_number())),
+            (REGION_ROWS, &Value::from(region_stat.num_rows)),
+            (DISK_SIZE, &Value::from(region_stat.approximate_bytes)),
             (MEMTABLE_SIZE, &Value::from(region_stat.memtable_size)),
             (MANIFEST_SIZE, &Value::from(region_stat.manifest_size)),
             (SST_SIZE, &Value::from(region_stat.sst_size)),
+            (INDEX_SIZE, &Value::from(region_stat.index_size)),
             (ENGINE, &Value::from(region_stat.engine.as_str())),
             (REGION_ROLE, &Value::from(region_stat.role.to_string())),
         ];
@@ -192,9 +210,12 @@ impl InformationSchemaRegionStatisticsBuilder {
         self.table_ids.push(Some(region_stat.id.table_id()));
         self.region_numbers
             .push(Some(region_stat.id.region_number()));
+        self.region_rows.push(Some(region_stat.num_rows));
+        self.disk_sizes.push(Some(region_stat.approximate_bytes));
         self.memtable_sizes.push(Some(region_stat.memtable_size));
         self.manifest_sizes.push(Some(region_stat.manifest_size));
         self.sst_sizes.push(Some(region_stat.sst_size));
+        self.index_sizes.push(Some(region_stat.index_size));
         self.engines.push(Some(&region_stat.engine));
         self.region_roles.push(Some(&region_stat.role.to_string()));
     }
@@ -204,9 +225,12 @@ impl InformationSchemaRegionStatisticsBuilder {
             Arc::new(self.region_ids.finish()),
             Arc::new(self.table_ids.finish()),
             Arc::new(self.region_numbers.finish()),
+            Arc::new(self.region_rows.finish()),
+            Arc::new(self.disk_sizes.finish()),
             Arc::new(self.memtable_sizes.finish()),
             Arc::new(self.manifest_sizes.finish()),
             Arc::new(self.sst_sizes.finish()),
+            Arc::new(self.index_sizes.finish()),
             Arc::new(self.engines.finish()),
             Arc::new(self.region_roles.finish()),
         ];

src/catalog/src/system_schema/information_schema/table_names.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/// All table names in `information_schema`.
+//! All table names in `information_schema`.
 
 pub const TABLES: &str = "tables";
 pub const COLUMNS: &str = "columns";

src/catalog/src/system_schema/memory_table.rs

@@ -74,7 +74,7 @@ impl MemoryTableBuilder {
     /// Construct the `information_schema.{table_name}` virtual table
     pub async fn memory_records(&mut self) -> Result<RecordBatch> {
         if self.columns.is_empty() {
-            RecordBatch::new_empty(self.schema.clone()).context(CreateRecordBatchSnafu)
+            Ok(RecordBatch::new_empty(self.schema.clone()))
         } else {
             RecordBatch::new(self.schema.clone(), std::mem::take(&mut self.columns))
                 .context(CreateRecordBatchSnafu)

src/catalog/src/system_schema/pg_catalog/pg_namespace.rs

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! The `pg_catalog.pg_namespace` table implementation.
+//! namespace is a schema in greptime
+
 pub(super) mod oid_map;
 
 use std::sync::{Arc, Weak};
@@ -40,9 +43,6 @@ use crate::system_schema::utils::tables::{string_column, u32_column};
 use crate::system_schema::SystemTable;
 use crate::CatalogManager;
 
-/// The `pg_catalog.pg_namespace` table implementation.
-/// namespace is a schema in greptime
-
 const NSPNAME: &str = "nspname";
 const INIT_CAPACITY: usize = 42;
 

src/client/Cargo.toml

@@ -28,7 +28,7 @@ enum_dispatch = "0.3"
 futures-util.workspace = true
 lazy_static.workspace = true
 moka = { workspace = true, features = ["future"] }
-parking_lot = "0.12"
+parking_lot.workspace = true
 prometheus.workspace = true
 prost.workspace = true
 query.workspace = true
@@ -45,7 +45,6 @@ common-grpc-expr.workspace = true
 datanode.workspace = true
 derive-new = "0.5"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [dev-dependencies.substrait_proto]
 package = "substrait"

src/cmd/Cargo.toml

@@ -78,7 +78,7 @@ table.workspace = true
 tokio.workspace = true
 toml.workspace = true
 tonic.workspace = true
-tracing-appender = "0.2"
+tracing-appender.workspace = true
 
 [target.'cfg(not(windows))'.dependencies]
 tikv-jemallocator = "0.6"

src/cmd/src/cli/repl.rs

@@ -174,7 +174,7 @@ impl Repl {
 
             let plan = query_engine
                 .planner()
-                .plan(stmt, query_ctx.clone())
+                .plan(&stmt, query_ctx.clone())
                 .await
                 .context(PlanStatementSnafu)?;
 

src/cmd/src/datanode.rs

@@ -272,9 +272,10 @@ impl StartCommand {
         info!("Datanode start command: {:#?}", self);
         info!("Datanode options: {:#?}", opts);
 
+        let plugin_opts = opts.plugins;
         let opts = opts.component;
         let mut plugins = Plugins::new();
-        plugins::setup_datanode_plugins(&mut plugins, &opts)
+        plugins::setup_datanode_plugins(&mut plugins, &plugin_opts, &opts)
             .await
             .context(StartDatanodeSnafu)?;
 

src/cmd/src/frontend.rs

@@ -266,9 +266,10 @@ impl StartCommand {
         info!("Frontend start command: {:#?}", self);
         info!("Frontend options: {:#?}", opts);
 
+        let plugin_opts = opts.plugins;
         let opts = opts.component;
         let mut plugins = Plugins::new();
-        plugins::setup_frontend_plugins(&mut plugins, &opts)
+        plugins::setup_frontend_plugins(&mut plugins, &plugin_opts, &opts)
             .await
             .context(StartFrontendSnafu)?;
 
@@ -342,6 +343,8 @@ impl StartCommand {
         // Some queries are expected to take long time.
         let channel_config = ChannelConfig {
             timeout: None,
+            tcp_nodelay: opts.datanode.client.tcp_nodelay,
+            connect_timeout: Some(opts.datanode.client.connect_timeout),
             ..Default::default()
         };
         let client = NodeClients::new(channel_config);
@@ -472,7 +475,7 @@ mod tests {
         };
 
         let mut plugins = Plugins::new();
-        plugins::setup_frontend_plugins(&mut plugins, &fe_opts)
+        plugins::setup_frontend_plugins(&mut plugins, &[], &fe_opts)
             .await
             .unwrap();
 

src/cmd/src/lib.rs

@@ -84,6 +84,7 @@ pub trait App: Send {
 }
 
 /// Log the versions of the application, and the arguments passed to the cli.
+///
 /// `version` should be the same as the output of cli "--version";
 /// and the `short_version` is the short version of the codes, often consist of git branch and commit.
 pub fn log_versions(version: &str, short_version: &str, app: &str) {

src/cmd/src/metasrv.rs

@@ -273,9 +273,10 @@ impl StartCommand {
         info!("Metasrv start command: {:#?}", self);
         info!("Metasrv options: {:#?}", opts);
 
+        let plugin_opts = opts.plugins;
         let opts = opts.component;
         let mut plugins = Plugins::new();
-        plugins::setup_metasrv_plugins(&mut plugins, &opts)
+        plugins::setup_metasrv_plugins(&mut plugins, &plugin_opts, &opts)
             .await
             .context(StartMetaServerSnafu)?;
 

src/cmd/src/options.rs

@@ -15,6 +15,7 @@
 use clap::Parser;
 use common_config::Configurable;
 use common_runtime::global::RuntimeOptions;
+use plugins::PluginOptions;
 use serde::{Deserialize, Serialize};
 
 #[derive(Parser, Default, Debug, Clone)]
@@ -40,6 +41,8 @@ pub struct GlobalOptions {
 pub struct GreptimeOptions<T> {
     /// The runtime options.
     pub runtime: RuntimeOptions,
+    /// The plugin options.
+    pub plugins: Vec<PluginOptions>,
 
     /// The options of each component (like Datanode or Standalone) of GreptimeDB.
     #[serde(flatten)]

src/cmd/src/standalone.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::net::SocketAddr;
 use std::sync::Arc;
 use std::{fs, path};
 
@@ -250,6 +251,13 @@ pub struct Instance {
     _guard: Vec<WorkerGuard>,
 }
 
+impl Instance {
+    /// Find the socket addr of a server by its `name`.
+    pub async fn server_addr(&self, name: &str) -> Option<SocketAddr> {
+        self.frontend.server_handlers().addr(name).await
+    }
+}
+
 #[async_trait]
 impl App for Instance {
     fn name(&self) -> &str {
@@ -340,7 +348,8 @@ pub struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(
+    /// Load the GreptimeDB options from various sources (command line, config file or env).
+    pub fn load_options(
         &self,
         global_options: &GlobalOptions,
     ) -> Result<GreptimeOptions<StandaloneOptions>> {
@@ -430,7 +439,8 @@ impl StartCommand {
     #[allow(unreachable_code)]
     #[allow(unused_variables)]
     #[allow(clippy::diverging_sub_expression)]
-    async fn build(&self, opts: GreptimeOptions<StandaloneOptions>) -> Result<Instance> {
+    /// Build GreptimeDB instance with the loaded options.
+    pub async fn build(&self, opts: GreptimeOptions<StandaloneOptions>) -> Result<Instance> {
         common_runtime::init_global_runtimes(&opts.runtime);
 
         let guard = common_telemetry::init_global_logging(
@@ -445,15 +455,16 @@ impl StartCommand {
         info!("Standalone options: {opts:#?}");
 
         let mut plugins = Plugins::new();
+        let plugin_opts = opts.plugins;
         let opts = opts.component;
         let fe_opts = opts.frontend_options();
         let dn_opts = opts.datanode_options();
 
-        plugins::setup_frontend_plugins(&mut plugins, &fe_opts)
+        plugins::setup_frontend_plugins(&mut plugins, &plugin_opts, &fe_opts)
             .await
             .context(StartFrontendSnafu)?;
 
-        plugins::setup_datanode_plugins(&mut plugins, &dn_opts)
+        plugins::setup_datanode_plugins(&mut plugins, &plugin_opts, &dn_opts)
             .await
             .context(StartDatanodeSnafu)?;
 
@@ -725,12 +736,14 @@ impl InformationExtension for StandaloneInformationExtension {
                     id: stat.region_id,
                     rcus: 0,
                     wcus: 0,
-                    approximate_bytes: region_stat.estimated_disk_size() as i64,
+                    approximate_bytes: region_stat.estimated_disk_size(),
                     engine: stat.engine,
                     role: RegionRole::from(stat.role).into(),
+                    num_rows: region_stat.num_rows,
                     memtable_size: region_stat.memtable_size,
                     manifest_size: region_stat.manifest_size,
                     sst_size: region_stat.sst_size,
+                    index_size: region_stat.index_size,
                 }
             })
             .collect::<Vec<_>>();
@@ -762,7 +775,7 @@ mod tests {
         };
 
         let mut plugins = Plugins::new();
-        plugins::setup_frontend_plugins(&mut plugins, &fe_opts)
+        plugins::setup_frontend_plugins(&mut plugins, &[], &fe_opts)
             .await
             .unwrap();
 

src/cmd/tests/load_config_test.rs

@@ -20,7 +20,7 @@ use common_config::Configurable;
 use common_grpc::channel_manager::{
     DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
 };
-use common_telemetry::logging::{LoggingOptions, DEFAULT_OTLP_ENDPOINT};
+use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, DEFAULT_OTLP_ENDPOINT};
 use common_wal::config::raft_engine::RaftEngineConfig;
 use common_wal::config::DatanodeWalConfig;
 use datanode::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
@@ -159,8 +159,20 @@ fn test_load_metasrv_example_config() {
                 level: Some("info".to_string()),
                 otlp_endpoint: Some(DEFAULT_OTLP_ENDPOINT.to_string()),
                 tracing_sample_ratio: Some(Default::default()),
+                slow_query: SlowQueryOptions {
+                    enable: false,
+                    threshold: Some(Duration::from_secs(10)),
+                    sample_ratio: Some(1.0),
+                },
                 ..Default::default()
             },
+            datanode: meta_srv::metasrv::DatanodeOptions {
+                client: meta_srv::metasrv::DatanodeClientOptions {
+                    timeout: Duration::from_secs(10),
+                    connect_timeout: Duration::from_secs(10),
+                    tcp_nodelay: true,
+                },
+            },
             export_metrics: ExportMetricsOption {
                 self_import: Some(Default::default()),
                 remote_write: Some(Default::default()),

src/common/base/src/secrets.rs

@@ -46,8 +46,9 @@ impl From<String> for SecretString {
     }
 }
 
-/// Wrapper type for values that contains secrets, which attempts to limit
-/// accidental exposure and ensure secrets are wiped from memory when dropped.
+/// Wrapper type for values that contains secrets.
+///
+/// It attempts to limit accidental exposure and ensure secrets are wiped from memory when dropped.
 /// (e.g. passwords, cryptographic keys, access tokens or other credentials)
 ///
 /// Access to the secret inner value occurs through the [`ExposeSecret`]

src/common/catalog/src/consts.rs

@@ -103,14 +103,15 @@ pub const INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID: u32 = 34;
 /// id for information_schema.region_statistics
 pub const INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID: u32 = 35;
 
-/// ----- End of information_schema tables -----
+// ----- End of information_schema tables -----
 
 /// ----- Begin of pg_catalog tables -----
 pub const PG_CATALOG_PG_CLASS_TABLE_ID: u32 = 256;
 pub const PG_CATALOG_PG_TYPE_TABLE_ID: u32 = 257;
 pub const PG_CATALOG_PG_NAMESPACE_TABLE_ID: u32 = 258;
 
-/// ----- End of pg_catalog tables -----
+// ----- End of pg_catalog tables -----
+
 pub const MITO_ENGINE: &str = "mito";
 pub const MITO2_ENGINE: &str = "mito2";
 pub const METRIC_ENGINE: &str = "metric";

src/common/function/src/scalars/aggregate.rs

@@ -31,7 +31,6 @@ pub use polyval::PolyvalAccumulatorCreator;
 pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;
 
-use super::geo::encoding::JsonPathEncodeFunctionCreator;
 use crate::function_registry::FunctionRegistry;
 
 /// A function creates `AggregateFunctionCreator`.
@@ -93,6 +92,11 @@ impl AggregateFunctions {
         register_aggr_func!("scipystatsnormcdf", 2, ScipyStatsNormCdfAccumulatorCreator);
         register_aggr_func!("scipystatsnormpdf", 2, ScipyStatsNormPdfAccumulatorCreator);
 
-        register_aggr_func!("json_encode_path", 3, JsonPathEncodeFunctionCreator);
+        #[cfg(feature = "geo")]
+        register_aggr_func!(
+            "json_encode_path",
+            3,
+            super::geo::encoding::JsonPathEncodeFunctionCreator
+        );
     }
 }

src/common/function/src/scalars/json.rs

@@ -16,6 +16,7 @@ use std::sync::Arc;
 mod json_get;
 mod json_is;
 mod json_path_exists;
+mod json_path_match;
 mod json_to_string;
 mod parse_json;
 
@@ -49,5 +50,6 @@ impl JsonFunction {
         registry.register(Arc::new(JsonIsObject));
 
         registry.register(Arc::new(json_path_exists::JsonPathExistsFunction));
+        registry.register(Arc::new(json_path_match::JsonPathMatchFunction));
     }
 }

src/common/function/src/scalars/json/json_path_match.rs

@@ -0,0 +1,202 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{self, Display};
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datafusion::logical_expr::Volatility;
+use datatypes::data_type::ConcreteDataType;
+use datatypes::prelude::VectorRef;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BooleanVectorBuilder, MutableVector};
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+
+/// Check if the given JSON data match the given JSON path's predicate.
+#[derive(Clone, Debug, Default)]
+pub struct JsonPathMatchFunction;
+
+const NAME: &str = "json_path_match";
+
+impl Function for JsonPathMatchFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::boolean_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::exact(
+            vec![
+                ConcreteDataType::json_datatype(),
+                ConcreteDataType::string_datatype(),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+        let jsons = &columns[0];
+        let paths = &columns[1];
+
+        let size = jsons.len();
+        let mut results = BooleanVectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let json = jsons.get_ref(i);
+            let path = paths.get_ref(i);
+
+            match json.data_type() {
+                // JSON data type uses binary vector
+                ConcreteDataType::Binary(_) => {
+                    let json = json.as_binary();
+                    let path = path.as_string();
+                    let result = match (json, path) {
+                        (Ok(Some(json)), Ok(Some(path))) => {
+                            if !jsonb::is_null(json) {
+                                let json_path = jsonb::jsonpath::parse_json_path(path.as_bytes());
+                                match json_path {
+                                    Ok(json_path) => jsonb::path_match(json, json_path).ok(),
+                                    Err(_) => None,
+                                }
+                            } else {
+                                None
+                            }
+                        }
+                        _ => None,
+                    };
+
+                    results.push(result);
+                }
+
+                _ => {
+                    return UnsupportedInputDataTypeSnafu {
+                        function: NAME,
+                        datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+                    }
+                    .fail();
+                }
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+impl Display for JsonPathMatchFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "JSON_PATH_MATCH")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::TypeSignature;
+    use datatypes::vectors::{BinaryVector, StringVector};
+
+    use super::*;
+
+    #[test]
+    fn test_json_path_match_function() {
+        let json_path_match = JsonPathMatchFunction;
+
+        assert_eq!("json_path_match", json_path_match.name());
+        assert_eq!(
+            ConcreteDataType::boolean_datatype(),
+            json_path_match
+                .return_type(&[ConcreteDataType::json_datatype()])
+                .unwrap()
+        );
+
+        assert!(matches!(json_path_match.signature(),
+                         Signature {
+                             type_signature: TypeSignature::Exact(valid_types),
+                             volatility: Volatility::Immutable
+                         } if valid_types == vec![ConcreteDataType::json_datatype(), ConcreteDataType::string_datatype()],
+        ));
+
+        let json_strings = [
+            Some(r#"{"a": {"b": 2}, "b": 2, "c": 3}"#.to_string()),
+            Some(r#"{"a": 1, "b": [1,2,3]}"#.to_string()),
+            Some(r#"{"a": 1 ,"b": [1,2,3]}"#.to_string()),
+            Some(r#"[1,2,3]"#.to_string()),
+            Some(r#"{"a":1,"b":[1,2,3]}"#.to_string()),
+            Some(r#"null"#.to_string()),
+            Some(r#"null"#.to_string()),
+        ];
+
+        let paths = vec![
+            Some("$.a.b == 2".to_string()),
+            Some("$.b[1 to last] >= 2".to_string()),
+            Some("$.c > 0".to_string()),
+            Some("$[0 to last] > 0".to_string()),
+            Some(r#"null"#.to_string()),
+            Some("$.c > 0".to_string()),
+            Some(r#"null"#.to_string()),
+        ];
+
+        let results = [
+            Some(true),
+            Some(true),
+            Some(false),
+            Some(true),
+            None,
+            None,
+            None,
+        ];
+
+        let jsonbs = json_strings
+            .into_iter()
+            .map(|s| s.map(|json| jsonb::parse_value(json.as_bytes()).unwrap().to_vec()))
+            .collect::<Vec<_>>();
+
+        let json_vector = BinaryVector::from(jsonbs);
+        let path_vector = StringVector::from(paths);
+        let args: Vec<VectorRef> = vec![Arc::new(json_vector), Arc::new(path_vector)];
+        let vector = json_path_match
+            .eval(FunctionContext::default(), &args)
+            .unwrap();
+
+        assert_eq!(7, vector.len());
+        for (i, expected) in results.iter().enumerate() {
+            let result = vector.get_ref(i);
+
+            match expected {
+                Some(expected_value) => {
+                    assert!(!result.is_null());
+                    let result_value = result.as_boolean().unwrap().unwrap();
+                    assert_eq!(*expected_value, result_value);
+                }
+                None => {
+                    assert!(result.is_null());
+                }
+            }
+        }
+    }
+}

src/common/greptimedb-telemetry/src/lib.rs

@@ -199,6 +199,7 @@ pub fn default_get_uuid(working_home: &Option<String>) -> Option<String> {
 }
 
 /// Report version info to GreptimeDB.
+///
 /// We do not collect any identity-sensitive information.
 /// This task is scheduled to run every 30 minutes.
 /// The task will be disabled default. It can be enabled by setting the build feature `greptimedb-telemetry`
@@ -324,7 +325,7 @@ mod tests {
             });
             let addr = ([127, 0, 0, 1], port).into();
 
-            let server = Server::bind(&addr).serve(make_svc);
+            let server = Server::try_bind(&addr).unwrap().serve(make_svc);
             let graceful = server.with_graceful_shutdown(async {
                 rx.await.ok();
             });

src/common/grpc-expr/Cargo.toml

@@ -18,6 +18,7 @@ common-time.workspace = true
 datatypes.workspace = true
 prost.workspace = true
 snafu.workspace = true
+store-api.workspace = true
 table.workspace = true
 
 [dev-dependencies]

src/common/grpc-expr/src/alter.rs

@@ -22,12 +22,13 @@ use api::v1::{
 use common_query::AddColumnLocation;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use snafu::{ensure, OptionExt, ResultExt};
+use store_api::region_request::ChangeOption;
 use table::metadata::TableId;
 use table::requests::{AddColumnRequest, AlterKind, AlterTableRequest, ChangeColumnTypeRequest};
 
 use crate::error::{
-    InvalidColumnDefSnafu, MissingFieldSnafu, MissingTimestampColumnSnafu, Result,
-    UnknownLocationTypeSnafu,
+    InvalidChangeTableOptionRequestSnafu, InvalidColumnDefSnafu, MissingFieldSnafu,
+    MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu,
 };
 
 const LOCATION_TYPE_FIRST: i32 = LocationType::First as i32;
@@ -92,6 +93,15 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterExpr) -> Result<Alter
         Kind::RenameTable(RenameTable { new_table_name }) => {
             AlterKind::RenameTable { new_table_name }
         }
+        Kind::ChangeTableOptions(api::v1::ChangeTableOptions {
+            change_table_options,
+        }) => AlterKind::ChangeTableOptions {
+            options: change_table_options
+                .iter()
+                .map(ChangeOption::try_from)
+                .collect::<std::result::Result<Vec<_>, _>>()
+                .context(InvalidChangeTableOptionRequestSnafu)?,
+        },
     };
 
     let request = AlterTableRequest {

src/common/grpc-expr/src/error.rs

@@ -19,6 +19,7 @@ use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
+use store_api::metadata::MetadataError;
 
 #[derive(Snafu)]
 #[snafu(visibility(pub))]
@@ -118,6 +119,12 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Invalid change table option request"))]
+    InvalidChangeTableOptionRequest {
+        #[snafu(source)]
+        error: MetadataError,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -141,6 +148,7 @@ impl ErrorExt for Error {
             Error::UnknownColumnDataType { .. } | Error::InvalidFulltextColumnType { .. } => {
                 StatusCode::InvalidArguments
             }
+            Error::InvalidChangeTableOptionRequest { .. } => StatusCode::InvalidArguments,
         }
     }
 

src/common/macro/src/lib.rs

@@ -35,7 +35,9 @@ pub fn aggr_func_type_store_derive(input: TokenStream) -> TokenStream {
 }
 
 /// A struct can be used as a creator for aggregate function if it has been annotated with this
-/// attribute first. This attribute add a necessary field which is intended to store the input
+/// attribute first.
+///
+/// This attribute add a necessary field which is intended to store the input
 /// data's types to the struct.
 /// This attribute is expected to be used along with derive macro [AggrFuncTypeStore].
 #[proc_macro_attribute]
@@ -44,9 +46,10 @@ pub fn as_aggr_func_creator(args: TokenStream, input: TokenStream) -> TokenStrea
 }
 
 /// Attribute macro to convert an arithimetic function to a range function. The annotated function
-/// should accept servaral arrays as input and return a single value as output. This procedure
-/// macro can works on any number of input parameters. Return type can be either primitive type
-/// or wrapped in `Option`.
+/// should accept servaral arrays as input and return a single value as output.
+///
+/// This procedure macro can works on any number of input parameters. Return type can be either
+/// primitive type or wrapped in `Option`.
 ///
 /// # Example
 /// Take `count_over_time()` in PromQL as an example:

src/common/meta/Cargo.toml

@@ -60,7 +60,7 @@ table.workspace = true
 tokio.workspace = true
 tokio-postgres = { workspace = true, optional = true }
 tonic.workspace = true
-typetag = "0.2"
+typetag.workspace = true
 
 [dev-dependencies]
 chrono.workspace = true

src/common/meta/src/cluster.rs

@@ -55,6 +55,7 @@ pub trait ClusterInfo {
 }
 
 /// The key of [NodeInfo] in the storage. The format is `__meta_cluster_node_info-{cluster_id}-{role}-{node_id}`.
+///
 /// This key cannot be used to describe the `Metasrv` because the `Metasrv` does not have
 /// a `cluster_id`, it serves multiple clusters.
 #[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]

src/common/meta/src/datanode.rs

@@ -78,17 +78,21 @@ pub struct RegionStat {
     /// The write capacity units during this period
     pub wcus: i64,
     /// Approximate bytes of this region
-    pub approximate_bytes: i64,
+    pub approximate_bytes: u64,
     /// The engine name.
     pub engine: String,
     /// The region role.
     pub role: RegionRole,
+    /// The number of rows
+    pub num_rows: u64,
     /// The size of the memtable in bytes.
     pub memtable_size: u64,
     /// The size of the manifest in bytes.
     pub manifest_size: u64,
-    /// The size of the SST files in bytes.
+    /// The size of the SST data files in bytes.
     pub sst_size: u64,
+    /// The size of the SST index files in bytes.
+    pub index_size: u64,
 }
 
 impl Stat {
@@ -178,12 +182,14 @@ impl From<&api::v1::meta::RegionStat> for RegionStat {
             id: RegionId::from_u64(value.region_id),
             rcus: value.rcus,
             wcus: value.wcus,
-            approximate_bytes: value.approximate_bytes,
+            approximate_bytes: value.approximate_bytes as u64,
             engine: value.engine.to_string(),
             role: RegionRole::from(value.role()),
+            num_rows: region_stat.num_rows,
             memtable_size: region_stat.memtable_size,
             manifest_size: region_stat.manifest_size,
             sst_size: region_stat.sst_size,
+            index_size: region_stat.index_size,
         }
     }
 }

src/common/meta/src/ddl/alter_logical_tables/update_metadata.rs

@@ -43,10 +43,10 @@ impl AlterLogicalTablesProcedure {
             &self.data.physical_columns,
         );
 
-        // Updates physical table's metadata
+        // Updates physical table's metadata, and we don't need to touch per-region settings.
         self.context
             .table_metadata_manager
-            .update_table_info(physical_table_info, new_raw_table_info)
+            .update_table_info(physical_table_info, None, new_raw_table_info)
             .await?;
 
         Ok(())

src/common/meta/src/ddl/alter_table.rs

@@ -43,10 +43,10 @@ use crate::ddl::DdlContext;
 use crate::error::{Error, Result};
 use crate::instruction::CacheIdent;
 use crate::key::table_info::TableInfoValue;
-use crate::key::DeserializedValueWithBytes;
+use crate::key::{DeserializedValueWithBytes, RegionDistribution};
 use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
 use crate::rpc::ddl::AlterTableTask;
-use crate::rpc::router::{find_leader_regions, find_leaders};
+use crate::rpc::router::{find_leader_regions, find_leaders, region_distribution};
 use crate::{metrics, ClusterId};
 
 /// The alter table procedure
@@ -101,6 +101,9 @@ impl AlterTableProcedure {
             .get_physical_table_route(table_id)
             .await?;
 
+        self.data.region_distribution =
+            Some(region_distribution(&physical_table_route.region_routes));
+
         let leaders = find_leaders(&physical_table_route.region_routes);
         let mut alter_region_tasks = Vec::with_capacity(leaders.len());
 
@@ -161,8 +164,14 @@ impl AlterTableProcedure {
             self.on_update_metadata_for_rename(new_table_name.to_string(), table_info_value)
                 .await?;
         } else {
-            self.on_update_metadata_for_alter(new_info.into(), table_info_value)
-                .await?;
+            // region distribution is set in submit_alter_region_requests
+            let region_distribution = self.data.region_distribution.as_ref().unwrap().clone();
+            self.on_update_metadata_for_alter(
+                new_info.into(),
+                region_distribution,
+                table_info_value,
+            )
+            .await?;
         }
 
         info!("Updated table metadata for table {table_ref}, table_id: {table_id}");
@@ -271,6 +280,8 @@ pub struct AlterTableData {
     table_id: TableId,
     /// Table info value before alteration.
     table_info_value: Option<DeserializedValueWithBytes<TableInfoValue>>,
+    /// Region distribution for table in case we need to update region options.
+    region_distribution: Option<RegionDistribution>,
 }
 
 impl AlterTableData {
@@ -281,6 +292,7 @@ impl AlterTableData {
             table_id,
             cluster_id,
             table_info_value: None,
+            region_distribution: None,
         }
     }
 

src/common/meta/src/ddl/alter_table/region_request.rs

@@ -106,6 +106,7 @@ fn create_proto_alter_kind(
             })))
         }
         Kind::RenameTable(_) => Ok(None),
+        Kind::ChangeTableOptions(v) => Ok(Some(alter_request::Kind::ChangeTableOptions(v.clone()))),
     }
 }
 

src/common/meta/src/ddl/alter_table/update_metadata.rs

@@ -20,7 +20,7 @@ use table::requests::AlterKind;
 use crate::ddl::alter_table::AlterTableProcedure;
 use crate::error::{self, Result};
 use crate::key::table_info::TableInfoValue;
-use crate::key::DeserializedValueWithBytes;
+use crate::key::{DeserializedValueWithBytes, RegionDistribution};
 
 impl AlterTableProcedure {
     /// Builds new_meta
@@ -51,7 +51,9 @@ impl AlterTableProcedure {
             AlterKind::RenameTable { new_table_name } => {
                 new_info.name = new_table_name.to_string();
             }
-            AlterKind::DropColumns { .. } | AlterKind::ChangeColumnTypes { .. } => {}
+            AlterKind::DropColumns { .. }
+            | AlterKind::ChangeColumnTypes { .. }
+            | AlterKind::ChangeTableOptions { .. } => {}
         }
 
         Ok(new_info)
@@ -75,11 +77,16 @@ impl AlterTableProcedure {
     pub(crate) async fn on_update_metadata_for_alter(
         &self,
         new_table_info: RawTableInfo,
+        region_distribution: RegionDistribution,
         current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
     ) -> Result<()> {
         let table_metadata_manager = &self.context.table_metadata_manager;
         table_metadata_manager
-            .update_table_info(current_table_info_value, new_table_info)
+            .update_table_info(
+                current_table_info_value,
+                Some(region_distribution),
+                new_table_info,
+            )
             .await?;
 
         Ok(())

src/common/meta/src/ddl/create_logical_tables/update_metadata.rs

@@ -58,10 +58,10 @@ impl CreateLogicalTablesProcedure {
             &new_table_info.name,
         );
 
-        // Update physical table's metadata
+        // Update physical table's metadata and we don't need to touch per-region settings.
         self.context
             .table_metadata_manager
-            .update_table_info(&physical_table_info, new_table_info)
+            .update_table_info(&physical_table_info, None, new_table_info)
             .await?;
 
         // Invalid physical table cache

src/common/meta/src/ddl/test_util/datanode_handler.rs

@@ -29,7 +29,10 @@ use crate::test_util::MockDatanodeHandler;
 #[async_trait::async_trait]
 impl MockDatanodeHandler for () {
     async fn handle(&self, _peer: &Peer, _request: RegionRequest) -> Result<RegionResponse> {
-        unreachable!()
+        Ok(RegionResponse {
+            affected_rows: 0,
+            extensions: Default::default(),
+        })
     }
 
     async fn handle_query(

src/common/meta/src/ddl/tests/alter_table.rs

@@ -19,13 +19,14 @@ use std::sync::Arc;
 use api::v1::alter_expr::Kind;
 use api::v1::region::{region_request, RegionRequest};
 use api::v1::{
-    AddColumn, AddColumns, AlterExpr, ColumnDataType, ColumnDef as PbColumnDef, DropColumn,
-    DropColumns, SemanticType,
+    AddColumn, AddColumns, AlterExpr, ChangeTableOption, ChangeTableOptions, ColumnDataType,
+    ColumnDef as PbColumnDef, DropColumn, DropColumns, SemanticType,
 };
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use store_api::storage::RegionId;
+use table::requests::TTL_KEY;
 use tokio::sync::mpsc::{self};
 
 use crate::ddl::alter_table::AlterTableProcedure;
@@ -34,6 +35,7 @@ use crate::ddl::test_util::create_table::test_create_table_task;
 use crate::ddl::test_util::datanode_handler::{
     DatanodeWatcher, RequestOutdatedErrorDatanodeHandler,
 };
+use crate::key::datanode_table::DatanodeTableKey;
 use crate::key::table_name::TableNameKey;
 use crate::key::table_route::TableRouteValue;
 use crate::peer::Peer;
@@ -293,12 +295,21 @@ async fn test_on_update_metadata_add_columns() {
     let table_name = "foo";
     let table_id = 1024;
     let task = test_create_table_task(table_name, table_id);
+
+    let region_id = RegionId::new(table_id, 0);
+    let mock_table_routes = vec![RegionRoute {
+        region: Region::new_test(region_id),
+        leader_peer: Some(Peer::default()),
+        follower_peers: vec![],
+        leader_state: None,
+        leader_down_since: None,
+    }];
     // Puts a value to table name key.
     ddl_context
         .table_metadata_manager
         .create_table_metadata(
             task.table_info.clone(),
-            TableRouteValue::physical(vec![]),
+            TableRouteValue::physical(mock_table_routes),
             HashMap::new(),
         )
         .await
@@ -326,6 +337,7 @@ async fn test_on_update_metadata_add_columns() {
     let mut procedure =
         AlterTableProcedure::new(cluster_id, table_id, task, ddl_context.clone()).unwrap();
     procedure.on_prepare().await.unwrap();
+    procedure.submit_alter_region_requests().await.unwrap();
     procedure.on_update_metadata().await.unwrap();
 
     let table_info = ddl_context
@@ -343,3 +355,76 @@ async fn test_on_update_metadata_add_columns() {
         table_info.meta.next_column_id
     );
 }
+
+#[tokio::test]
+async fn test_on_update_table_options() {
+    let node_manager = Arc::new(MockDatanodeManager::new(()));
+    let ddl_context = new_ddl_context(node_manager);
+    let cluster_id = 1;
+    let table_name = "foo";
+    let table_id = 1024;
+    let task = test_create_table_task(table_name, table_id);
+
+    let region_id = RegionId::new(table_id, 0);
+    let mock_table_routes = vec![RegionRoute {
+        region: Region::new_test(region_id),
+        leader_peer: Some(Peer::default()),
+        follower_peers: vec![],
+        leader_state: None,
+        leader_down_since: None,
+    }];
+    // Puts a value to table name key.
+    ddl_context
+        .table_metadata_manager
+        .create_table_metadata(
+            task.table_info.clone(),
+            TableRouteValue::physical(mock_table_routes),
+            HashMap::new(),
+        )
+        .await
+        .unwrap();
+
+    let task = AlterTableTask {
+        alter_table: AlterExpr {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            kind: Some(Kind::ChangeTableOptions(ChangeTableOptions {
+                change_table_options: vec![ChangeTableOption {
+                    key: TTL_KEY.to_string(),
+                    value: "1d".to_string(),
+                }],
+            })),
+        },
+    };
+    let mut procedure =
+        AlterTableProcedure::new(cluster_id, table_id, task, ddl_context.clone()).unwrap();
+    procedure.on_prepare().await.unwrap();
+    procedure.submit_alter_region_requests().await.unwrap();
+    procedure.on_update_metadata().await.unwrap();
+
+    let table_info = ddl_context
+        .table_metadata_manager
+        .table_info_manager()
+        .get(table_id)
+        .await
+        .unwrap()
+        .unwrap()
+        .into_inner()
+        .table_info;
+
+    let datanode_key = DatanodeTableKey::new(0, table_id);
+    let region_info = ddl_context
+        .table_metadata_manager
+        .datanode_table_manager()
+        .get(&datanode_key)
+        .await
+        .unwrap()
+        .unwrap()
+        .region_info;
+
+    assert_eq!(
+        region_info.region_options,
+        HashMap::from(&table_info.meta.options)
+    );
+}

src/common/meta/src/error.rs

@@ -652,6 +652,18 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display(
+        "Datanode table info not found, table id: {}, datanode id: {}",
+        table_id,
+        datanode_id
+    ))]
+    DatanodeTableInfoNotFound {
+        datanode_id: DatanodeId,
+        table_id: TableId,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -752,6 +764,7 @@ impl ErrorExt for Error {
             PostgresExecution { .. } => StatusCode::Internal,
             #[cfg(feature = "pg_kvbackend")]
             ConnectPostgres { .. } => StatusCode::Internal,
+            Error::DatanodeTableInfoNotFound { .. } => StatusCode::Internal,
         }
     }
 

src/common/meta/src/key.rs

@@ -133,7 +133,6 @@ use self::flow::flow_name::FlowNameValue;
 use self::schema_name::{SchemaManager, SchemaNameKey, SchemaNameValue};
 use self::table_route::{TableRouteManager, TableRouteValue};
 use self::tombstone::TombstoneManager;
-use crate::ddl::utils::region_storage_path;
 use crate::error::{self, Result, SerdeJsonSnafu};
 use crate::key::node_address::NodeAddressValue;
 use crate::key::table_route::TableRouteKey;
@@ -593,8 +592,6 @@ impl TableMetadataManager {
         table_info.meta.region_numbers = region_numbers;
         let table_id = table_info.ident.table_id;
         let engine = table_info.meta.engine.clone();
-        let region_storage_path =
-            region_storage_path(&table_info.catalog_name, &table_info.schema_name);
 
         // Creates table name.
         let table_name = TableNameKey::new(
@@ -606,7 +603,7 @@ impl TableMetadataManager {
             .table_name_manager()
             .build_create_txn(&table_name, table_id)?;
 
-        let region_options = (&table_info.meta.options).into();
+        let region_options = table_info.to_region_options();
         // Creates table info.
         let table_info_value = TableInfoValue::new(table_info);
         let (create_table_info_txn, on_create_table_info_failure) = self
@@ -625,6 +622,7 @@ impl TableMetadataManager {
         ]);
 
         if let TableRouteValue::Physical(x) = &table_route_value {
+            let region_storage_path = table_info_value.region_storage_path();
             let create_datanode_table_txn = self.datanode_table_manager().build_create_txn(
                 table_id,
                 &engine,
@@ -926,13 +924,15 @@ impl TableMetadataManager {
     }
 
     /// Updates table info and returns an error if different metadata exists.
+    /// And cascade-ly update all redundant table options for each region
+    /// if region_distribution is present.
     pub async fn update_table_info(
         &self,
         current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
+        region_distribution: Option<RegionDistribution>,
         new_table_info: RawTableInfo,
     ) -> Result<()> {
         let table_id = current_table_info_value.table_info.ident.table_id;
-
         let new_table_info_value = current_table_info_value.update(new_table_info);
 
         // Updates table info.
@@ -940,8 +940,19 @@ impl TableMetadataManager {
             .table_info_manager()
             .build_update_txn(table_id, current_table_info_value, &new_table_info_value)?;
 
-        let mut r = self.kv_backend.txn(update_table_info_txn).await?;
+        let txn = if let Some(region_distribution) = region_distribution {
+            // region options induced from table info.
+            let new_region_options = new_table_info_value.table_info.to_region_options();
+            let update_datanode_table_options_txn = self
+                .datanode_table_manager
+                .build_update_table_options_txn(table_id, region_distribution, new_region_options)
+                .await?;
+            Txn::merge_all([update_table_info_txn, update_datanode_table_options_txn])
+        } else {
+            update_table_info_txn
+        };
 
+        let mut r = self.kv_backend.txn(txn).await?;
         // Checks whether metadata was already updated.
         if !r.succeeded {
             let mut set = TxnOpGetResponseSet::from(&mut r.responses);
@@ -1669,12 +1680,12 @@ mod tests {
             DeserializedValueWithBytes::from_inner(TableInfoValue::new(table_info.clone()));
         // should be ok.
         table_metadata_manager
-            .update_table_info(&current_table_info_value, new_table_info.clone())
+            .update_table_info(&current_table_info_value, None, new_table_info.clone())
             .await
             .unwrap();
         // if table info was updated, it should be ok.
         table_metadata_manager
-            .update_table_info(&current_table_info_value, new_table_info.clone())
+            .update_table_info(&current_table_info_value, None, new_table_info.clone())
             .await
             .unwrap();
 
@@ -1696,7 +1707,7 @@ mod tests {
         // if the current_table_info_value is wrong, it should return an error.
         // The ABA problem.
         assert!(table_metadata_manager
-            .update_table_info(&wrong_table_info_value, new_table_info)
+            .update_table_info(&wrong_table_info_value, None, new_table_info)
             .await
             .is_err())
     }

src/common/meta/src/key/catalog_name.rs

@@ -35,7 +35,7 @@ pub struct CatalogNameKey<'a> {
     pub catalog: &'a str,
 }
 
-impl<'a> Default for CatalogNameKey<'a> {
+impl Default for CatalogNameKey<'_> {
     fn default() -> Self {
         Self {
             catalog: DEFAULT_CATALOG_NAME,

src/common/meta/src/key/datanode_table.rs

@@ -23,7 +23,7 @@ use store_api::storage::RegionNumber;
 use table::metadata::TableId;
 
 use super::MetadataKey;
-use crate::error::{InvalidMetadataSnafu, Result};
+use crate::error::{DatanodeTableInfoNotFoundSnafu, InvalidMetadataSnafu, Result};
 use crate::key::{
     MetadataValue, RegionDistribution, DATANODE_TABLE_KEY_PATTERN, DATANODE_TABLE_KEY_PREFIX,
 };
@@ -77,7 +77,7 @@ impl DatanodeTableKey {
     }
 }
 
-impl<'a> MetadataKey<'a, DatanodeTableKey> for DatanodeTableKey {
+impl MetadataKey<'_, DatanodeTableKey> for DatanodeTableKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
@@ -209,6 +209,49 @@ impl DatanodeTableManager {
         Ok(txn)
     }
 
+    /// Builds a transaction to updates the redundant table options (including WAL options)
+    /// for given table id, if provided.
+    ///
+    /// Note that the provided `new_region_options` must be a
+    /// complete set of all options rather than incremental changes.
+    pub(crate) async fn build_update_table_options_txn(
+        &self,
+        table_id: TableId,
+        region_distribution: RegionDistribution,
+        new_region_options: HashMap<String, String>,
+    ) -> Result<Txn> {
+        assert!(!region_distribution.is_empty());
+        // safety: region_distribution must not be empty
+        let (any_datanode, _) = region_distribution.first_key_value().unwrap();
+
+        let mut region_info = self
+            .kv_backend
+            .get(&DatanodeTableKey::new(*any_datanode, table_id).to_bytes())
+            .await
+            .transpose()
+            .context(DatanodeTableInfoNotFoundSnafu {
+                datanode_id: *any_datanode,
+                table_id,
+            })?
+            .and_then(|r| DatanodeTableValue::try_from_raw_value(&r.value))?
+            .region_info;
+        // substitute region options only.
+        region_info.region_options = new_region_options;
+
+        let mut txns = Vec::with_capacity(region_distribution.len());
+
+        for (datanode, regions) in region_distribution.into_iter() {
+            let key = DatanodeTableKey::new(datanode, table_id);
+            let key_bytes = key.to_bytes();
+            let value_bytes = DatanodeTableValue::new(table_id, regions, region_info.clone())
+                .try_as_raw_value()?;
+            txns.push(TxnOp::Put(key_bytes, value_bytes));
+        }
+
+        let txn = Txn::new().and_then(txns);
+        Ok(txn)
+    }
+
     /// Builds the update datanode table transactions. It only executes while the primary keys comparing successes.
     pub(crate) fn build_update_txn(
         &self,

src/common/meta/src/key/flow/flow_info.rs

@@ -42,6 +42,8 @@ lazy_static! {
 /// The layout: `__flow/info/{flow_id}`.
 pub struct FlowInfoKey(FlowScoped<FlowInfoKeyInner>);
 
+pub type FlowInfoDecodeResult = Result<Option<DeserializedValueWithBytes<FlowInfoValue>>>;
+
 impl<'a> MetadataKey<'a, FlowInfoKey> for FlowInfoKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.0.to_bytes()
@@ -203,9 +205,7 @@ impl FlowInfoManager {
         flow_value: &FlowInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<FlowInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> FlowInfoDecodeResult,
     )> {
         let key = FlowInfoKey::new(flow_id).to_bytes();
         let txn = Txn::put_if_not_exists(key.clone(), flow_value.try_as_raw_value()?);

src/common/meta/src/key/flow/flow_name.rs

@@ -46,6 +46,8 @@ lazy_static! {
 /// The layout: `__flow/name/{catalog_name}/{flow_name}`.
 pub struct FlowNameKey<'a>(FlowScoped<FlowNameKeyInner<'a>>);
 
+pub type FlowNameDecodeResult = Result<Option<DeserializedValueWithBytes<FlowNameValue>>>;
+
 #[allow(dead_code)]
 impl<'a> FlowNameKey<'a> {
     /// Returns the [FlowNameKey]
@@ -104,7 +106,7 @@ impl<'a> MetadataKey<'a, FlowNameKeyInner<'a>> for FlowNameKeyInner<'_> {
         .into_bytes()
     }
 
-    fn from_bytes(bytes: &'a [u8]) -> Result<FlowNameKeyInner> {
+    fn from_bytes(bytes: &'a [u8]) -> Result<FlowNameKeyInner<'a>> {
         let key = std::str::from_utf8(bytes).map_err(|e| {
             error::InvalidMetadataSnafu {
                 err_msg: format!(
@@ -223,9 +225,7 @@ impl FlowNameManager {
         flow_id: FlowId,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<FlowNameValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> FlowNameDecodeResult,
     )> {
         let key = FlowNameKey::new(catalog_name, flow_name);
         let raw_key = key.to_bytes();

src/common/meta/src/key/node_address.rs

@@ -52,7 +52,7 @@ impl NodeAddressValue {
     }
 }
 
-impl<'a> MetadataKey<'a, NodeAddressKey> for NodeAddressKey {
+impl MetadataKey<'_, NodeAddressKey> for NodeAddressKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }

src/common/meta/src/key/schema_name.rs

@@ -41,7 +41,7 @@ pub struct SchemaNameKey<'a> {
     pub schema: &'a str,
 }
 
-impl<'a> Default for SchemaNameKey<'a> {
+impl Default for SchemaNameKey<'_> {
     fn default() -> Self {
         Self {
             catalog: DEFAULT_CATALOG_NAME,

src/common/meta/src/key/table_info.rs

@@ -23,6 +23,7 @@ use table::table_name::TableName;
 use table::table_reference::TableReference;
 
 use super::TABLE_INFO_KEY_PATTERN;
+use crate::ddl::utils::region_storage_path;
 use crate::error::{InvalidMetadataSnafu, Result};
 use crate::key::txn_helper::TxnOpGetResponseSet;
 use crate::key::{DeserializedValueWithBytes, MetadataKey, MetadataValue, TABLE_INFO_KEY_PREFIX};
@@ -51,7 +52,7 @@ impl Display for TableInfoKey {
     }
 }
 
-impl<'a> MetadataKey<'a, TableInfoKey> for TableInfoKey {
+impl MetadataKey<'_, TableInfoKey> for TableInfoKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
@@ -125,6 +126,11 @@ impl TableInfoValue {
             table_name: self.table_info.name.to_string(),
         }
     }
+
+    /// Builds storage path for all regions in table.
+    pub fn region_storage_path(&self) -> String {
+        region_storage_path(&self.table_info.catalog_name, &self.table_info.schema_name)
+    }
 }
 
 pub type TableInfoManagerRef = Arc<TableInfoManager>;
@@ -132,6 +138,7 @@ pub type TableInfoManagerRef = Arc<TableInfoManager>;
 pub struct TableInfoManager {
     kv_backend: KvBackendRef,
 }
+pub type TableInfoDecodeResult = Result<Option<DeserializedValueWithBytes<TableInfoValue>>>;
 
 impl TableInfoManager {
     pub fn new(kv_backend: KvBackendRef) -> Self {
@@ -145,9 +152,7 @@ impl TableInfoManager {
         table_info_value: &TableInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<TableInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> TableInfoDecodeResult,
     )> {
         let key = TableInfoKey::new(table_id);
         let raw_key = key.to_bytes();
@@ -169,9 +174,7 @@ impl TableInfoManager {
         new_table_info_value: &TableInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<TableInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> TableInfoDecodeResult,
     )> {
         let key = TableInfoKey::new(table_id);
         let raw_key = key.to_bytes();

src/common/meta/src/key/table_route.rs

@@ -245,7 +245,7 @@ impl LogicalTableRouteValue {
     }
 }
 
-impl<'a> MetadataKey<'a, TableRouteKey> for TableRouteKey {
+impl MetadataKey<'_, TableRouteKey> for TableRouteKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
@@ -472,6 +472,8 @@ pub struct TableRouteStorage {
     kv_backend: KvBackendRef,
 }
 
+pub type TableRouteValueDecodeResult = Result<Option<DeserializedValueWithBytes<TableRouteValue>>>;
+
 impl TableRouteStorage {
     pub fn new(kv_backend: KvBackendRef) -> Self {
         Self { kv_backend }
@@ -485,9 +487,7 @@ impl TableRouteStorage {
         table_route_value: &TableRouteValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> TableRouteValueDecodeResult,
     )> {
         let key = TableRouteKey::new(table_id);
         let raw_key = key.to_bytes();
@@ -510,9 +510,7 @@ impl TableRouteStorage {
         new_table_route_value: &TableRouteValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> TableRouteValueDecodeResult,
     )> {
         let key = TableRouteKey::new(table_id);
         let raw_key = key.to_bytes();

src/common/meta/src/key/view_info.rs

@@ -53,7 +53,7 @@ impl Display for ViewInfoKey {
     }
 }
 
-impl<'a> MetadataKey<'a, ViewInfoKey> for ViewInfoKey {
+impl MetadataKey<'_, ViewInfoKey> for ViewInfoKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
@@ -139,6 +139,8 @@ pub struct ViewInfoManager {
 
 pub type ViewInfoManagerRef = Arc<ViewInfoManager>;
 
+pub type ViewInfoValueDecodeResult = Result<Option<DeserializedValueWithBytes<ViewInfoValue>>>;
+
 impl ViewInfoManager {
     pub fn new(kv_backend: KvBackendRef) -> Self {
         Self { kv_backend }
@@ -151,9 +153,7 @@ impl ViewInfoManager {
         view_info_value: &ViewInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<ViewInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> ViewInfoValueDecodeResult,
     )> {
         let key = ViewInfoKey::new(view_id);
         let raw_key = key.to_bytes();
@@ -175,9 +175,7 @@ impl ViewInfoManager {
         new_view_info_value: &ViewInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<ViewInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> ViewInfoValueDecodeResult,
     )> {
         let key = ViewInfoKey::new(view_id);
         let raw_key = key.to_bytes();

src/common/meta/src/lock_key.rs

@@ -34,7 +34,7 @@ pub enum CatalogLock<'a> {
     Write(&'a str),
 }
 
-impl<'a> Display for CatalogLock<'a> {
+impl Display for CatalogLock<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let key = match self {
             CatalogLock::Read(s) => s,
@@ -44,7 +44,7 @@ impl<'a> Display for CatalogLock<'a> {
     }
 }
 
-impl<'a> From<CatalogLock<'a>> for StringKey {
+impl From<CatalogLock<'_>> for StringKey {
     fn from(value: CatalogLock) -> Self {
         match value {
             CatalogLock::Write(_) => StringKey::Exclusive(value.to_string()),

src/common/meta/src/rpc/router.rs

@@ -289,6 +289,7 @@ pub enum LeaderState {
     ///
     /// - The [`Region`] may be unavailable (e.g., Crashed, Network disconnected).
     /// - The [`Region`] was planned to migrate to another [`Peer`].
+    #[serde(alias = "Downgraded")]
     Downgrading,
 }
 
@@ -516,6 +517,73 @@ mod tests {
         assert_eq!(decoded, region_route);
     }
 
+    #[test]
+    fn test_region_route_compatibility() {
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_state: Some(LeaderState::Downgrading),
+            leader_down_since: None,
+        };
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}],"leader_state":"Downgraded","leader_down_since":null}"#;
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+        assert_eq!(decoded, region_route);
+
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_state: Some(LeaderState::Downgrading),
+            leader_down_since: None,
+        };
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}],"leader_status":"Downgraded","leader_down_since":null}"#;
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+        assert_eq!(decoded, region_route);
+
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_state: Some(LeaderState::Downgrading),
+            leader_down_since: None,
+        };
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}],"leader_state":"Downgrading","leader_down_since":null}"#;
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+        assert_eq!(decoded, region_route);
+
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_state: Some(LeaderState::Downgrading),
+            leader_down_since: None,
+        };
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}],"leader_status":"Downgrading","leader_down_since":null}"#;
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+        assert_eq!(decoded, region_route);
+    }
+
     #[test]
     fn test_de_serialize_partition() {
         let p = Partition {

src/common/procedure/src/store.rs

@@ -297,7 +297,7 @@ struct ParsedKey<'a> {
     key_type: KeyType,
 }
 
-impl<'a> fmt::Display for ParsedKey<'a> {
+impl fmt::Display for ParsedKey<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
             f,

src/common/recordbatch/src/recordbatch.rs

@@ -17,6 +17,7 @@ use std::slice;
 use std::sync::Arc;
 
 use datafusion::arrow::util::pretty::pretty_format_batches;
+use datatypes::prelude::DataType;
 use datatypes::schema::SchemaRef;
 use datatypes::value::Value;
 use datatypes::vectors::{Helper, VectorRef};
@@ -58,13 +59,18 @@ impl RecordBatch {
     }
 
     /// Create an empty [`RecordBatch`] from `schema`.
-    pub fn new_empty(schema: SchemaRef) -> Result<RecordBatch> {
+    pub fn new_empty(schema: SchemaRef) -> RecordBatch {
         let df_record_batch = DfRecordBatch::new_empty(schema.arrow_schema().clone());
-        Ok(RecordBatch {
+        let columns = schema
+            .column_schemas()
+            .iter()
+            .map(|col| col.data_type.create_mutable_vector(0).to_vector())
+            .collect();
+        RecordBatch {
             schema,
-            columns: vec![],
+            columns,
             df_record_batch,
-        })
+        }
     }
 
     pub fn try_project(&self, indices: &[usize]) -> Result<Self> {
@@ -220,7 +226,7 @@ pub struct RecordBatchRowIterator<'a> {
 }
 
 impl<'a> RecordBatchRowIterator<'a> {
-    fn new(record_batch: &'a RecordBatch) -> RecordBatchRowIterator {
+    fn new(record_batch: &'a RecordBatch) -> RecordBatchRowIterator<'a> {
         RecordBatchRowIterator {
             record_batch,
             rows: record_batch.df_record_batch.num_rows(),
@@ -230,7 +236,7 @@ impl<'a> RecordBatchRowIterator<'a> {
     }
 }
 
-impl<'a> Iterator for RecordBatchRowIterator<'a> {
+impl Iterator for RecordBatchRowIterator<'_> {
     type Item = Vec<Value>;
 
     fn next(&mut self) -> Option<Self::Item> {

src/common/runtime/Cargo.toml

@@ -4,21 +4,36 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+[lib]
+path = "src/lib.rs"
+
+[[bin]]
+name = "common-runtime-bin"
+path = "src/bin.rs"
+
 [lints]
 workspace = true
 
 [dependencies]
 async-trait.workspace = true
+clap.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 common-telemetry.workspace = true
+futures.workspace = true
 lazy_static.workspace = true
 num_cpus.workspace = true
 once_cell.workspace = true
+parking_lot.workspace = true
 paste.workspace = true
+pin-project.workspace = true
 prometheus.workspace = true
+rand.workspace = true
+ratelimit.workspace = true
 serde.workspace = true
+serde_json.workspace = true
 snafu.workspace = true
+tempfile.workspace = true
 tokio.workspace = true
 tokio-metrics = "0.3"
 tokio-metrics-collector = { git = "https://github.com/MichaelScofield/tokio-metrics-collector.git", rev = "89d692d5753d28564a7aac73c6ac5aba22243ba0" }

src/common/runtime/README.md

@@ -0,0 +1,60 @@
+# Greptime Runtime
+
+## Run performance test for different priority & workload type
+
+```
+# workspace is at this subcrate
+cargo run --release -- --loop-cnt 500
+```
+
+## Related PRs & issues
+
+- Preliminary support cpu limitation
+  
+  ISSUE: https://github.com/GreptimeTeam/greptimedb/issues/3685
+  
+  PR: https://github.com/GreptimeTeam/greptimedb/pull/4782
+
+## CPU resource constraints (ThrottleableRuntime)
+
+
+To achieve CPU resource constraints, we adopt the concept of rate limiting. When creating a future, we first wrap it with another layer of future to intercept the poll operation during runtime. By using the ratelimit library, we can simply implement a mechanism that allows only a limited number of polls for a batch of tasks under a certain priority within a specific time frame (the current token generation interval is set to 10ms).
+
+The default used runtime can be switched by 
+``` rust
+pub type Runtime = DefaultRuntime;
+```
+in `runtime.rs`.
+
+We tested four type of workload with 5 priorities, whose setup are as follows:
+
+``` rust
+impl Priority {
+    fn ratelimiter_count(&self) -> Result<Option<Ratelimiter>> {
+        let max = 8000;
+        let gen_per_10ms = match self {
+            Priority::VeryLow => Some(2000),
+            Priority::Low => Some(4000),
+            Priority::Middle => Some(6000),
+            Priority::High => Some(8000),
+            Priority::VeryHigh => None,
+        };
+        if let Some(gen_per_10ms) = gen_per_10ms {
+            Ratelimiter::builder(gen_per_10ms, Duration::from_millis(10)) // generate poll count per 10ms
+                .max_tokens(max) // reserved token for batch request
+                .build()
+                .context(BuildRuntimeRateLimiterSnafu)
+                .map(Some)
+        } else {
+            Ok(None)
+        }
+    }
+}
+```
+
+This is the preliminary experimental effect so far:
+
+![](resources/rdme-exp.png)
+
+## TODO
+- Introduce PID to achieve more accurate limitation.

src/common/runtime/src/bin.rs

@@ -0,0 +1,205 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::Parser;
+
+#[derive(Debug, Default, Parser)]
+pub struct Command {
+    #[clap(long)]
+    loop_cnt: usize,
+}
+
+fn main() {
+    common_telemetry::init_default_ut_logging();
+    let cmd = Command::parse();
+
+    test_diff_priority_cpu::test_diff_workload_priority(cmd.loop_cnt);
+}
+
+mod test_diff_priority_cpu {
+    use std::path::PathBuf;
+
+    use common_runtime::runtime::{BuilderBuild, Priority, RuntimeTrait};
+    use common_runtime::{Builder, Runtime};
+    use common_telemetry::debug;
+    use tempfile::TempDir;
+
+    fn compute_pi_str(precision: usize) -> String {
+        let mut pi = 0.0;
+        let mut sign = 1.0;
+
+        for i in 0..precision {
+            pi += sign / (2 * i + 1) as f64;
+            sign *= -1.0;
+        }
+
+        pi *= 4.0;
+        format!("{:.prec$}", pi, prec = precision)
+    }
+
+    macro_rules! def_workload_enum {
+        ($($variant:ident),+) => {
+            #[derive(Debug)]
+            enum WorkloadType {
+                $($variant),+
+            }
+
+            /// array of workloads for iteration
+            const WORKLOADS: &'static [WorkloadType] = &[
+                $( WorkloadType::$variant ),+
+            ];
+        };
+    }
+
+    def_workload_enum!(
+        ComputeHeavily,
+        ComputeHeavily2,
+        WriteFile,
+        SpawnBlockingWriteFile
+    );
+
+    async fn workload_compute_heavily() {
+        let prefix = 10;
+
+        for _ in 0..3000 {
+            let _ = compute_pi_str(prefix);
+            tokio::task::yield_now().await;
+        }
+    }
+    async fn workload_compute_heavily2() {
+        let prefix = 30;
+        for _ in 0..2000 {
+            let _ = compute_pi_str(prefix);
+            tokio::task::yield_now().await;
+        }
+    }
+    async fn workload_write_file(_idx: u64, tempdir: PathBuf) {
+        use tokio::io::AsyncWriteExt;
+        let prefix = 50;
+
+        let mut file = tokio::fs::OpenOptions::new()
+            .write(true)
+            .append(true)
+            .create(true)
+            .open(tempdir.join(format!("pi_{}", prefix)))
+            .await
+            .unwrap();
+        for i in 0..200 {
+            let pi = compute_pi_str(prefix);
+
+            if i % 2 == 0 {
+                file.write_all(pi.as_bytes()).await.unwrap();
+            }
+        }
+    }
+    async fn workload_spawn_blocking_write_file(tempdir: PathBuf) {
+        use std::io::Write;
+        let prefix = 100;
+        let mut file = Some(
+            std::fs::OpenOptions::new()
+                .append(true)
+                .create(true)
+                .open(tempdir.join(format!("pi_{}", prefix)))
+                .unwrap(),
+        );
+        for i in 0..100 {
+            let pi = compute_pi_str(prefix);
+            if i % 2 == 0 {
+                let mut file1 = file.take().unwrap();
+                file = Some(
+                    tokio::task::spawn_blocking(move || {
+                        file1.write_all(pi.as_bytes()).unwrap();
+                        file1
+                    })
+                    .await
+                    .unwrap(),
+                );
+            }
+        }
+    }
+
+    pub fn test_diff_workload_priority(loop_cnt: usize) {
+        let tempdir = tempfile::tempdir().unwrap();
+        let priorities = [
+            Priority::VeryLow,
+            Priority::Low,
+            Priority::Middle,
+            Priority::High,
+            Priority::VeryHigh,
+        ];
+        for wl in WORKLOADS {
+            for p in priorities.iter() {
+                let runtime: Runtime = Builder::default()
+                    .runtime_name("test")
+                    .thread_name("test")
+                    .worker_threads(8)
+                    .priority(*p)
+                    .build()
+                    .expect("Fail to create runtime");
+                let runtime2 = runtime.clone();
+                runtime.block_on(test_spec_priority_and_workload(
+                    *p, runtime2, wl, &tempdir, loop_cnt,
+                ));
+            }
+        }
+    }
+
+    async fn test_spec_priority_and_workload(
+        priority: Priority,
+        runtime: Runtime,
+        workload_id: &WorkloadType,
+        tempdir: &TempDir,
+        loop_cnt: usize,
+    ) {
+        tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
+        debug!(
+            "testing cpu usage for priority {:?} workload_id {:?}",
+            priority, workload_id,
+        );
+        // start monitor thread
+        let mut tasks = vec![];
+        let start = std::time::Instant::now();
+        for i in 0..loop_cnt {
+            // persist cpu usage in json: {priority}.{workload_id}
+            match *workload_id {
+                WorkloadType::ComputeHeavily => {
+                    tasks.push(runtime.spawn(workload_compute_heavily()));
+                }
+                WorkloadType::ComputeHeavily2 => {
+                    tasks.push(runtime.spawn(workload_compute_heavily2()));
+                }
+                WorkloadType::SpawnBlockingWriteFile => {
+                    tasks.push(runtime.spawn(workload_spawn_blocking_write_file(
+                        tempdir.path().to_path_buf(),
+                    )));
+                }
+                WorkloadType::WriteFile => {
+                    tasks.push(
+                        runtime.spawn(workload_write_file(i as u64, tempdir.path().to_path_buf())),
+                    );
+                }
+            }
+        }
+        for task in tasks {
+            task.await.unwrap();
+        }
+        let elapsed = start.elapsed();
+        debug!(
+            "test cpu usage for priority {:?} workload_id {:?} elapsed {}ms",
+            priority,
+            workload_id,
+            elapsed.as_millis()
+        );
+    }
+}

src/common/runtime/src/error.rs

@@ -33,6 +33,14 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to build runtime rate limiter"))]
+    BuildRuntimeRateLimiter {
+        #[snafu(implicit)]
+        location: Location,
+        #[snafu(source)]
+        error: ratelimit::Error,
+    },
+
     #[snafu(display("Repeated task {} is already started", name))]
     IllegalState {
         name: String,

src/common/runtime/src/global.rs

@@ -21,6 +21,7 @@ use once_cell::sync::Lazy;
 use paste::paste;
 use serde::{Deserialize, Serialize};
 
+use crate::runtime::{BuilderBuild, RuntimeTrait};
 use crate::{Builder, JoinHandle, Runtime};
 
 const GLOBAL_WORKERS: usize = 8;

src/common/runtime/src/lib.rs

@@ -17,6 +17,8 @@ pub mod global;
 mod metrics;
 mod repeated_task;
 pub mod runtime;
+pub mod runtime_default;
+pub mod runtime_throttleable;
 
 pub use global::{
     block_on_compact, block_on_global, compact_runtime, create_runtime, global_runtime,

src/common/runtime/src/repeated_task.rs

@@ -23,6 +23,7 @@ use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
 
 use crate::error::{IllegalStateSnafu, Result, WaitGcTaskStopSnafu};
+use crate::runtime::RuntimeTrait;
 use crate::Runtime;
 
 /// Task to execute repeatedly.

src/common/runtime/src/runtime.rs

@@ -19,24 +19,20 @@ use std::thread;
 use std::time::Duration;
 
 use snafu::ResultExt;
-use tokio::runtime::{Builder as RuntimeBuilder, Handle};
+use tokio::runtime::Builder as RuntimeBuilder;
 use tokio::sync::oneshot;
 pub use tokio::task::{JoinError, JoinHandle};
 
 use crate::error::*;
 use crate::metrics::*;
+use crate::runtime_default::DefaultRuntime;
+use crate::runtime_throttleable::ThrottleableRuntime;
+
+// configurations
+pub type Runtime = DefaultRuntime;
 
 static RUNTIME_ID: AtomicUsize = AtomicUsize::new(0);
 
-/// A runtime to run future tasks
-#[derive(Clone, Debug)]
-pub struct Runtime {
-    name: String,
-    handle: Handle,
-    // Used to receive a drop signal when dropper is dropped, inspired by databend
-    _dropper: Arc<Dropper>,
-}
-
 /// Dropping the dropper will cause runtime to shutdown.
 #[derive(Debug)]
 pub struct Dropper {
@@ -50,45 +46,42 @@ impl Drop for Dropper {
     }
 }
 
-impl Runtime {
-    pub fn builder() -> Builder {
+pub trait RuntimeTrait {
+    /// Get a runtime builder
+    fn builder() -> Builder {
         Builder::default()
     }
 
     /// Spawn a future and execute it in this thread pool
     ///
     /// Similar to tokio::runtime::Runtime::spawn()
-    pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+    fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
     where
         F: Future + Send + 'static,
-        F::Output: Send + 'static,
-    {
-        self.handle.spawn(future)
-    }
+        F::Output: Send + 'static;
 
     /// Run the provided function on an executor dedicated to blocking
     /// operations.
-    pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
+    fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
     where
         F: FnOnce() -> R + Send + 'static,
-        R: Send + 'static,
-    {
-        self.handle.spawn_blocking(func)
-    }
+        R: Send + 'static;
 
     /// Run a future to complete, this is the runtime's entry point
-    pub fn block_on<F: Future>(&self, future: F) -> F::Output {
-        self.handle.block_on(future)
-    }
+    fn block_on<F: Future>(&self, future: F) -> F::Output;
 
-    pub fn name(&self) -> &str {
-        &self.name
-    }
+    /// Get the name of the runtime
+    fn name(&self) -> &str;
+}
+
+pub trait BuilderBuild<R: RuntimeTrait> {
+    fn build(&mut self) -> Result<R>;
 }
 
 pub struct Builder {
     runtime_name: String,
     thread_name: String,
+    priority: Priority,
     builder: RuntimeBuilder,
 }
 
@@ -98,11 +91,17 @@ impl Default for Builder {
             runtime_name: format!("runtime-{}", RUNTIME_ID.fetch_add(1, Ordering::Relaxed)),
             thread_name: "default-worker".to_string(),
             builder: RuntimeBuilder::new_multi_thread(),
+            priority: Priority::VeryHigh,
         }
     }
 }
 
 impl Builder {
+    pub fn priority(&mut self, priority: Priority) -> &mut Self {
+        self.priority = priority;
+        self
+    }
+
     /// Sets the number of worker threads the Runtime will use.
     ///
     /// This can be any number above 0. The default value is the number of cores available to the system.
@@ -139,8 +138,10 @@ impl Builder {
         self.thread_name = val.into();
         self
     }
+}
 
-    pub fn build(&mut self) -> Result<Runtime> {
+impl BuilderBuild<DefaultRuntime> for Builder {
+    fn build(&mut self) -> Result<DefaultRuntime> {
         let runtime = self
             .builder
             .enable_all()
@@ -163,18 +164,53 @@ impl Builder {
         #[cfg(tokio_unstable)]
         register_collector(name.clone(), &handle);
 
-        Ok(Runtime {
-            name,
+        Ok(DefaultRuntime::new(
+            &name,
             handle,
-            _dropper: Arc::new(Dropper {
+            Arc::new(Dropper {
                 close: Some(send_stop),
             }),
-        })
+        ))
+    }
+}
+
+impl BuilderBuild<ThrottleableRuntime> for Builder {
+    fn build(&mut self) -> Result<ThrottleableRuntime> {
+        let runtime = self
+            .builder
+            .enable_all()
+            .thread_name(self.thread_name.clone())
+            .on_thread_start(on_thread_start(self.thread_name.clone()))
+            .on_thread_stop(on_thread_stop(self.thread_name.clone()))
+            .on_thread_park(on_thread_park(self.thread_name.clone()))
+            .on_thread_unpark(on_thread_unpark(self.thread_name.clone()))
+            .build()
+            .context(BuildRuntimeSnafu)?;
+
+        let name = self.runtime_name.clone();
+        let handle = runtime.handle().clone();
+        let (send_stop, recv_stop) = oneshot::channel();
+        // Block the runtime to shutdown.
+        let _ = thread::Builder::new()
+            .name(format!("{}-blocker", self.thread_name))
+            .spawn(move || runtime.block_on(recv_stop));
+
+        #[cfg(tokio_unstable)]
+        register_collector(name.clone(), &handle);
+
+        ThrottleableRuntime::new(
+            &name,
+            self.priority,
+            handle,
+            Arc::new(Dropper {
+                close: Some(send_stop),
+            }),
+        )
     }
 }
 
 #[cfg(tokio_unstable)]
-pub fn register_collector(name: String, handle: &Handle) {
+pub fn register_collector(name: String, handle: &tokio::runtime::Handle) {
     let name = name.replace("-", "_");
     let monitor = tokio_metrics::RuntimeMonitor::new(handle);
     let collector = tokio_metrics_collector::RuntimeCollector::new(monitor, name);
@@ -213,8 +249,18 @@ fn on_thread_unpark(thread_name: String) -> impl Fn() + 'static {
     }
 }
 
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+pub enum Priority {
+    VeryLow = 0,
+    Low = 1,
+    Middle = 2,
+    High = 3,
+    VeryHigh = 4,
+}
+
 #[cfg(test)]
 mod tests {
+
     use std::sync::Arc;
     use std::thread;
     use std::time::Duration;
@@ -235,12 +281,12 @@ mod tests {
 
     #[test]
     fn test_metric() {
-        let runtime = Builder::default()
+        let runtime: Runtime = Builder::default()
             .worker_threads(5)
             .thread_name("test_runtime_metric")
             .build()
             .unwrap();
-        // wait threads created
+        // wait threads create
         thread::sleep(Duration::from_millis(50));
 
         let _handle = runtime.spawn(async {

src/common/runtime/src/runtime_default.rs

@@ -0,0 +1,77 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+use std::sync::Arc;
+
+use tokio::runtime::Handle;
+pub use tokio::task::JoinHandle;
+
+use crate::runtime::{Dropper, RuntimeTrait};
+use crate::Builder;
+
+/// A runtime to run future tasks
+#[derive(Clone, Debug)]
+pub struct DefaultRuntime {
+    name: String,
+    handle: Handle,
+    // Used to receive a drop signal when dropper is dropped, inspired by databend
+    _dropper: Arc<Dropper>,
+}
+
+impl DefaultRuntime {
+    pub(crate) fn new(name: &str, handle: Handle, dropper: Arc<Dropper>) -> Self {
+        Self {
+            name: name.to_string(),
+            handle,
+            _dropper: dropper,
+        }
+    }
+}
+
+impl RuntimeTrait for DefaultRuntime {
+    fn builder() -> Builder {
+        Builder::default()
+    }
+
+    /// Spawn a future and execute it in this thread pool
+    ///
+    /// Similar to tokio::runtime::Runtime::spawn()
+    fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+    where
+        F: Future + Send + 'static,
+        F::Output: Send + 'static,
+    {
+        self.handle.spawn(future)
+    }
+
+    /// Run the provided function on an executor dedicated to blocking
+    /// operations.
+    fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
+    where
+        F: FnOnce() -> R + Send + 'static,
+        R: Send + 'static,
+    {
+        self.handle.spawn_blocking(func)
+    }
+
+    /// Run a future to complete, this is the runtime's entry point
+    fn block_on<F: Future>(&self, future: F) -> F::Output {
+        self.handle.block_on(future)
+    }
+
+    fn name(&self) -> &str {
+        &self.name
+    }
+}

src/common/runtime/src/runtime_throttleable.rs

@@ -0,0 +1,285 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use futures::FutureExt;
+use ratelimit::Ratelimiter;
+use snafu::ResultExt;
+use tokio::runtime::Handle;
+pub use tokio::task::JoinHandle;
+use tokio::time::Sleep;
+
+use crate::error::{BuildRuntimeRateLimiterSnafu, Result};
+use crate::runtime::{Dropper, Priority, RuntimeTrait};
+use crate::Builder;
+
+struct RuntimeRateLimiter {
+    pub ratelimiter: Option<Ratelimiter>,
+}
+
+impl Debug for RuntimeRateLimiter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RuntimeThrottleShareWithFuture")
+            .field(
+                "ratelimiter_max_tokens",
+                &self.ratelimiter.as_ref().map(|v| v.max_tokens()),
+            )
+            .field(
+                "ratelimiter_refill_amount",
+                &self.ratelimiter.as_ref().map(|v| v.refill_amount()),
+            )
+            .finish()
+    }
+}
+
+/// A runtime to run future tasks
+#[derive(Clone, Debug)]
+pub struct ThrottleableRuntime {
+    name: String,
+    handle: Handle,
+    shared_with_future: Arc<RuntimeRateLimiter>,
+    // Used to receive a drop signal when dropper is dropped, inspired by databend
+    _dropper: Arc<Dropper>,
+}
+
+impl ThrottleableRuntime {
+    pub(crate) fn new(
+        name: &str,
+        priority: Priority,
+        handle: Handle,
+        dropper: Arc<Dropper>,
+    ) -> Result<Self> {
+        Ok(Self {
+            name: name.to_string(),
+            handle,
+            shared_with_future: Arc::new(RuntimeRateLimiter {
+                ratelimiter: priority.ratelimiter_count()?,
+            }),
+            _dropper: dropper,
+        })
+    }
+}
+
+impl RuntimeTrait for ThrottleableRuntime {
+    fn builder() -> Builder {
+        Builder::default()
+    }
+
+    /// Spawn a future and execute it in this thread pool
+    ///
+    /// Similar to tokio::runtime::Runtime::spawn()
+    fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+    where
+        F: Future + Send + 'static,
+        F::Output: Send + 'static,
+    {
+        self.handle
+            .spawn(ThrottleFuture::new(self.shared_with_future.clone(), future))
+    }
+
+    /// Run the provided function on an executor dedicated to blocking
+    /// operations.
+    fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
+    where
+        F: FnOnce() -> R + Send + 'static,
+        R: Send + 'static,
+    {
+        self.handle.spawn_blocking(func)
+    }
+
+    /// Run a future to complete, this is the runtime's entry point
+    fn block_on<F: Future>(&self, future: F) -> F::Output {
+        self.handle.block_on(future)
+    }
+
+    fn name(&self) -> &str {
+        &self.name
+    }
+}
+
+enum State {
+    Pollable,
+    Throttled(Pin<Box<Sleep>>),
+}
+
+impl State {
+    fn unwrap_backoff(&mut self) -> &mut Pin<Box<Sleep>> {
+        match self {
+            State::Throttled(sleep) => sleep,
+            _ => panic!("unwrap_backoff failed"),
+        }
+    }
+}
+
+#[pin_project::pin_project]
+pub struct ThrottleFuture<F: Future + Send + 'static> {
+    #[pin]
+    future: F,
+
+    /// RateLimiter of this future
+    handle: Arc<RuntimeRateLimiter>,
+
+    state: State,
+}
+
+impl<F> ThrottleFuture<F>
+where
+    F: Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    fn new(handle: Arc<RuntimeRateLimiter>, future: F) -> Self {
+        Self {
+            future,
+            handle,
+            state: State::Pollable,
+        }
+    }
+}
+
+impl<F> Future for ThrottleFuture<F>
+where
+    F: Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    type Output = F::Output;
+
+    fn poll(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let this = self.project();
+
+        match this.state {
+            State::Pollable => {}
+            State::Throttled(ref mut sleep) => match sleep.poll_unpin(cx) {
+                Poll::Ready(_) => {
+                    *this.state = State::Pollable;
+                }
+                Poll::Pending => return Poll::Pending,
+            },
+        };
+
+        if let Some(ratelimiter) = &this.handle.ratelimiter {
+            if let Err(wait) = ratelimiter.try_wait() {
+                *this.state = State::Throttled(Box::pin(tokio::time::sleep(wait)));
+                match this.state.unwrap_backoff().poll_unpin(cx) {
+                    Poll::Ready(_) => {
+                        *this.state = State::Pollable;
+                    }
+                    Poll::Pending => {
+                        return Poll::Pending;
+                    }
+                }
+            }
+        }
+
+        let poll_res = this.future.poll(cx);
+
+        match poll_res {
+            Poll::Ready(r) => Poll::Ready(r),
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}
+
+impl Priority {
+    fn ratelimiter_count(&self) -> Result<Option<Ratelimiter>> {
+        let max = 8000;
+        let gen_per_10ms = match self {
+            Priority::VeryLow => Some(2000),
+            Priority::Low => Some(4000),
+            Priority::Middle => Some(6000),
+            Priority::High => Some(8000),
+            Priority::VeryHigh => None,
+        };
+        if let Some(gen_per_10ms) = gen_per_10ms {
+            Ratelimiter::builder(gen_per_10ms, Duration::from_millis(10)) // generate poll count per 10ms
+                .max_tokens(max) // reserved token for batch request
+                .build()
+                .context(BuildRuntimeRateLimiterSnafu)
+                .map(Some)
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use tokio::fs::File;
+    use tokio::io::AsyncWriteExt;
+    use tokio::time::Duration;
+
+    use super::*;
+    use crate::runtime::BuilderBuild;
+
+    #[tokio::test]
+    async fn test_throttleable_runtime_spawn_simple() {
+        for p in [
+            Priority::VeryLow,
+            Priority::Low,
+            Priority::Middle,
+            Priority::High,
+            Priority::VeryHigh,
+        ] {
+            let runtime: ThrottleableRuntime = Builder::default()
+                .runtime_name("test")
+                .thread_name("test")
+                .worker_threads(8)
+                .priority(p)
+                .build()
+                .expect("Fail to create runtime");
+
+            // Spawn a simple future that returns 42
+            let handle = runtime.spawn(async {
+                tokio::time::sleep(Duration::from_millis(10)).await;
+                42
+            });
+            let result = handle.await.expect("Task panicked");
+            assert_eq!(result, 42);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_throttleable_runtime_spawn_complex() {
+        let tempdir = tempfile::tempdir().unwrap();
+        for p in [
+            Priority::VeryLow,
+            Priority::Low,
+            Priority::Middle,
+            Priority::High,
+            Priority::VeryHigh,
+        ] {
+            let runtime: ThrottleableRuntime = Builder::default()
+                .runtime_name("test")
+                .thread_name("test")
+                .worker_threads(8)
+                .priority(p)
+                .build()
+                .expect("Fail to create runtime");
+            let tempdirpath = tempdir.path().to_path_buf();
+            let handle = runtime.spawn(async move {
+                let mut file = File::create(tempdirpath.join("test.txt")).await.unwrap();
+                file.write_all(b"Hello, world!").await.unwrap();
+                42
+            });
+            let result = handle.await.expect("Task panicked");
+            assert_eq!(result, 42);
+        }
+    }
+}

src/common/telemetry/Cargo.toml

@@ -26,13 +26,13 @@ opentelemetry = { version = "0.21.0", default-features = false, features = [
 opentelemetry-otlp = { version = "0.14.0", features = ["tokio"] }
 opentelemetry-semantic-conventions = "0.13.0"
 opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"] }
-parking_lot = { version = "0.12" }
+parking_lot.workspace = true
 prometheus.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tokio.workspace = true
 tracing = "0.1"
-tracing-appender = "0.2"
+tracing-appender.workspace = true
 tracing-log = "0.1"
 tracing-opentelemetry = "0.22.0"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
+tracing-subscriber.workspace = true

src/common/wal/src/config/kafka/datanode.rs

@@ -46,6 +46,8 @@ pub struct DatanodeKafkaConfig {
     pub create_index: bool,
     #[serde(with = "humantime_serde")]
     pub dump_index_interval: Duration,
+    /// Ignore missing entries during read WAL.
+    pub overwrite_entry_start_id: bool,
 }
 
 impl Default for DatanodeKafkaConfig {
@@ -60,6 +62,7 @@ impl Default for DatanodeKafkaConfig {
             auto_create_topics: true,
             create_index: true,
             dump_index_interval: Duration::from_secs(60),
+            overwrite_entry_start_id: false,
         }
     }
 }

src/datanode/src/tests.rs

@@ -23,6 +23,7 @@ use common_function::function::FunctionRef;
 use common_function::scalars::aggregate::AggregateFunctionMetaRef;
 use common_query::prelude::ScalarUdf;
 use common_query::Output;
+use common_runtime::runtime::{BuilderBuild, RuntimeTrait};
 use common_runtime::Runtime;
 use datafusion_expr::LogicalPlan;
 use query::dataframe::DataFrame;

src/datatypes/src/error.rs

@@ -189,6 +189,13 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Invalid JSON text: {}", value))]
+    InvalidJson {
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Value exceeds the precision {} bound", precision))]
     ValueExceedsPrecision {
         precision: u8,
@@ -222,7 +229,8 @@ impl ErrorExt for Error {
             | DefaultValueType { .. }
             | DuplicateMeta { .. }
             | InvalidTimestampPrecision { .. }
-            | InvalidPrecisionOrScale { .. } => StatusCode::InvalidArguments,
+            | InvalidPrecisionOrScale { .. }
+            | InvalidJson { .. } => StatusCode::InvalidArguments,
 
             ValueExceedsPrecision { .. }
             | CastType { .. }

src/datatypes/src/lib.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #![feature(let_chains)]
+#![feature(assert_matches)]
 
 pub mod arrow_array;
 pub mod data_type;

src/datatypes/src/scalars.rs

@@ -203,7 +203,7 @@ impl Scalar for bool {
     }
 }
 
-impl<'a> ScalarRef<'a> for bool {
+impl ScalarRef<'_> for bool {
     type ScalarType = bool;
 
     #[inline]
@@ -273,7 +273,7 @@ impl Scalar for Date {
     }
 }
 
-impl<'a> ScalarRef<'a> for Date {
+impl ScalarRef<'_> for Date {
     type ScalarType = Date;
 
     fn to_owned_scalar(&self) -> Self::ScalarType {
@@ -294,7 +294,7 @@ impl Scalar for Decimal128 {
     }
 }
 
-impl<'a> ScalarRef<'a> for Decimal128 {
+impl ScalarRef<'_> for Decimal128 {
     type ScalarType = Decimal128;
 
     fn to_owned_scalar(&self) -> Self::ScalarType {
@@ -315,7 +315,7 @@ impl Scalar for DateTime {
     }
 }
 
-impl<'a> ScalarRef<'a> for DateTime {
+impl ScalarRef<'_> for DateTime {
     type ScalarType = DateTime;
 
     fn to_owned_scalar(&self) -> Self::ScalarType {

src/datatypes/src/types/cast.rs

@@ -82,8 +82,8 @@ pub fn cast_with_opt(
     }
 }
 
-/// Return true if the src_value can be casted to dest_type,
-/// Otherwise, return false.
+/// Return true if the src_value can be casted to dest_type, Otherwise, return false.
+///
 /// Notice: this function does not promise that the `cast_with_opt` will succeed,
 /// it only checks whether the src_value can be casted to dest_type.
 pub fn can_cast_type(src_value: &Value, dest_type: &ConcreteDataType) -> bool {

src/datatypes/src/types/primitive_type.rs

@@ -83,9 +83,10 @@ pub trait LogicalPrimitiveType: 'static + Sized {
     fn cast_value_ref(value: ValueRef) -> Result<Option<Self::Wrapper>>;
 }
 
-/// A new type for [WrapperType], complement the `Ord` feature for it. Wrapping non ordered
-/// primitive types like `f32` and `f64` in `OrdPrimitive` can make them be used in places that
-/// require `Ord`. For example, in `Median` UDAFs.
+/// A new type for [WrapperType], complement the `Ord` feature for it.
+///
+/// Wrapping non ordered primitive types like `f32` and `f64` in `OrdPrimitive`
+/// can make them be used in places that require `Ord`. For example, in `Median` UDAFs.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub struct OrdPrimitive<T: WrapperType>(pub T);
 

src/datatypes/src/value.rs

@@ -38,6 +38,7 @@ use snafu::{ensure, ResultExt};
 
 use crate::error::{self, ConvertArrowArrayToScalarsSnafu, Error, Result, TryFromValueSnafu};
 use crate::prelude::*;
+use crate::schema::ColumnSchema;
 use crate::type_id::LogicalTypeId;
 use crate::types::{IntervalType, ListType};
 use crate::vectors::ListVector;
@@ -1086,7 +1087,7 @@ macro_rules! impl_as_for_value_ref {
     };
 }
 
-impl<'a> ValueRef<'a> {
+impl ValueRef<'_> {
     define_data_type_func!(ValueRef);
 
     /// Returns true if this is null.
@@ -1213,13 +1214,13 @@ impl<'a> ValueRef<'a> {
     }
 }
 
-impl<'a> PartialOrd for ValueRef<'a> {
+impl PartialOrd for ValueRef<'_> {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<'a> Ord for ValueRef<'a> {
+impl Ord for ValueRef<'_> {
     fn cmp(&self, other: &Self) -> Ordering {
         impl_ord_for_value_like!(ValueRef, self, other)
     }
@@ -1286,39 +1287,52 @@ impl<'a> From<Option<ListValueRef<'a>>> for ValueRef<'a> {
     }
 }
 
-impl<'a> TryFrom<ValueRef<'a>> for serde_json::Value {
-    type Error = serde_json::Error;
+/// transform a [ValueRef] to a [serde_json::Value].
+/// The json type will be handled specially
+pub fn transform_value_ref_to_json_value<'a>(
+    value: ValueRef<'a>,
+    schema: &'a ColumnSchema,
+) -> serde_json::Result<serde_json::Value> {
+    let json_value = match value {
+        ValueRef::Null => serde_json::Value::Null,
+        ValueRef::Boolean(v) => serde_json::Value::Bool(v),
+        ValueRef::UInt8(v) => serde_json::Value::from(v),
+        ValueRef::UInt16(v) => serde_json::Value::from(v),
+        ValueRef::UInt32(v) => serde_json::Value::from(v),
+        ValueRef::UInt64(v) => serde_json::Value::from(v),
+        ValueRef::Int8(v) => serde_json::Value::from(v),
+        ValueRef::Int16(v) => serde_json::Value::from(v),
+        ValueRef::Int32(v) => serde_json::Value::from(v),
+        ValueRef::Int64(v) => serde_json::Value::from(v),
+        ValueRef::Float32(v) => serde_json::Value::from(v.0),
+        ValueRef::Float64(v) => serde_json::Value::from(v.0),
+        ValueRef::String(bytes) => serde_json::Value::String(bytes.to_string()),
+        ValueRef::Binary(bytes) => {
+            if let ConcreteDataType::Json(_) = schema.data_type {
+                match jsonb::from_slice(bytes) {
+                    Ok(json) => json.into(),
+                    Err(e) => {
+                        error!(e; "Failed to parse jsonb");
+                        serde_json::Value::Null
+                    }
+                }
+            } else {
+                serde_json::to_value(bytes)?
+            }
+        }
+        ValueRef::Date(v) => serde_json::Value::Number(v.val().into()),
+        ValueRef::DateTime(v) => serde_json::Value::Number(v.val().into()),
+        ValueRef::List(v) => serde_json::to_value(v)?,
+        ValueRef::Timestamp(v) => serde_json::to_value(v.value())?,
+        ValueRef::Time(v) => serde_json::to_value(v.value())?,
+        ValueRef::IntervalYearMonth(v) => serde_json::Value::from(v),
+        ValueRef::IntervalDayTime(v) => serde_json::Value::from(v),
+        ValueRef::IntervalMonthDayNano(v) => serde_json::Value::from(v),
+        ValueRef::Duration(v) => serde_json::to_value(v.value())?,
+        ValueRef::Decimal128(v) => serde_json::to_value(v.to_string())?,
+    };
 
-    fn try_from(value: ValueRef<'a>) -> serde_json::Result<serde_json::Value> {
-        let json_value = match value {
-            ValueRef::Null => serde_json::Value::Null,
-            ValueRef::Boolean(v) => serde_json::Value::Bool(v),
-            ValueRef::UInt8(v) => serde_json::Value::from(v),
-            ValueRef::UInt16(v) => serde_json::Value::from(v),
-            ValueRef::UInt32(v) => serde_json::Value::from(v),
-            ValueRef::UInt64(v) => serde_json::Value::from(v),
-            ValueRef::Int8(v) => serde_json::Value::from(v),
-            ValueRef::Int16(v) => serde_json::Value::from(v),
-            ValueRef::Int32(v) => serde_json::Value::from(v),
-            ValueRef::Int64(v) => serde_json::Value::from(v),
-            ValueRef::Float32(v) => serde_json::Value::from(v.0),
-            ValueRef::Float64(v) => serde_json::Value::from(v.0),
-            ValueRef::String(bytes) => serde_json::Value::String(bytes.to_string()),
-            ValueRef::Binary(bytes) => serde_json::to_value(bytes)?,
-            ValueRef::Date(v) => serde_json::Value::Number(v.val().into()),
-            ValueRef::DateTime(v) => serde_json::Value::Number(v.val().into()),
-            ValueRef::List(v) => serde_json::to_value(v)?,
-            ValueRef::Timestamp(v) => serde_json::to_value(v.value())?,
-            ValueRef::Time(v) => serde_json::to_value(v.value())?,
-            ValueRef::IntervalYearMonth(v) => serde_json::Value::from(v),
-            ValueRef::IntervalDayTime(v) => serde_json::Value::from(v),
-            ValueRef::IntervalMonthDayNano(v) => serde_json::Value::from(v),
-            ValueRef::Duration(v) => serde_json::to_value(v.value())?,
-            ValueRef::Decimal128(v) => serde_json::to_value(v.to_string())?,
-        };
-
-        Ok(json_value)
-    }
+    Ok(json_value)
 }
 
 /// Reference to a [ListValue].
@@ -1333,7 +1347,7 @@ pub enum ListValueRef<'a> {
     Ref { val: &'a ListValue },
 }
 
-impl<'a> ListValueRef<'a> {
+impl ListValueRef<'_> {
     /// Convert self to [Value]. This method would clone the underlying data.
     fn to_value(self) -> Value {
         match self {
@@ -1351,7 +1365,7 @@ impl<'a> ListValueRef<'a> {
     }
 }
 
-impl<'a> Serialize for ListValueRef<'a> {
+impl Serialize for ListValueRef<'_> {
     fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
         match self {
             ListValueRef::Indexed { vector, idx } => match vector.get(*idx) {
@@ -1363,28 +1377,28 @@ impl<'a> Serialize for ListValueRef<'a> {
     }
 }
 
-impl<'a> PartialEq for ListValueRef<'a> {
+impl PartialEq for ListValueRef<'_> {
     fn eq(&self, other: &Self) -> bool {
         self.to_value().eq(&other.to_value())
     }
 }
 
-impl<'a> Eq for ListValueRef<'a> {}
+impl Eq for ListValueRef<'_> {}
 
-impl<'a> Ord for ListValueRef<'a> {
+impl Ord for ListValueRef<'_> {
     fn cmp(&self, other: &Self) -> Ordering {
         // Respect the order of `Value` by converting into value before comparison.
         self.to_value().cmp(&other.to_value())
     }
 }
 
-impl<'a> PartialOrd for ListValueRef<'a> {
+impl PartialOrd for ListValueRef<'_> {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<'a> ValueRef<'a> {
+impl ValueRef<'_> {
     /// Returns the size of the underlying data in bytes,
     /// The size is estimated and only considers the data size.
     pub fn data_size(&self) -> usize {

src/datatypes/src/vectors/binary.rs

@@ -36,6 +36,36 @@ impl BinaryVector {
     pub(crate) fn as_arrow(&self) -> &dyn Array {
         &self.array
     }
+
+    /// Creates a new binary vector of JSONB from a binary vector.
+    /// The binary vector must contain valid JSON strings.
+    pub fn convert_binary_to_json(&self) -> Result<BinaryVector> {
+        let arrow_array = self.to_arrow_array();
+        let mut vector = vec![];
+        for binary in arrow_array
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap()
+            .iter()
+        {
+            let jsonb = if let Some(binary) = binary {
+                match jsonb::from_slice(binary) {
+                    Ok(jsonb) => Some(jsonb.to_vec()),
+                    Err(_) => {
+                        let s = String::from_utf8_lossy(binary);
+                        return error::InvalidJsonSnafu {
+                            value: s.to_string(),
+                        }
+                        .fail();
+                    }
+                }
+            } else {
+                None
+            };
+            vector.push(jsonb);
+        }
+        Ok(BinaryVector::from(vector))
+    }
 }
 
 impl From<BinaryArray> for BinaryVector {
@@ -233,6 +263,8 @@ vectors::impl_try_from_arrow_array_for_vector!(BinaryArray, BinaryVector);
 
 #[cfg(test)]
 mod tests {
+    use std::assert_matches::assert_matches;
+
     use arrow::datatypes::DataType as ArrowDataType;
     use common_base::bytes::Bytes;
     use serde_json;
@@ -383,4 +415,52 @@ mod tests {
         assert_eq!(b"four", vector.get_data(3).unwrap());
         assert_eq!(builder.len(), 4);
     }
+
+    #[test]
+    fn test_binary_json_conversion() {
+        // json strings
+        let json_strings = vec![
+            b"{\"hello\": \"world\"}".to_vec(),
+            b"{\"foo\": 1}".to_vec(),
+            b"123".to_vec(),
+        ];
+        let json_vector = BinaryVector::from(json_strings.clone())
+            .convert_binary_to_json()
+            .unwrap();
+        let jsonbs = json_strings
+            .iter()
+            .map(|v| jsonb::parse_value(v).unwrap().to_vec())
+            .collect::<Vec<_>>();
+        for i in 0..3 {
+            assert_eq!(
+                json_vector.get_ref(i).as_binary().unwrap().unwrap(),
+                jsonbs.get(i).unwrap().as_slice()
+            );
+        }
+
+        // jsonb
+        let json_vector = BinaryVector::from(jsonbs.clone())
+            .convert_binary_to_json()
+            .unwrap();
+        for i in 0..3 {
+            assert_eq!(
+                json_vector.get_ref(i).as_binary().unwrap().unwrap(),
+                jsonbs.get(i).unwrap().as_slice()
+            );
+        }
+
+        // binary with jsonb header (0x80, 0x40, 0x20)
+        let binary_with_jsonb_header: Vec<u8> = [0x80, 0x23, 0x40, 0x22].to_vec();
+        let error = BinaryVector::from(vec![binary_with_jsonb_header])
+            .convert_binary_to_json()
+            .unwrap_err();
+        assert_matches!(error, error::Error::InvalidJson { .. });
+
+        // invalid json string
+        let json_strings = vec![b"{\"hello\": \"world\"".to_vec()];
+        let error = BinaryVector::from(json_strings)
+            .convert_binary_to_json()
+            .unwrap_err();
+        assert_matches!(error, error::Error::InvalidJson { .. });
+    }
 }

src/datatypes/src/vectors/decimal.rs

@@ -247,7 +247,7 @@ pub struct Decimal128Iter<'a> {
     iter: ArrayIter<&'a Decimal128Array>,
 }
 
-impl<'a> Iterator for Decimal128Iter<'a> {
+impl Iterator for Decimal128Iter<'_> {
     type Item = Option<Decimal128>;
 
     fn next(&mut self) -> Option<Self::Item> {

src/datatypes/src/vectors/list.rs

@@ -157,7 +157,7 @@ pub struct ListIter<'a> {
 }
 
 impl<'a> ListIter<'a> {
-    fn new(vector: &'a ListVector) -> ListIter {
+    fn new(vector: &'a ListVector) -> ListIter<'a> {
         ListIter { vector, idx: 0 }
     }
 }

src/datatypes/src/vectors/operations.rs

@@ -18,6 +18,8 @@ mod find_unique;
 mod replicate;
 mod take;
 
+use std::sync::Arc;
+
 use common_base::BitVec;
 
 use crate::error::{self, Result};
@@ -89,6 +91,12 @@ macro_rules! impl_scalar_vector_op {
             }
 
             fn cast(&self, to_type: &ConcreteDataType) -> Result<VectorRef> {
+                if to_type == &ConcreteDataType::json_datatype() {
+                    if let Some(vector) = self.as_any().downcast_ref::<BinaryVector>() {
+                        let json_vector = vector.convert_binary_to_json()?;
+                        return Ok(Arc::new(json_vector) as VectorRef);
+                    }
+                }
                 cast::cast_non_constant!(self, to_type)
             }
 

src/datatypes/src/vectors/primitive.rs

@@ -207,7 +207,7 @@ pub struct PrimitiveIter<'a, T: LogicalPrimitiveType> {
     iter: ArrayIter<&'a PrimitiveArray<T::ArrowPrimitive>>,
 }
 
-impl<'a, T: LogicalPrimitiveType> Iterator for PrimitiveIter<'a, T> {
+impl<T: LogicalPrimitiveType> Iterator for PrimitiveIter<'_, T> {
     type Item = Option<T::Wrapper>;
 
     fn next(&mut self) -> Option<Option<T::Wrapper>> {

src/file-engine/src/engine.rs

@@ -91,8 +91,9 @@ impl RegionEngine for FileRegionEngine {
         request: ScanRequest,
     ) -> Result<RegionScannerRef, BoxedError> {
         let stream = self.handle_query(region_id, request).await?;
+        let metadata = self.get_metadata(region_id).await?;
         // We don't support enabling append mode for file engine.
-        let scanner = Box::new(SinglePartitionScanner::new(stream, false));
+        let scanner = Box::new(SinglePartitionScanner::new(stream, false, metadata));
         Ok(scanner)
     }
 

src/flow/src/adapter.rs

@@ -271,10 +271,17 @@ impl FlowWorkerManager {
                         let rows_proto: Vec<v1::Row> = insert
                             .into_iter()
                             .map(|(mut row, _ts)| {
-                                // `update_at` col
-                                row.extend([Value::from(common_time::Timestamp::new_millisecond(
-                                    now,
-                                ))]);
+                                // extend `update_at` col if needed
+                                // if schema include a millisecond timestamp here, and result row doesn't have it, add it
+                                if row.len() < proto_schema.len()
+                                    && proto_schema[row.len()].datatype
+                                        == greptime_proto::v1::ColumnDataType::TimestampMillisecond
+                                            as i32
+                                {
+                                    row.extend([Value::from(
+                                        common_time::Timestamp::new_millisecond(now),
+                                    )]);
+                                }
                                 // ts col, if auto create
                                 if is_ts_placeholder {
                                     ensure!(
@@ -291,6 +298,17 @@ impl FlowWorkerManager {
                                         common_time::Timestamp::new_millisecond(0),
                                     )]);
                                 }
+                                if row.len() != proto_schema.len() {
+                                    InternalSnafu {
+                                        reason: format!(
+                                            "Flow output row length mismatch, expect {} got {}, the columns in schema are: {:?}",
+                                            proto_schema.len(),
+                                            row.len(),
+                                            proto_schema.iter().map(|c|&c.column_name).collect_vec()
+                                        ),
+                                    }
+                                    .fail()?;
+                                }
                                 Ok(row.into())
                             })
                             .collect::<Result<Vec<_>, Error>>()?;

src/flow/src/compute/render.rs

@@ -61,7 +61,7 @@ pub struct Context<'referred, 'df> {
     pub err_collector: ErrCollector,
 }
 
-impl<'referred, 'df> Drop for Context<'referred, 'df> {
+impl Drop for Context<'_, '_> {
     fn drop(&mut self) {
         for bundle in std::mem::take(&mut self.input_collection)
             .into_values()
@@ -92,7 +92,7 @@ impl<'referred, 'df> Drop for Context<'referred, 'df> {
     }
 }
 
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     pub fn insert_global(&mut self, id: GlobalId, collection: CollectionBundle) {
         self.input_collection.insert(id, collection);
     }
@@ -120,7 +120,7 @@ impl<'referred, 'df> Context<'referred, 'df> {
     }
 }
 
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     /// Like `render_plan` but in Batch Mode
     pub fn render_plan_batch(&mut self, plan: TypedPlan) -> Result<CollectionBundle<Batch>, Error> {
         match plan.plan {

src/flow/src/compute/render/map.rs

@@ -28,7 +28,7 @@ use crate::plan::TypedPlan;
 use crate::repr::{self, DiffRow, KeyValDiffRow, Row};
 use crate::utils::ArrangeHandler;
 
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     /// Like `render_mfp` but in batch mode
     pub fn render_mfp_batch(
         &mut self,

src/flow/src/compute/render/reduce.rs

@@ -34,7 +34,7 @@ use crate::plan::{AccumulablePlan, AggrWithIndex, KeyValPlan, ReducePlan, TypedP
 use crate::repr::{self, DiffRow, KeyValDiffRow, RelationType, Row};
 use crate::utils::{ArrangeHandler, ArrangeReader, ArrangeWriter, KeyExpiryManager};
 
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     const REDUCE_BATCH: &'static str = "reduce_batch";
     /// Like `render_reduce`, but for batch mode, and only barebone implementation
     /// no support for distinct aggregation for now

src/flow/src/compute/render/src_sink.rs

@@ -31,7 +31,7 @@ use crate::expr::{Batch, EvalError};
 use crate::repr::{DiffRow, Row, BROADCAST_CAP};
 
 #[allow(clippy::mutable_key_type)]
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     /// simply send the batch to downstream, without fancy features like buffering
     pub fn render_source_batch(
         &mut self,

src/flow/src/df_optimizer.rs

@@ -106,7 +106,7 @@ pub async fn sql_to_flow_plan(
         .context(ExternalSnafu)?;
     let plan = engine
         .planner()
-        .plan(stmt, query_ctx)
+        .plan(&stmt, query_ctx)
         .await
         .map_err(BoxedError::new)
         .context(ExternalSnafu)?;
@@ -273,7 +273,7 @@ impl<'a> ExpandAvgRewriter<'a> {
     }
 }
 
-impl<'a> TreeNodeRewriter for ExpandAvgRewriter<'a> {
+impl TreeNodeRewriter for ExpandAvgRewriter<'_> {
     type Node = Expr;
 
     fn f_up(&mut self, expr: Expr) -> Result<Transformed<Expr>, DataFusionError> {