### Added Deduplication and Merge Functionality

- Introduced `DedupReader` and `MergeReader` in `src/mito2/src/read/sync/dedup.rs` and `src/mito2/src/read/sync/merge.rs` to handle deduplication and merging of sorted batches.
 ### Enhanced `BulkMemtable` Iteration
 - Updated `BulkMemtable` in `src/mito2/src/memtable/bulk.rs` to support deduplication and merge modes during iteration.
 - Added `BulkIterContext` to manage iteration context.
 ### Testing Enhancements
 - Added comprehensive tests for `BulkMemtable` and `BulkPart` in `src/mito2/src/memtable/bulk.rs` and `src/mito2/src/memtable/bulk/part.rs`.
 ### Code Refactoring
 - Made `BulkPart` and `BulkPartMeta` cloneable in `src/mito2/src/memtable/bulk/part.rs`.
 - Exposed internal test modules for better test coverage in `src/mito2/src/memtable/time_series.rs` and `src/mito2/src/read/merge.rs`.
 ### New Modules
 - Created `sync` module in `src/mito2/src/read.rs` to organize synchronous read operations.

.github/actions/build-linux-artifacts/action.yml

@@ -48,8 +48,7 @@ runs:
         path: /tmp/greptime-*.log
         retention-days: 3
 
-    - name: Build greptime
-      if: ${{ inputs.dev-mode == 'false' }}
+    - name: Build greptime # Builds standard greptime binary
       uses: ./.github/actions/build-greptime-binary
       with:
         base-image: ubuntu

.github/workflows/dependency-check.yml

@@ -1,9 +1,6 @@
 name: Check Dependencies
 
 on:
-  push:
-    branches:
-      - main
   pull_request:
     branches:
       - main

.github/workflows/develop.yml

@@ -1,4 +1,6 @@
 on:
+  schedule:
+    - cron: "0 15 * * 1-5"
   merge_group:
   pull_request:
     types: [ opened, synchronize, reopened, ready_for_review ]
@@ -43,7 +45,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ windows-2022, ubuntu-20.04 ]
+        os: [ ubuntu-20.04 ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
@@ -51,12 +53,14 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      # - name: Rust Cache
-      #   uses: Swatinem/rust-cache@v2
-      #   with:
-      #     # Shares across multiple jobs
-      #     # Shares with `Clippy` job
-      #     shared-key: "check-lint"
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # Shares across multiple jobs
+          # Shares with `Clippy` job
+          shared-key: "check-lint"
+          cache-all-crates: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Run cargo check
         run: cargo check --locked --workspace --all-targets
 
@@ -67,11 +71,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      # - name: Rust Cache
-      #   uses: Swatinem/rust-cache@v2
-      #   with:
-      #     # Shares across multiple jobs
-      #     shared-key: "check-toml"
       - name: Install taplo
         run: cargo +stable install taplo-cli --version ^0.9 --locked --force
       - name: Run taplo
@@ -94,6 +93,8 @@ jobs:
         with:
           # Shares across multiple jobs
           shared-key: "build-binaries"
+          cache-all-crates: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install cargo-gc-bin
         shell: bash
         run: cargo install cargo-gc-bin --force
@@ -142,11 +143,6 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      # - name: Rust Cache
-      #   uses: Swatinem/rust-cache@v2
-      #   with:
-      #     # Shares across multiple jobs
-      #     shared-key: "fuzz-test-targets"
       - name: Set Rust Fuzz
         shell: bash
         run: |
@@ -200,11 +196,6 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      # - name: Rust Cache
-      #   uses: Swatinem/rust-cache@v2
-      #   with:
-      #     # Shares across multiple jobs
-      #     shared-key: "fuzz-test-targets"
       - name: Set Rust Fuzz
         shell: bash
         run: |
@@ -255,6 +246,8 @@ jobs:
         with:
           # Shares across multiple jobs
           shared-key: "build-greptime-ci"
+          cache-all-crates: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install cargo-gc-bin
         shell: bash
         run: cargo install cargo-gc-bin --force
@@ -317,11 +310,6 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      # - name: Rust Cache
-      #   uses: Swatinem/rust-cache@v2
-      #   with:
-      #     # Shares across multiple jobs
-      #     shared-key: "fuzz-test-targets"
       - name: Set Rust Fuzz
         shell: bash
         run: |
@@ -466,11 +454,6 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      # - name: Rust Cache
-      #   uses: Swatinem/rust-cache@v2
-      #   with:
-      #     # Shares across multiple jobs
-      #     shared-key: "fuzz-test-targets"
       - name: Set Rust Fuzz
         shell: bash
         run: |
@@ -612,11 +595,6 @@ jobs:
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           components: rustfmt
-      # - name: Rust Cache
-      #   uses: Swatinem/rust-cache@v2
-      #   with:
-      #     # Shares across multiple jobs
-      #     shared-key: "check-rust-fmt"
       - name: Check format
         run: make fmt-check
 
@@ -638,6 +616,8 @@ jobs:
           # Shares across multiple jobs
           # Shares with `Check` job
           shared-key: "check-lint"
+          cache-all-crates: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Run cargo clippy
         run: make clippy
 
@@ -649,11 +629,59 @@ jobs:
       - name: Merge Conflict Finder
         uses: olivernybroe/action-conflict-finder@v4.0
 
-  coverage:
-    if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04-8-cores
+  test:
+    if: github.event_name != 'merge_group'
+    runs-on: ubuntu-22.04-arm
     timeout-minutes: 60
     needs:  [conflict-check, clippy, fmt]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: rui314/setup-mold@v1
+      - name: Install toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+            cache: false
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # Shares cross multiple jobs
+          shared-key: "coverage-test"
+          cache-all-crates: "true"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@nextest
+      - name: Setup external services
+        working-directory: tests-integration/fixtures
+        run: docker compose up -d --wait
+      - name: Run nextest cases
+        run: cargo nextest run --workspace -F dashboard -F pg_kvbackend
+        env:
+          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
+          RUST_BACKTRACE: 1
+          RUST_MIN_STACK: 8388608 # 8MB
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
+          GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          GT_MINIO_BUCKET: greptime
+          GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
+          GT_MINIO_ACCESS_KEY: superpower_password
+          GT_MINIO_REGION: us-west-2
+          GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
+          GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
+          GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
+          GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
+          GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
+          UNITTEST_LOG_DIR: "__unittest_logs"
+
+  coverage:
+    if: github.event_name == 'merge_group'
+    runs-on: ubuntu-20.04-8-cores
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - uses: arduino/setup-protoc@v3
@@ -671,11 +699,6 @@ jobs:
           # Shares cross multiple jobs
           shared-key: "coverage-test"
           save-if: ${{ github.ref == 'refs/heads/main' }}
-      # Disabled temporarily to see performance
-      # - name: Docker Cache
-      #   uses: ScribeMD/docker-cache@0.5.0
-      #   with:
-      #     key: docker-${{ runner.os }}-coverage
       - name: Install latest nextest release
         uses: taiki-e/install-action@nextest
       - name: Install cargo-llvm-cov

.github/workflows/docs.yml

@@ -66,6 +66,11 @@ jobs:
     steps:
       - run: 'echo "No action required"'
 
+  test:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: 'echo "No action required"'
+
   sqlness:
     name: Sqlness Test (${{ matrix.mode.name }})
     runs-on: ${{ matrix.os }}

.github/workflows/nightly-ci.yml

@@ -108,61 +108,16 @@ jobs:
           GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
           UNITTEST_LOG_DIR: "__unittest_logs"
 
-  ## this is designed for generating cache that usable for pull requests
-  test-on-linux:
-    name: Run tests on Linux
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-20.04-8-cores
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v4
-      - uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: rui314/setup-mold@v1
-      - name: Install Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares cross multiple jobs
-          shared-key: "coverage-test"
-      - name: Install Cargo Nextest
-        uses: taiki-e/install-action@nextest
-      - name: Setup external services
-        working-directory: tests-integration/fixtures
-        run: docker compose up -d --wait
-      - name: Running tests
-        run: cargo nextest run -F dashboard -F pg_kvbackend
-        env:
-          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
-          RUST_BACKTRACE: 1
-          CARGO_INCREMENTAL: 0
-          GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
-          GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
-          GT_MINIO_BUCKET: greptime
-          GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
-          GT_MINIO_ACCESS_KEY: superpower_password
-          GT_MINIO_REGION: us-west-2
-          GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
-          GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
-          GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
-          GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
-          GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
-          UNITTEST_LOG_DIR: "__unittest_logs"
-
   cleanbuild-linux-nix:
     name: Run clean build on Linux
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - uses: cachix/install-nix-action@v27
         with:
-          nix_path: nixpkgs=channel:nixos-unstable
-      - run: nix-shell --pure --run "cargo build"
+          nix_path: nixpkgs=channel:nixos-24.11
+      - run: nix develop --command cargo build
 
   check-status:
     name: Check status

.github/workflows/release.yml

@@ -436,6 +436,22 @@ jobs:
           aws-region: ${{ vars.EC2_RUNNER_REGION }}
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
 
+  bump-doc-version:
+    name: Bump doc version
+    if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [allocate-runners]
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-cyborg
+      - name: Bump doc version
+        working-directory: cyborg
+        run: pnpm tsx bin/bump-doc-version.ts
+        env:
+          VERSION: ${{ needs.allocate-runners.outputs.version }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCS_REPO_TOKEN: ${{ secrets.DOCS_REPO_TOKEN }}
+
   notification:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || github.event_name == 'schedule') && always() }}
     name: Send notification to Greptime team

Cargo.toml

@@ -88,14 +88,17 @@ rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
 # See for more detaiils: https://github.com/rust-lang/cargo/issues/11329
 ahash = { version = "0.8", features = ["compile-time-rng"] }
 aquamarine = "0.3"
-arrow = { version = "51.0.0", features = ["prettyprint"] }
-arrow-array = { version = "51.0.0", default-features = false, features = ["chrono-tz"] }
-arrow-flight = "51.0"
-arrow-ipc = { version = "51.0.0", default-features = false, features = ["lz4", "zstd"] }
-arrow-schema = { version = "51.0", features = ["serde"] }
+arrow = { version = "53.0.0", features = ["prettyprint"] }
+arrow-array = { version = "53.0.0", default-features = false, features = ["chrono-tz"] }
+arrow-flight = "53.0"
+arrow-ipc = { version = "53.0.0", default-features = false, features = ["lz4", "zstd"] }
+arrow-schema = { version = "53.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
-axum = { version = "0.6", features = ["headers"] }
+# Remember to update axum-extra, axum-macros when updating axum
+axum = "0.8"
+axum-extra = "0.10"
+axum-macros = "0.4"
 backon = "1"
 base64 = "0.21"
 bigdecimal = "0.4.2"
@@ -107,32 +110,36 @@ clap = { version = "4.4", features = ["derive"] }
 config = "0.13.0"
 crossbeam-utils = "0.8"
 dashmap = "5.4"
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
-datafusion-functions = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
-datafusion-physical-plan = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
+datafusion = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
+datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
+datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
+datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
+datafusion-optimizer = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
+datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
+datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
+datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
+datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "2464703c84c400a09cc59277018813f0e797bb4e" }
 deadpool = "0.10"
 deadpool-postgres = "0.12"
 derive_builder = "0.12"
 dotenv = "0.15"
-etcd-client = "0.13"
+etcd-client = "0.14"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "43ddd8dea69f4df0fe2e8b5cdc0044d2cfa35908" }
+# branch: poc-write-path
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "1915576b113a494f5352fd61f211d899b7f87aab" }
 hex = "0.4"
-http = "0.2"
+http = "1"
 humantime = "2.1"
 humantime-serde = "1.1"
+hyper = "1.1"
+hyper-util = "0.1"
 itertools = "0.10"
 jsonb = { git = "https://github.com/databendlabs/jsonb.git", rev = "8c8d2fc294a39f3ff08909d60f718639cfba3875", default-features = false }
 lazy_static = "1.4"
 local-ip-address = "0.6"
+loki-api = { git = "https://github.com/shuiyisong/tracing-loki", branch = "chore/prost_version" }
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "a10facb353b41460eeb98578868ebf19c2084fac" }
 mockall = "0.11.4"
 moka = "0.12"
@@ -140,7 +147,7 @@ nalgebra = "0.33"
 notify = "6.1"
 num_cpus = "1.16"
 once_cell = "1.18"
-opentelemetry-proto = { version = "0.5", features = [
+opentelemetry-proto = { version = "0.27", features = [
     "gen-tonic",
     "metrics",
     "trace",
@@ -148,12 +155,12 @@ opentelemetry-proto = { version = "0.5", features = [
     "logs",
 ] }
 parking_lot = "0.12"
-parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
+parquet = { version = "53.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
 paste = "1.0"
 pin-project = "1.0"
 prometheus = { version = "0.13.3", features = ["process"] }
 promql-parser = { version = "0.4.3", features = ["ser"] }
-prost = "0.12"
+prost = "0.13"
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.8"
 ratelimit = "0.9"
@@ -172,17 +179,17 @@ rstest = "0.21"
 rstest_reuse = "0.7"
 rust_decimal = "1.33"
 rustc-hash = "2.0"
+rustls = { version = "0.23.20", default-features = false } # override by patch, see [patch.crates-io]
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
 serde_with = "3"
-shadow-rs = "0.35"
+shadow-rs = "0.38"
 similar-asserts = "1.6.0"
 smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
 sysinfo = "0.30"
-
-rustls = { version = "0.23.20", default-features = false } # override by patch, see [patch.crates-io]
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "54a267ac89c09b11c0c88934690530807185d3e7", features = [
+# on branch v0.52.x
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "71dd86058d2af97b9925093d40c4e03360403170", features = [
     "visitor",
     "serde",
 ] } # on branch v0.44.x
@@ -194,8 +201,8 @@ tokio-rustls = { version = "0.26.0", default-features = false } # override by pa
 tokio-stream = "0.1"
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
-tonic = { version = "0.11", features = ["tls", "gzip", "zstd"] }
-tower = "0.4"
+tonic = { version = "0.12", features = ["tls", "gzip", "zstd"] }
+tower = "0.5"
 tracing-appender = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
 typetag = "0.2"

Makefile

@@ -8,7 +8,7 @@ CARGO_BUILD_OPTS := --locked
 IMAGE_REGISTRY ?= docker.io
 IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
-DEV_BUILDER_IMAGE_TAG ?= 2024-10-19-a5c00e85-20241024184445
+DEV_BUILDER_IMAGE_TAG ?= 2024-12-25-9d0fa5d5-20250124085746
 BUILDX_MULTI_PLATFORM_BUILD ?= false
 BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu
@@ -165,15 +165,14 @@ nextest: ## Install nextest tools.
 sqlness-test: ## Run sqlness test.
 	cargo sqlness ${SQLNESS_OPTS}
 
-# Run fuzz test ${FUZZ_TARGET}.
 RUNS ?= 1
 FUZZ_TARGET ?= fuzz_alter_table
 .PHONY: fuzz
-fuzz:
+fuzz: ## Run fuzz test ${FUZZ_TARGET}.
 	cargo fuzz run ${FUZZ_TARGET} --fuzz-dir tests-fuzz -D -s none -- -runs=${RUNS}
 
 .PHONY: fuzz-ls
-fuzz-ls:
+fuzz-ls: ## List all fuzz targets.
 	cargo fuzz list --fuzz-dir tests-fuzz
 
 .PHONY: check

README.md

@@ -13,7 +13,7 @@
   <a href="https://greptime.com/product/cloud">GreptimeCloud</a> |
   <a href="https://docs.greptime.com/">User Guide</a> |
   <a href="https://greptimedb.rs/">API Docs</a> |
-  <a href="https://github.com/GreptimeTeam/greptimedb/issues/3412">Roadmap 2024</a>
+  <a href="https://github.com/GreptimeTeam/greptimedb/issues/5446">Roadmap 2025</a>
 </h4>
 
 <a href="https://github.com/GreptimeTeam/greptimedb/releases/latest">

config/config.md

@@ -26,6 +26,8 @@
 | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
 | `http.timeout` | String | `30s` | HTTP request timeout. Set to 0 to disable timeout. |
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
+| `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
+| `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
@@ -91,6 +93,8 @@
 | `procedure` | -- | -- | Procedure storage options. |
 | `procedure.max_retry_times` | Integer | `3` | Procedure max retry time. |
 | `procedure.retry_delay` | String | `500ms` | Initial retry delay of procedures, increases exponentially |
+| `flow` | -- | -- | flow engine options. |
+| `flow.num_workers` | Integer | `0` | The number of flow worker in flownode.<br/>Not setting(or set to 0) this value will use the number of CPU cores divided by 2. |
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
@@ -143,15 +147,15 @@
 | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. |
 | `region_engine.mito.index.aux_path` | String | `""` | Auxiliary directory path for the index in filesystem, used to store intermediate files for<br/>creating the index and staging files for searching the index, defaults to `{data_home}/index_intermediate`.<br/>The default name for this directory is `index_intermediate` for backward compatibility.<br/><br/>This path contains two subdirectories:<br/>- `__intm`: for storing intermediate files used during creating index.<br/>- `staging`: for storing staging files used during searching index. |
 | `region_engine.mito.index.staging_size` | String | `2GB` | The max capacity of the staging directory. |
+| `region_engine.mito.index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. |
+| `region_engine.mito.index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. |
+| `region_engine.mito.index.content_cache_page_size` | String | `64KiB` | Page size for inverted index content cache. |
 | `region_engine.mito.inverted_index` | -- | -- | The options for inverted index in Mito engine. |
 | `region_engine.mito.inverted_index.create_on_flush` | String | `auto` | Whether to create the index on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.inverted_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.inverted_index.apply_on_query` | String | `auto` | Whether to apply the index on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.inverted_index.mem_threshold_on_create` | String | `auto` | Memory threshold for performing an external sort during index creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
 | `region_engine.mito.inverted_index.intermediate_path` | String | `""` | Deprecated, use `region_engine.mito.index.aux_path` instead. |
-| `region_engine.mito.inverted_index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. |
-| `region_engine.mito.inverted_index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. |
-| `region_engine.mito.inverted_index.content_cache_page_size` | String | `64KiB` | Page size for inverted index content cache. |
 | `region_engine.mito.fulltext_index` | -- | -- | The options for full-text index in Mito engine. |
 | `region_engine.mito.fulltext_index.create_on_flush` | String | `auto` | Whether to create the index on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
@@ -168,6 +172,8 @@
 | `region_engine.mito.memtable.data_freeze_threshold` | Integer | `32768` | The max rows of data inside the actively writing buffer in one shard.<br/>Only available for `partition_tree` memtable. |
 | `region_engine.mito.memtable.fork_dictionary_bytes` | String | `1GiB` | Max dictionary bytes.<br/>Only available for `partition_tree` memtable. |
 | `region_engine.file` | -- | -- | Enable the file engine. |
+| `region_engine.metric` | -- | -- | Metric engine options. |
+| `region_engine.metric.experimental_sparse_primary_key_encoding` | Bool | `false` | Whether to enable the experimental sparse primary key encoding. |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
@@ -212,6 +218,8 @@
 | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
 | `http.timeout` | String | `30s` | HTTP request timeout. Set to 0 to disable timeout. |
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
+| `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
+| `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
 | `grpc.hostname` | String | `127.0.0.1:4001` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
@@ -325,7 +333,7 @@
 | `wal.auto_create_topics` | Bool | `true` | Automatically create topics for WAL.<br/>Set to `true` to automatically create topics for WAL.<br/>Otherwise, use topics named `topic_name_prefix_[0..num_topics)` |
 | `wal.num_topics` | Integer | `64` | Number of topics. |
 | `wal.selector_type` | String | `round_robin` | Topic selector type.<br/>Available selector types:<br/>- `round_robin` (default) |
-| `wal.topic_name_prefix` | String | `greptimedb_wal_topic` | A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.<br/>i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1. |
+| `wal.topic_name_prefix` | String | `greptimedb_wal_topic` | A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.<br/>Only accepts strings that match the following regular expression pattern:<br/>[a-zA-Z_:-][a-zA-Z0-9_:\-\.@#]*<br/>i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1. |
 | `wal.replication_factor` | Integer | `1` | Expected number of replicas of each partition. |
 | `wal.create_topic_timeout` | String | `30s` | Above which a topic creation operation will be cancelled. |
 | `wal.backoff_init` | String | `500ms` | The initial backoff for kafka clients. |
@@ -479,15 +487,15 @@
 | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. |
 | `region_engine.mito.index.aux_path` | String | `""` | Auxiliary directory path for the index in filesystem, used to store intermediate files for<br/>creating the index and staging files for searching the index, defaults to `{data_home}/index_intermediate`.<br/>The default name for this directory is `index_intermediate` for backward compatibility.<br/><br/>This path contains two subdirectories:<br/>- `__intm`: for storing intermediate files used during creating index.<br/>- `staging`: for storing staging files used during searching index. |
 | `region_engine.mito.index.staging_size` | String | `2GB` | The max capacity of the staging directory. |
+| `region_engine.mito.index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. |
+| `region_engine.mito.index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. |
+| `region_engine.mito.index.content_cache_page_size` | String | `64KiB` | Page size for inverted index content cache. |
 | `region_engine.mito.inverted_index` | -- | -- | The options for inverted index in Mito engine. |
 | `region_engine.mito.inverted_index.create_on_flush` | String | `auto` | Whether to create the index on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.inverted_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.inverted_index.apply_on_query` | String | `auto` | Whether to apply the index on query<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.inverted_index.mem_threshold_on_create` | String | `auto` | Memory threshold for performing an external sort during index creation.<br/>- `auto`: automatically determine the threshold based on the system memory size (default)<br/>- `unlimited`: no memory limit<br/>- `[size]` e.g. `64MB`: fixed memory threshold |
 | `region_engine.mito.inverted_index.intermediate_path` | String | `""` | Deprecated, use `region_engine.mito.index.aux_path` instead. |
-| `region_engine.mito.inverted_index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. |
-| `region_engine.mito.inverted_index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. |
-| `region_engine.mito.inverted_index.content_cache_page_size` | String | `64KiB` | Page size for inverted index content cache. |
 | `region_engine.mito.fulltext_index` | -- | -- | The options for full-text index in Mito engine. |
 | `region_engine.mito.fulltext_index.create_on_flush` | String | `auto` | Whether to create the index on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
@@ -504,6 +512,8 @@
 | `region_engine.mito.memtable.data_freeze_threshold` | Integer | `32768` | The max rows of data inside the actively writing buffer in one shard.<br/>Only available for `partition_tree` memtable. |
 | `region_engine.mito.memtable.fork_dictionary_bytes` | String | `1GiB` | Max dictionary bytes.<br/>Only available for `partition_tree` memtable. |
 | `region_engine.file` | -- | -- | Enable the file engine. |
+| `region_engine.metric` | -- | -- | Metric engine options. |
+| `region_engine.metric.experimental_sparse_primary_key_encoding` | Bool | `false` | Whether to enable the experimental sparse primary key encoding. |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
@@ -536,12 +546,18 @@
 | --- | -----| ------- | ----------- |
 | `mode` | String | `distributed` | The running mode of the flownode. It can be `standalone` or `distributed`. |
 | `node_id` | Integer | Unset | The flownode identifier and should be unique in the cluster. |
+| `flow` | -- | -- | flow engine options. |
+| `flow.num_workers` | Integer | `0` | The number of flow worker in flownode.<br/>Not setting(or set to 0) this value will use the number of CPU cores divided by 2. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:6800` | The address to bind the gRPC server. |
 | `grpc.hostname` | String | `127.0.0.1` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
 | `grpc.runtime_size` | Integer | `2` | The number of server worker threads. |
 | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
+| `http` | -- | -- | The HTTP server options. |
+| `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
+| `http.timeout` | String | `30s` | HTTP request timeout. Set to 0 to disable timeout. |
+| `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `meta_client` | -- | -- | The metasrv client options. |
 | `meta_client.metasrv_addrs` | Array | -- | The addresses of the metasrv. |
 | `meta_client.timeout` | String | `3s` | Operation timeout. |

config/datanode.example.toml

@@ -516,6 +516,15 @@ aux_path = ""
 ## The max capacity of the staging directory.
 staging_size = "2GB"
 
+## Cache size for inverted index metadata.
+metadata_cache_size = "64MiB"
+
+## Cache size for inverted index content.
+content_cache_size = "128MiB"
+
+## Page size for inverted index content cache.
+content_cache_page_size = "64KiB"
+
 ## The options for inverted index in Mito engine.
 [region_engine.mito.inverted_index]
 
@@ -543,15 +552,6 @@ mem_threshold_on_create = "auto"
 ## Deprecated, use `region_engine.mito.index.aux_path` instead.
 intermediate_path = ""
 
-## Cache size for inverted index metadata.
-metadata_cache_size = "64MiB"
-
-## Cache size for inverted index content.
-content_cache_size = "128MiB"
-
-## Page size for inverted index content cache.
-content_cache_page_size = "64KiB"
-
 ## The options for full-text index in Mito engine.
 [region_engine.mito.fulltext_index]
 
@@ -622,6 +622,12 @@ fork_dictionary_bytes = "1GiB"
 ## Enable the file engine.
 [region_engine.file]
 
+[[region_engine]]
+## Metric engine options.
+[region_engine.metric]
+## Whether to enable the experimental sparse primary key encoding.
+experimental_sparse_primary_key_encoding = false
+
 ## The logging options.
 [logging]
 ## The directory to store the log files. If set to empty, logs will not be written to files.

config/flownode.example.toml

@@ -5,6 +5,12 @@ mode = "distributed"
 ## @toml2docs:none-default
 node_id = 14
 
+## flow engine options.
+[flow]
+## The number of flow worker in flownode.
+## Not setting(or set to 0) this value will use the number of CPU cores divided by 2.
+#+num_workers=0
+
 ## The gRPC server options.
 [grpc]
 ## The address to bind the gRPC server.
@@ -19,6 +25,16 @@ max_recv_message_size = "512MB"
 ## The maximum send message size for gRPC server.
 max_send_message_size = "512MB"
 
+## The HTTP server options.
+[http]
+## The address to bind the HTTP server.
+addr = "127.0.0.1:4000"
+## HTTP request timeout. Set to 0 to disable timeout.
+timeout = "30s"
+## HTTP request body limit.
+## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
+## Set to 0 to disable limit.
+body_limit = "64MB"
 
 ## The metasrv client options.
 [meta_client]

config/frontend.example.toml

@@ -31,6 +31,12 @@ timeout = "30s"
 ## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
 ## Set to 0 to disable limit.
 body_limit = "64MB"
+## HTTP CORS support, it's turned on by default
+## This allows browser to access http APIs without CORS restrictions
+enable_cors = true
+## Customize allowed origins for HTTP CORS.
+## @toml2docs:none-default
+cors_allowed_origins = ["https://example.com"]
 
 ## The gRPC server options.
 [grpc]

config/metasrv.example.toml

@@ -129,6 +129,8 @@ num_topics = 64
 selector_type = "round_robin"
 
 ## A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
+## Only accepts strings that match the following regular expression pattern:
+## [a-zA-Z_:-][a-zA-Z0-9_:\-\.@#]*
 ## i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1.
 topic_name_prefix = "greptimedb_wal_topic"
 

config/standalone.example.toml

@@ -39,6 +39,12 @@ timeout = "30s"
 ## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
 ## Set to 0 to disable limit.
 body_limit = "64MB"
+## HTTP CORS support, it's turned on by default
+## This allows browser to access http APIs without CORS restrictions
+enable_cors = true
+## Customize allowed origins for HTTP CORS.
+## @toml2docs:none-default
+cors_allowed_origins = ["https://example.com"]
 
 ## The gRPC server options.
 [grpc]
@@ -284,6 +290,12 @@ max_retry_times = 3
 ## Initial retry delay of procedures, increases exponentially
 retry_delay = "500ms"
 
+## flow engine options.
+[flow]
+## The number of flow worker in flownode.
+## Not setting(or set to 0) this value will use the number of CPU cores divided by 2.
+#+num_workers=0
+
 # Example of using S3 as the storage.
 # [storage]
 # type = "S3"
@@ -559,6 +571,15 @@ aux_path = ""
 ## The max capacity of the staging directory.
 staging_size = "2GB"
 
+## Cache size for inverted index metadata.
+metadata_cache_size = "64MiB"
+
+## Cache size for inverted index content.
+content_cache_size = "128MiB"
+
+## Page size for inverted index content cache.
+content_cache_page_size = "64KiB"
+
 ## The options for inverted index in Mito engine.
 [region_engine.mito.inverted_index]
 
@@ -586,15 +607,6 @@ mem_threshold_on_create = "auto"
 ## Deprecated, use `region_engine.mito.index.aux_path` instead.
 intermediate_path = ""
 
-## Cache size for inverted index metadata.
-metadata_cache_size = "64MiB"
-
-## Cache size for inverted index content.
-content_cache_size = "128MiB"
-
-## Page size for inverted index content cache.
-content_cache_page_size = "64KiB"
-
 ## The options for full-text index in Mito engine.
 [region_engine.mito.fulltext_index]
 
@@ -665,6 +677,12 @@ fork_dictionary_bytes = "1GiB"
 ## Enable the file engine.
 [region_engine.file]
 
+[[region_engine]]
+## Metric engine options.
+[region_engine.metric]
+## Whether to enable the experimental sparse primary key encoding.
+experimental_sparse_primary_key_encoding = false
+
 ## The logging options.
 [logging]
 ## The directory to store the log files. If set to empty, logs will not be written to files.

cyborg/bin/bump-doc-version.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2023 Greptime Team
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as core from "@actions/core";
+import {obtainClient} from "@/common";
+
+async function triggerWorkflow(workflowId: string, version: string) {
+  const docsClient = obtainClient("DOCS_REPO_TOKEN")
+  try {
+    await docsClient.rest.actions.createWorkflowDispatch({
+      owner: "GreptimeTeam",
+      repo: "docs",
+      workflow_id: workflowId,
+      ref: "main",
+      inputs: {
+        version,
+      },
+    });
+    console.log(`Successfully triggered ${workflowId} workflow with version ${version}`);
+  } catch (error) {
+    core.setFailed(`Failed to trigger workflow: ${error.message}`);
+  }
+}
+
+function determineWorkflow(version: string): [string, string] {
+  // Check if it's a nightly version
+  if (version.includes('nightly')) {
+    return ['bump-nightly-version.yml', version];
+  }
+
+  const parts = version.split('.');
+
+  if (parts.length !== 3) {
+    throw new Error('Invalid version format');
+  }
+
+  // If patch version (last number) is 0, it's a major version
+  // Return only major.minor version
+  if (parts[2] === '0') {
+    return ['bump-version.yml', `${parts[0]}.${parts[1]}`];
+  }
+
+  // Otherwise it's a patch version, use full version
+  return ['bump-patch-version.yml', version];
+}
+
+const version = process.env.VERSION;
+if (!version) {
+  core.setFailed("VERSION environment variable is required");
+  process.exit(1);
+}
+
+// Remove 'v' prefix if exists
+const cleanVersion = version.startsWith('v') ? version.slice(1) : version;
+
+try {
+  const [workflowId, apiVersion] = determineWorkflow(cleanVersion);
+  triggerWorkflow(workflowId, apiVersion);
+} catch (error) {
+  core.setFailed(`Error processing version: ${error.message}`);
+  process.exit(1);
+}

docker/dev-builder/android/Dockerfile

@@ -9,12 +9,21 @@ RUN cp ${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/14.0.7/lib/
 # Install dependencies.
 RUN apt-get update && apt-get install -y \
     libssl-dev \
-    protobuf-compiler \
     curl \
     git \
+    unzip \
     build-essential \
     pkg-config
 
+# Install protoc
+ARG PROTOBUF_VERSION=29.3
+
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip && \
+    unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d protoc3;
+    
+RUN mv protoc3/bin/* /usr/local/bin/
+RUN mv protoc3/include/* /usr/local/include/
+
 # Trust workdir
 RUN git config --global --add safe.directory /greptimedb
 

docker/dev-builder/centos/Dockerfile

@@ -15,8 +15,13 @@ RUN yum install -y epel-release  \
     which
 
 # Install protoc
-RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
-RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
+ARG PROTOBUF_VERSION=29.3
+
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip && \
+    unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d protoc3;
+    
+RUN mv protoc3/bin/* /usr/local/bin/
+RUN mv protoc3/include/* /usr/local/include/
 
 # Install Rust
 SHELL ["/bin/bash", "-c"]

docker/dev-builder/ubuntu/Dockerfile

@@ -22,13 +22,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 ARG TARGETPLATFORM
 RUN echo "target platform: $TARGETPLATFORM"
 
+ARG PROTOBUF_VERSION=29.3
+
 # Install protobuf, because the one in the apt is too old (v3.12).
 RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
-    curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v29.1/protoc-29.1-linux-aarch_64.zip && \
-    unzip protoc-29.1-linux-aarch_64.zip -d protoc3; \
+    curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-aarch_64.zip && \
+    unzip protoc-${PROTOBUF_VERSION}-linux-aarch_64.zip -d protoc3; \
 elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
-    curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v29.1/protoc-29.1-linux-x86_64.zip && \
-    unzip protoc-29.1-linux-x86_64.zip -d protoc3; \
+    curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip && \
+    unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d protoc3; \
 fi
 RUN mv protoc3/bin/* /usr/local/bin/
 RUN mv protoc3/include/* /usr/local/include/

docker/dev-builder/ubuntu/Dockerfile-18.10

@@ -21,7 +21,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     pkg-config
 
 # Install protoc.
-ENV PROTOC_VERSION=25.1
+ENV PROTOC_VERSION=29.3
 RUN if [ "$(uname -m)" = "x86_64" ]; then \
         PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-x86_64.zip; \
     elif [ "$(uname -m)" = "aarch64" ]; then \

docker/docker-compose/cluster-with-etcd.yaml

@@ -39,14 +39,16 @@ services:
     container_name: metasrv
     ports:
       - 3002:3002
+      - 3000:3000
     command:
       - metasrv
       - start
       - --bind-addr=0.0.0.0:3002
       - --server-addr=metasrv:3002
       - --store-addrs=etcd0:2379
+      - --http-addr=0.0.0.0:3000
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://metasrv:3002/health" ]
+      test: [ "CMD", "curl", "-f", "http://metasrv:3000/health" ]
       interval: 5s
       timeout: 3s
       retries: 5
@@ -73,10 +75,10 @@ services:
     volumes:
       - /tmp/greptimedb-cluster-docker-compose/datanode0:/tmp/greptimedb
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://datanode0:5000/health" ]
+      test: [ "CMD", "curl", "-fv", "http://datanode0:5000/health" ]
       interval: 5s
       timeout: 3s
-      retries: 5
+      retries: 10
     depends_on:
       metasrv:
         condition: service_healthy
@@ -115,6 +117,7 @@ services:
     container_name: flownode0
     ports:
       - 4004:4004
+      - 4005:4005
     command:
       - flownode
       - start
@@ -122,9 +125,15 @@ services:
       - --metasrv-addrs=metasrv:3002
       - --rpc-addr=0.0.0.0:4004
       - --rpc-hostname=flownode0:4004
+      - --http-addr=0.0.0.0:4005
     depends_on:
       frontend0:
         condition: service_healthy
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://flownode0:4005/health" ]
+      interval: 5s
+      timeout: 3s
+      retries: 5
     networks:
       - greptimedb
 

flake.lock

@@ -0,0 +1,100 @@
+{
+  "nodes": {
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1737613896,
+        "narHash": "sha256-ldqXIglq74C7yKMFUzrS9xMT/EVs26vZpOD68Sh7OcU=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "303a062fdd8e89f233db05868468975d17855d80",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1737569578,
+        "narHash": "sha256-6qY0pk2QmUtBT9Mywdvif0i/CLVgpCjMUn6g9vB+f3M=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "47addd76727f42d351590c905d9d1905ca895b82",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "fenix": "fenix",
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1737581772,
+        "narHash": "sha256-t1P2Pe3FAX9TlJsCZbmJ3wn+C4qr6aSMypAOu8WNsN0=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "582af7ee9c8d84f5d534272fc7de9f292bd849be",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,56 @@
+{
+  description = "Development environment flake";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, fenix, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+        buildInputs = with pkgs; [
+          libgit2
+          libz
+        ];
+        lib = nixpkgs.lib;
+        rustToolchain = fenix.packages.${system}.fromToolchainName {
+          name = (lib.importTOML ./rust-toolchain.toml).toolchain.channel;
+          sha256 = "sha256-f/CVA1EC61EWbh0SjaRNhLL0Ypx2ObupbzigZp8NmL4=";
+        };
+      in
+      {
+        devShells.default = pkgs.mkShell {
+          nativeBuildInputs = with pkgs; [
+            pkg-config
+            git
+            clang
+            gcc
+            protobuf
+            gnumake
+            mold
+            (rustToolchain.withComponents [
+              "cargo"
+              "clippy"
+              "rust-src"
+              "rustc"
+              "rustfmt"
+              "rust-analyzer"
+              "llvm-tools"
+            ])
+            cargo-nextest
+            cargo-llvm-cov
+            taplo
+            curl
+            gnuplot ## for cargo bench
+          ];
+
+          LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;
+        };
+      });
+}

rust-toolchain.toml

@@ -1,3 +1,2 @@
 [toolchain]
-channel = "nightly-2024-10-19"
-components = ["rust-analyzer"]
+channel = "nightly-2024-12-25"

shell.nix

@@ -1,29 +0,0 @@
-let
-  nixpkgs = fetchTarball "https://github.com/NixOS/nixpkgs/tarball/nixos-24.11";
-  fenix = import (fetchTarball "https://github.com/nix-community/fenix/archive/main.tar.gz") {};
-  pkgs = import nixpkgs { config = {}; overlays = []; };
-in
-
-pkgs.mkShell rec {
-  nativeBuildInputs = with pkgs; [
-    pkg-config
-    git
-    clang
-    gcc
-    protobuf
-    mold
-    (fenix.fromToolchainFile {
-      dir = ./.;
-    })
-    cargo-nextest
-    taplo
-    curl
-  ];
-
-  buildInputs = with pkgs; [
-    libgit2
-    libz
-  ];
-
-  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;
-}

src/api/src/error.rs

@@ -33,7 +33,7 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
         #[snafu(source)]
-        error: prost::DecodeError,
+        error: prost::UnknownEnumValue,
     },
 
     #[snafu(display("Failed to create column datatype from {:?}", from))]

src/api/src/helper.rs

@@ -86,7 +86,7 @@ impl ColumnDataTypeWrapper {
 
     /// Get a tuple of ColumnDataType and ColumnDataTypeExtension.
     pub fn to_parts(&self) -> (ColumnDataType, Option<ColumnDataTypeExtension>) {
-        (self.datatype, self.datatype_ext.clone())
+        (self.datatype, self.datatype_ext)
     }
 }
 
@@ -685,14 +685,18 @@ pub fn pb_values_to_vector_ref(data_type: &ConcreteDataType, values: Values) ->
             IntervalType::YearMonth(_) => Arc::new(IntervalYearMonthVector::from_vec(
                 values.interval_year_month_values,
             )),
-            IntervalType::DayTime(_) => Arc::new(IntervalDayTimeVector::from_vec(
-                values.interval_day_time_values,
+            IntervalType::DayTime(_) => Arc::new(IntervalDayTimeVector::from_iter_values(
+                values
+                    .interval_day_time_values
+                    .iter()
+                    .map(|x| IntervalDayTime::from_i64(*x).into()),
             )),
             IntervalType::MonthDayNano(_) => {
                 Arc::new(IntervalMonthDayNanoVector::from_iter_values(
-                    values.interval_month_day_nano_values.iter().map(|x| {
-                        IntervalMonthDayNano::new(x.months, x.days, x.nanoseconds).to_i128()
-                    }),
+                    values
+                        .interval_month_day_nano_values
+                        .iter()
+                        .map(|x| IntervalMonthDayNano::new(x.months, x.days, x.nanoseconds).into()),
                 ))
             }
         },
@@ -1495,14 +1499,22 @@ mod tests {
             column.values.as_ref().unwrap().interval_year_month_values
         );
 
-        let vector = Arc::new(IntervalDayTimeVector::from_vec(vec![4, 5, 6]));
+        let vector = Arc::new(IntervalDayTimeVector::from_vec(vec![
+            IntervalDayTime::new(0, 4).into(),
+            IntervalDayTime::new(0, 5).into(),
+            IntervalDayTime::new(0, 6).into(),
+        ]));
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![4, 5, 6],
             column.values.as_ref().unwrap().interval_day_time_values
         );
 
-        let vector = Arc::new(IntervalMonthDayNanoVector::from_vec(vec![7, 8, 9]));
+        let vector = Arc::new(IntervalMonthDayNanoVector::from_vec(vec![
+            IntervalMonthDayNano::new(0, 0, 7).into(),
+            IntervalMonthDayNano::new(0, 0, 8).into(),
+            IntervalMonthDayNano::new(0, 0, 9).into(),
+        ]));
         let len = vector.len();
         push_vals(&mut column, 3, vector);
         (0..len).for_each(|i| {

src/api/src/v1/column_def.rs

@@ -34,10 +34,8 @@ const SKIPPING_INDEX_GRPC_KEY: &str = "skipping_index";
 
 /// Tries to construct a `ColumnSchema` from the given  `ColumnDef`.
 pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
-    let data_type = ColumnDataTypeWrapper::try_new(
-        column_def.data_type,
-        column_def.datatype_extension.clone(),
-    )?;
+    let data_type =
+        ColumnDataTypeWrapper::try_new(column_def.data_type, column_def.datatype_extension)?;
 
     let constraint = if column_def.default_constraint.is_empty() {
         None
@@ -57,13 +55,13 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
     }
     if let Some(options) = column_def.options.as_ref() {
         if let Some(fulltext) = options.options.get(FULLTEXT_GRPC_KEY) {
-            metadata.insert(FULLTEXT_KEY.to_string(), fulltext.clone());
+            metadata.insert(FULLTEXT_KEY.to_string(), fulltext.to_owned());
         }
         if let Some(inverted_index) = options.options.get(INVERTED_INDEX_GRPC_KEY) {
-            metadata.insert(INVERTED_INDEX_KEY.to_string(), inverted_index.clone());
+            metadata.insert(INVERTED_INDEX_KEY.to_string(), inverted_index.to_owned());
         }
         if let Some(skipping_index) = options.options.get(SKIPPING_INDEX_GRPC_KEY) {
-            metadata.insert(SKIPPING_INDEX_KEY.to_string(), skipping_index.clone());
+            metadata.insert(SKIPPING_INDEX_KEY.to_string(), skipping_index.to_owned());
         }
     }
 
@@ -82,7 +80,7 @@ pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option<Column
     if let Some(fulltext) = column_schema.metadata().get(FULLTEXT_KEY) {
         options
             .options
-            .insert(FULLTEXT_GRPC_KEY.to_string(), fulltext.clone());
+            .insert(FULLTEXT_GRPC_KEY.to_string(), fulltext.to_owned());
     }
     if let Some(inverted_index) = column_schema.metadata().get(INVERTED_INDEX_KEY) {
         options
@@ -102,7 +100,7 @@ pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option<Column
 pub fn contains_fulltext(options: &Option<ColumnOptions>) -> bool {
     options
         .as_ref()
-        .map_or(false, |o| o.options.contains_key(FULLTEXT_GRPC_KEY))
+        .is_some_and(|o| o.options.contains_key(FULLTEXT_GRPC_KEY))
 }
 
 /// Tries to construct a `ColumnOptions` from the given `FulltextOptions`.
@@ -181,14 +179,14 @@ mod tests {
         let options = options_from_column_schema(&schema);
         assert!(options.is_none());
 
-        let schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true)
+        let mut schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true)
             .with_fulltext_options(FulltextOptions {
                 enable: true,
                 analyzer: FulltextAnalyzer::English,
                 case_sensitive: false,
             })
-            .unwrap()
-            .set_inverted_index(true);
+            .unwrap();
+        schema.set_inverted_index(true);
         let options = options_from_column_schema(&schema).unwrap();
         assert_eq!(
             options.options.get(FULLTEXT_GRPC_KEY).unwrap(),

src/catalog/src/kvbackend/client.rs

@@ -303,7 +303,7 @@ impl KvBackend for CachedKvBackend {
             .lock()
             .unwrap()
             .as_ref()
-            .map_or(false, |v| !self.validate_version(*v))
+            .is_some_and(|v| !self.validate_version(*v))
         {
             self.cache.invalidate(key).await;
         }

src/catalog/src/lib.rs

@@ -41,6 +41,7 @@ pub mod information_schema {
 }
 
 pub mod table_source;
+
 #[async_trait::async_trait]
 pub trait CatalogManager: Send + Sync {
     fn as_any(&self) -> &dyn Any;

src/catalog/src/system_schema/information_schema/cluster_info.rs

@@ -64,6 +64,7 @@ const INIT_CAPACITY: usize = 42;
 /// - `uptime`: the uptime of the peer.
 /// - `active_time`: the time since the last activity of the peer.
 ///
+#[derive(Debug)]
 pub(super) struct InformationSchemaClusterInfo {
     schema: SchemaRef,
     catalog_manager: Weak<dyn CatalogManager>,

src/catalog/src/system_schema/information_schema/columns.rs

@@ -45,6 +45,7 @@ use crate::error::{
 use crate::information_schema::Predicates;
 use crate::CatalogManager;
 
+#[derive(Debug)]
 pub(super) struct InformationSchemaColumns {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/information_schema/flows.rs

@@ -61,7 +61,7 @@ pub const FLOWNODE_IDS: &str = "flownode_ids";
 pub const OPTIONS: &str = "options";
 
 /// The `information_schema.flows` to provides information about flows in databases.
-///
+#[derive(Debug)]
 pub(super) struct InformationSchemaFlows {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/information_schema/key_column_usage.rs

@@ -62,6 +62,7 @@ pub(crate) const FULLTEXT_INDEX_CONSTRAINT_NAME: &str = "FULLTEXT INDEX";
 pub(crate) const SKIPPING_INDEX_CONSTRAINT_NAME: &str = "SKIPPING INDEX";
 
 /// The virtual table implementation for `information_schema.KEY_COLUMN_USAGE`.
+#[derive(Debug)]
 pub(super) struct InformationSchemaKeyColumnUsage {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/information_schema/partitions.rs

@@ -59,6 +59,7 @@ const INIT_CAPACITY: usize = 42;
 /// The `PARTITIONS` table provides information about partitioned tables.
 /// See https://dev.mysql.com/doc/refman/8.0/en/information-schema-partitions-table.html
 /// We provide an extral column `greptime_partition_id` for GreptimeDB region id.
+#[derive(Debug)]
 pub(super) struct InformationSchemaPartitions {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/information_schema/procedure_info.rs

@@ -56,7 +56,7 @@ const INIT_CAPACITY: usize = 42;
 /// - `end_time`: the ending execution time of the procedure.
 /// - `status`: the status of the procedure.
 /// - `lock_keys`: the lock keys of the procedure.
-///
+#[derive(Debug)]
 pub(super) struct InformationSchemaProcedureInfo {
     schema: SchemaRef,
     catalog_manager: Weak<dyn CatalogManager>,

src/catalog/src/system_schema/information_schema/region_peers.rs

@@ -59,7 +59,7 @@ const INIT_CAPACITY: usize = 42;
 /// - `is_leader`: whether the peer is the leader
 /// - `status`: the region status, `ALIVE` or `DOWNGRADED`.
 /// - `down_seconds`: the duration of being offline, in seconds.
-///
+#[derive(Debug)]
 pub(super) struct InformationSchemaRegionPeers {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/information_schema/region_statistics.rs

@@ -63,7 +63,7 @@ const INIT_CAPACITY: usize = 42;
 /// - `index_size`: The sst index files size in bytes.
 /// - `engine`: The engine type.
 /// - `region_role`: The region role.
-///
+#[derive(Debug)]
 pub(super) struct InformationSchemaRegionStatistics {
     schema: SchemaRef,
     catalog_manager: Weak<dyn CatalogManager>,

src/catalog/src/system_schema/information_schema/runtime_metrics.rs

@@ -38,6 +38,7 @@ use store_api::storage::{ScanRequest, TableId};
 use super::{InformationTable, RUNTIME_METRICS};
 use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
 
+#[derive(Debug)]
 pub(super) struct InformationSchemaMetrics {
     schema: SchemaRef,
 }

src/catalog/src/system_schema/information_schema/schemata.rs

@@ -49,6 +49,7 @@ pub const SCHEMA_OPTS: &str = "options";
 const INIT_CAPACITY: usize = 42;
 
 /// The `information_schema.schemata` table implementation.
+#[derive(Debug)]
 pub(super) struct InformationSchemaSchemata {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/information_schema/table_constraints.rs

@@ -43,6 +43,7 @@ use crate::information_schema::Predicates;
 use crate::CatalogManager;
 
 /// The `TABLE_CONSTRAINTS` table describes which tables have constraints.
+#[derive(Debug)]
 pub(super) struct InformationSchemaTableConstraints {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/information_schema/tables.rs

@@ -71,6 +71,7 @@ const TABLE_ID: &str = "table_id";
 pub const ENGINE: &str = "engine";
 const INIT_CAPACITY: usize = 42;
 
+#[derive(Debug)]
 pub(super) struct InformationSchemaTables {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/information_schema/views.rs

@@ -54,6 +54,7 @@ pub const CHARACTER_SET_CLIENT: &str = "character_set_client";
 pub const COLLATION_CONNECTION: &str = "collation_connection";
 
 /// The `information_schema.views` to provides information about views in databases.
+#[derive(Debug)]
 pub(super) struct InformationSchemaViews {
     schema: SchemaRef,
     catalog_name: String,

src/catalog/src/system_schema/memory_table.rs

@@ -33,6 +33,7 @@ use super::SystemTable;
 use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
 
 /// A memory table with specified schema and columns.
+#[derive(Debug)]
 pub(crate) struct MemoryTable {
     pub(crate) table_id: TableId,
     pub(crate) table_name: &'static str,

src/catalog/src/system_schema/pg_catalog.rs

@@ -14,6 +14,7 @@
 
 mod pg_catalog_memory_table;
 mod pg_class;
+mod pg_database;
 mod pg_namespace;
 mod table_names;
 
@@ -26,6 +27,7 @@ use lazy_static::lazy_static;
 use paste::paste;
 use pg_catalog_memory_table::get_schema_columns;
 use pg_class::PGClass;
+use pg_database::PGDatabase;
 use pg_namespace::PGNamespace;
 use session::context::{Channel, QueryContext};
 use table::TableRef;
@@ -113,6 +115,10 @@ impl PGCatalogProvider {
             PG_CLASS.to_string(),
             self.build_table(PG_CLASS).expect(PG_NAMESPACE),
         );
+        tables.insert(
+            PG_DATABASE.to_string(),
+            self.build_table(PG_DATABASE).expect(PG_DATABASE),
+        );
         self.tables = tables;
     }
 }
@@ -135,6 +141,11 @@ impl SystemSchemaProviderInner for PGCatalogProvider {
                 self.catalog_manager.clone(),
                 self.namespace_oid_map.clone(),
             ))),
+            table_names::PG_DATABASE => Some(Arc::new(PGDatabase::new(
+                self.catalog_name.clone(),
+                self.catalog_manager.clone(),
+                self.namespace_oid_map.clone(),
+            ))),
             _ => None,
         }
     }

src/catalog/src/system_schema/pg_catalog/pg_class.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt;
 use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
@@ -100,6 +101,15 @@ impl PGClass {
     }
 }
 
+impl fmt::Debug for PGClass {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PGClass")
+            .field("schema", &self.schema)
+            .field("catalog_name", &self.catalog_name)
+            .finish()
+    }
+}
+
 impl SystemTable for PGClass {
     fn table_id(&self) -> table::metadata::TableId {
         PG_CATALOG_PG_CLASS_TABLE_ID

src/catalog/src/system_schema/pg_catalog/pg_database.rs

@@ -0,0 +1,223 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::{Arc, Weak};
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_catalog::consts::PG_CATALOG_PG_DATABASE_TABLE_ID;
+use common_error::ext::BoxedError;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{DfSendableRecordBatchStream, RecordBatch};
+use datafusion::execution::TaskContext;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::schema::{Schema, SchemaRef};
+use datatypes::value::Value;
+use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder, VectorRef};
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::ScanRequest;
+
+use super::pg_namespace::oid_map::PGNamespaceOidMapRef;
+use super::{query_ctx, OID_COLUMN_NAME, PG_DATABASE};
+use crate::error::{
+    CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
+};
+use crate::information_schema::Predicates;
+use crate::system_schema::utils::tables::{string_column, u32_column};
+use crate::system_schema::SystemTable;
+use crate::CatalogManager;
+
+// === column name ===
+pub const DATNAME: &str = "datname";
+
+/// The initial capacity of the vector builders.
+const INIT_CAPACITY: usize = 42;
+
+/// The `pg_catalog.database` table implementation.
+pub(super) struct PGDatabase {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_manager: Weak<dyn CatalogManager>,
+
+    // Workaround to convert schema_name to a numeric id
+    namespace_oid_map: PGNamespaceOidMapRef,
+}
+
+impl std::fmt::Debug for PGDatabase {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PGDatabase")
+            .field("schema", &self.schema)
+            .field("catalog_name", &self.catalog_name)
+            .finish()
+    }
+}
+
+impl PGDatabase {
+    pub(super) fn new(
+        catalog_name: String,
+        catalog_manager: Weak<dyn CatalogManager>,
+        namespace_oid_map: PGNamespaceOidMapRef,
+    ) -> Self {
+        Self {
+            schema: Self::schema(),
+            catalog_name,
+            catalog_manager,
+            namespace_oid_map,
+        }
+    }
+
+    fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
+            u32_column(OID_COLUMN_NAME),
+            string_column(DATNAME),
+        ]))
+    }
+
+    fn builder(&self) -> PGCDatabaseBuilder {
+        PGCDatabaseBuilder::new(
+            self.schema.clone(),
+            self.catalog_name.clone(),
+            self.catalog_manager.clone(),
+            self.namespace_oid_map.clone(),
+        )
+    }
+}
+
+impl DfPartitionStream for PGDatabase {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_database(None)
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}
+
+impl SystemTable for PGDatabase {
+    fn table_id(&self) -> table::metadata::TableId {
+        PG_CATALOG_PG_DATABASE_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        PG_DATABASE
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(
+        &self,
+        request: ScanRequest,
+    ) -> Result<common_recordbatch::SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_database(Some(request))
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+/// Builds the `pg_catalog.pg_database` table row by row
+/// `oid` use schema name as a workaround since we don't have numeric schema id.
+/// `nspname` is the schema name.
+struct PGCDatabaseBuilder {
+    schema: SchemaRef,
+    catalog_name: String,
+    catalog_manager: Weak<dyn CatalogManager>,
+    namespace_oid_map: PGNamespaceOidMapRef,
+
+    oid: UInt32VectorBuilder,
+    datname: StringVectorBuilder,
+}
+
+impl PGCDatabaseBuilder {
+    fn new(
+        schema: SchemaRef,
+        catalog_name: String,
+        catalog_manager: Weak<dyn CatalogManager>,
+        namespace_oid_map: PGNamespaceOidMapRef,
+    ) -> Self {
+        Self {
+            schema,
+            catalog_name,
+            catalog_manager,
+            namespace_oid_map,
+
+            oid: UInt32VectorBuilder::with_capacity(INIT_CAPACITY),
+            datname: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+        }
+    }
+
+    async fn make_database(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
+        let catalog_name = self.catalog_name.clone();
+        let catalog_manager = self
+            .catalog_manager
+            .upgrade()
+            .context(UpgradeWeakCatalogManagerRefSnafu)?;
+        let predicates = Predicates::from_scan_request(&request);
+        for schema_name in catalog_manager
+            .schema_names(&catalog_name, query_ctx())
+            .await?
+        {
+            self.add_database(&predicates, &schema_name);
+        }
+        self.finish()
+    }
+
+    fn add_database(&mut self, predicates: &Predicates, schema_name: &str) {
+        let oid = self.namespace_oid_map.get_oid(schema_name);
+        let row: [(&str, &Value); 2] = [
+            (OID_COLUMN_NAME, &Value::from(oid)),
+            (DATNAME, &Value::from(schema_name)),
+        ];
+
+        if !predicates.eval(&row) {
+            return;
+        }
+
+        self.oid.push(Some(oid));
+        self.datname.push(Some(schema_name));
+    }
+
+    fn finish(&mut self) -> Result<RecordBatch> {
+        let columns: Vec<VectorRef> =
+            vec![Arc::new(self.oid.finish()), Arc::new(self.datname.finish())];
+        RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
+    }
+}

src/catalog/src/system_schema/pg_catalog/pg_namespace.rs

@@ -17,6 +17,7 @@
 
 pub(super) mod oid_map;
 
+use std::fmt;
 use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
@@ -87,6 +88,15 @@ impl PGNamespace {
     }
 }
 
+impl fmt::Debug for PGNamespace {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PGNamespace")
+            .field("schema", &self.schema)
+            .field("catalog_name", &self.catalog_name)
+            .finish()
+    }
+}
+
 impl SystemTable for PGNamespace {
     fn schema(&self) -> SchemaRef {
         self.schema.clone()

src/catalog/src/system_schema/pg_catalog/table_names.rs

@@ -12,7 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub const PG_DATABASE: &str = "pg_databases";
+// https://www.postgresql.org/docs/current/catalog-pg-database.html
+pub const PG_DATABASE: &str = "pg_database";
+// https://www.postgresql.org/docs/current/catalog-pg-namespace.html
 pub const PG_NAMESPACE: &str = "pg_namespace";
+// https://www.postgresql.org/docs/current/catalog-pg-class.html
 pub const PG_CLASS: &str = "pg_class";
+// https://www.postgresql.org/docs/current/catalog-pg-type.html
 pub const PG_TYPE: &str = "pg_type";

src/catalog/src/table_source.rs

@@ -365,7 +365,7 @@ mod tests {
 Projection: person.id AS a, person.name AS b
   Filter: person.id > Int32(500)
     TableScan: person"#,
-            format!("\n{:?}", source.get_logical_plan().unwrap())
+            format!("\n{}", source.get_logical_plan().unwrap())
         );
     }
 }

src/catalog/src/table_source/dummy_catalog.rs

@@ -15,12 +15,12 @@
 //! Dummy catalog for region server.
 
 use std::any::Any;
+use std::fmt;
 use std::sync::Arc;
 
 use async_trait::async_trait;
 use common_catalog::format_full_table_name;
-use datafusion::catalog::schema::SchemaProvider;
-use datafusion::catalog::{CatalogProvider, CatalogProviderList};
+use datafusion::catalog::{CatalogProvider, CatalogProviderList, SchemaProvider};
 use datafusion::datasource::TableProvider;
 use snafu::OptionExt;
 use table::table::adapter::DfTableProviderAdapter;
@@ -41,6 +41,12 @@ impl DummyCatalogList {
     }
 }
 
+impl fmt::Debug for DummyCatalogList {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("DummyCatalogList").finish()
+    }
+}
+
 impl CatalogProviderList for DummyCatalogList {
     fn as_any(&self) -> &dyn Any {
         self
@@ -91,6 +97,14 @@ impl CatalogProvider for DummyCatalogProvider {
     }
 }
 
+impl fmt::Debug for DummyCatalogProvider {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("DummyCatalogProvider")
+            .field("catalog_name", &self.catalog_name)
+            .finish()
+    }
+}
+
 /// A dummy schema provider for [DummyCatalogList].
 #[derive(Clone)]
 struct DummySchemaProvider {
@@ -127,3 +141,12 @@ impl SchemaProvider for DummySchemaProvider {
         true
     }
 }
+
+impl fmt::Debug for DummySchemaProvider {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("DummySchemaProvider")
+            .field("catalog_name", &self.catalog_name)
+            .field("schema_name", &self.schema_name)
+            .finish()
+    }
+}

src/cli/src/database.rs

@@ -17,6 +17,7 @@ use std::time::Duration;
 use base64::engine::general_purpose;
 use base64::Engine;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_error::ext::BoxedError;
 use humantime::format_duration;
 use serde_json::Value;
 use servers::http::header::constants::GREPTIME_DB_HEADER_TIMEOUT;
@@ -24,7 +25,9 @@ use servers::http::result::greptime_result_v1::GreptimedbV1Response;
 use servers::http::GreptimeQueryOutput;
 use snafu::ResultExt;
 
-use crate::error::{HttpQuerySqlSnafu, Result, SerdeJsonSnafu};
+use crate::error::{
+    BuildClientSnafu, HttpQuerySqlSnafu, ParseProxyOptsSnafu, Result, SerdeJsonSnafu,
+};
 
 #[derive(Debug, Clone)]
 pub struct DatabaseClient {
@@ -32,6 +35,23 @@ pub struct DatabaseClient {
     catalog: String,
     auth_header: Option<String>,
     timeout: Duration,
+    proxy: Option<reqwest::Proxy>,
+}
+
+pub fn parse_proxy_opts(
+    proxy: Option<String>,
+    no_proxy: bool,
+) -> std::result::Result<Option<reqwest::Proxy>, BoxedError> {
+    if no_proxy {
+        return Ok(None);
+    }
+    proxy
+        .map(|proxy| {
+            reqwest::Proxy::all(proxy)
+                .context(ParseProxyOptsSnafu)
+                .map_err(BoxedError::new)
+        })
+        .transpose()
 }
 
 impl DatabaseClient {
@@ -40,6 +60,7 @@ impl DatabaseClient {
         catalog: String,
         auth_basic: Option<String>,
         timeout: Duration,
+        proxy: Option<reqwest::Proxy>,
     ) -> Self {
         let auth_header = if let Some(basic) = auth_basic {
             let encoded = general_purpose::STANDARD.encode(basic);
@@ -48,11 +69,18 @@ impl DatabaseClient {
             None
         };
 
+        if let Some(ref proxy) = proxy {
+            common_telemetry::info!("Using proxy: {:?}", proxy);
+        } else {
+            common_telemetry::info!("Using system proxy(if any)");
+        }
+
         Self {
             addr,
             catalog,
             auth_header,
             timeout,
+            proxy,
         }
     }
 
@@ -67,7 +95,13 @@ impl DatabaseClient {
             ("db", format!("{}-{}", self.catalog, schema)),
             ("sql", sql.to_string()),
         ];
-        let mut request = reqwest::Client::new()
+        let client = self
+            .proxy
+            .clone()
+            .map(|proxy| reqwest::Client::builder().proxy(proxy).build())
+            .unwrap_or_else(|| Ok(reqwest::Client::new()))
+            .context(BuildClientSnafu)?;
+        let mut request = client
             .post(&url)
             .form(&params)
             .header("Content-Type", "application/x-www-form-urlencoded");

src/cli/src/error.rs

@@ -86,6 +86,22 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to parse proxy options: {}", error))]
+    ParseProxyOpts {
+        #[snafu(source)]
+        error: reqwest::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Failed to build reqwest client: {}", error))]
+    BuildClient {
+        #[snafu(implicit)]
+        location: Location,
+        #[snafu(source)]
+        error: reqwest::Error,
+    },
+
     #[snafu(display("Invalid REPL command: {reason}"))]
     InvalidReplCommand { reason: String },
 
@@ -278,7 +294,8 @@ impl ErrorExt for Error {
             | Error::InitTimezone { .. }
             | Error::ConnectEtcd { .. }
             | Error::CreateDir { .. }
-            | Error::EmptyResult { .. } => StatusCode::InvalidArguments,
+            | Error::EmptyResult { .. }
+            | Error::ParseProxyOpts { .. } => StatusCode::InvalidArguments,
 
             Error::StartProcedureManager { source, .. }
             | Error::StopProcedureManager { source, .. } => source.status_code(),
@@ -298,7 +315,8 @@ impl ErrorExt for Error {
             Error::SerdeJson { .. }
             | Error::FileIo { .. }
             | Error::SpawnThread { .. }
-            | Error::InitTlsProvider { .. } => StatusCode::Unexpected,
+            | Error::InitTlsProvider { .. }
+            | Error::BuildClient { .. } => StatusCode::Unexpected,
 
             Error::Other { source, .. } => source.status_code(),
 

src/cli/src/export.rs

@@ -28,7 +28,7 @@ use tokio::io::{AsyncWriteExt, BufWriter};
 use tokio::sync::Semaphore;
 use tokio::time::Instant;
 
-use crate::database::DatabaseClient;
+use crate::database::{parse_proxy_opts, DatabaseClient};
 use crate::error::{EmptyResultSnafu, Error, FileIoSnafu, Result, SchemaNotFoundSnafu};
 use crate::{database, Tool};
 
@@ -91,19 +91,30 @@ pub struct ExportCommand {
     /// The default behavior will disable server-side default timeout(i.e. `0s`).
     #[clap(long, value_parser = humantime::parse_duration)]
     timeout: Option<Duration>,
+
+    /// The proxy server address to connect, if set, will override the system proxy.
+    ///
+    /// The default behavior will use the system proxy if neither `proxy` nor `no_proxy` is set.
+    #[clap(long)]
+    proxy: Option<String>,
+
+    /// Disable proxy server, if set, will not use any proxy.
+    #[clap(long)]
+    no_proxy: bool,
 }
 
 impl ExportCommand {
     pub async fn build(&self) -> std::result::Result<Box<dyn Tool>, BoxedError> {
         let (catalog, schema) =
             database::split_database(&self.database).map_err(BoxedError::new)?;
-
+        let proxy = parse_proxy_opts(self.proxy.clone(), self.no_proxy)?;
         let database_client = DatabaseClient::new(
             self.addr.clone(),
             catalog.clone(),
             self.auth_basic.clone(),
             // Treats `None` as `0s` to disable server-side default timeout.
             self.timeout.unwrap_or_default(),
+            proxy,
         );
 
         Ok(Box::new(Export {

src/cli/src/import.rs

@@ -25,7 +25,7 @@ use snafu::{OptionExt, ResultExt};
 use tokio::sync::Semaphore;
 use tokio::time::Instant;
 
-use crate::database::DatabaseClient;
+use crate::database::{parse_proxy_opts, DatabaseClient};
 use crate::error::{Error, FileIoSnafu, Result, SchemaNotFoundSnafu};
 use crate::{database, Tool};
 
@@ -76,18 +76,30 @@ pub struct ImportCommand {
     /// The default behavior will disable server-side default timeout(i.e. `0s`).
     #[clap(long, value_parser = humantime::parse_duration)]
     timeout: Option<Duration>,
+
+    /// The proxy server address to connect, if set, will override the system proxy.
+    ///
+    /// The default behavior will use the system proxy if neither `proxy` nor `no_proxy` is set.
+    #[clap(long)]
+    proxy: Option<String>,
+
+    /// Disable proxy server, if set, will not use any proxy.
+    #[clap(long, default_value = "false")]
+    no_proxy: bool,
 }
 
 impl ImportCommand {
     pub async fn build(&self) -> std::result::Result<Box<dyn Tool>, BoxedError> {
         let (catalog, schema) =
             database::split_database(&self.database).map_err(BoxedError::new)?;
+        let proxy = parse_proxy_opts(self.proxy.clone(), self.no_proxy)?;
         let database_client = DatabaseClient::new(
             self.addr.clone(),
             catalog.clone(),
             self.auth_basic.clone(),
             // Treats `None` as `0s` to disable server-side default timeout.
             self.timeout.unwrap_or_default(),
+            proxy,
         );
 
         Ok(Box::new(Import {

src/cmd/Cargo.toml

@@ -57,6 +57,7 @@ humantime.workspace = true
 lazy_static.workspace = true
 meta-client.workspace = true
 meta-srv.workspace = true
+metric-engine.workspace = true
 mito2.workspace = true
 moka.workspace = true
 nu-ansi-term = "0.46"

src/cmd/src/cli.rs

@@ -51,8 +51,7 @@ impl App for Instance {
     }
 
     async fn start(&mut self) -> Result<()> {
-        self.start().await.unwrap();
-        Ok(())
+        self.start().await
     }
 
     fn wait_signal(&self) -> bool {

src/cmd/src/error.rs

@@ -345,6 +345,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Failed to build wal options allocator"))]
+    BuildWalOptionsAllocator {
+        #[snafu(implicit)]
+        location: Location,
+        source: common_meta::error::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -378,7 +385,8 @@ impl ErrorExt for Error {
 
             Error::StartProcedureManager { source, .. }
             | Error::StopProcedureManager { source, .. } => source.status_code(),
-            Error::StartWalOptionsAllocator { source, .. } => source.status_code(),
+            Error::BuildWalOptionsAllocator { source, .. }
+            | Error::StartWalOptionsAllocator { source, .. } => source.status_code(),
             Error::ReplCreation { .. } | Error::Readline { .. } | Error::HttpQuerySql { .. } => {
                 StatusCode::Internal
             }

src/cmd/src/flownode.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::sync::Arc;
+use std::time::Duration;
 
 use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
 use catalog::information_extension::DistributedInformationExtension;
@@ -142,6 +143,11 @@ struct StartCommand {
     /// The prefix of environment variables, default is `GREPTIMEDB_FLOWNODE`;
     #[clap(long, default_value = "GREPTIMEDB_FLOWNODE")]
     env_prefix: String,
+    #[clap(long)]
+    http_addr: Option<String>,
+    /// HTTP request timeout in seconds.
+    #[clap(long)]
+    http_timeout: Option<u64>,
 }
 
 impl StartCommand {
@@ -198,6 +204,14 @@ impl StartCommand {
             opts.mode = Mode::Distributed;
         }
 
+        if let Some(http_addr) = &self.http_addr {
+            opts.http.addr.clone_from(http_addr);
+        }
+
+        if let Some(http_timeout) = self.http_timeout {
+            opts.http.timeout = Duration::from_secs(http_timeout);
+        }
+
         if let (Mode::Distributed, None) = (&opts.mode, &opts.node_id) {
             return MissingConfigSnafu {
                 msg: "Missing node id option",

src/cmd/src/standalone.rs

@@ -43,7 +43,7 @@ use common_meta::node_manager::NodeManagerRef;
 use common_meta::peer::Peer;
 use common_meta::region_keeper::MemoryRegionKeeper;
 use common_meta::sequence::SequenceBuilder;
-use common_meta::wal_options_allocator::{WalOptionsAllocator, WalOptionsAllocatorRef};
+use common_meta::wal_options_allocator::{build_wal_options_allocator, WalOptionsAllocatorRef};
 use common_procedure::{ProcedureInfo, ProcedureManagerRef};
 use common_telemetry::info;
 use common_telemetry::logging::{LoggingOptions, TracingOptions};
@@ -54,7 +54,7 @@ use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, Sto
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use datanode::region_server::RegionServer;
 use file_engine::config::EngineConfig as FileEngineConfig;
-use flow::{FlowWorkerManager, FlownodeBuilder, FrontendInvoker};
+use flow::{FlowConfig, FlowWorkerManager, FlownodeBuilder, FlownodeOptions, FrontendInvoker};
 use frontend::frontend::FrontendOptions;
 use frontend::instance::builder::FrontendBuilder;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
@@ -76,10 +76,10 @@ use tokio::sync::{broadcast, RwLock};
 use tracing_appender::non_blocking::WorkerGuard;
 
 use crate::error::{
-    BuildCacheRegistrySnafu, CreateDirSnafu, IllegalConfigSnafu, InitDdlManagerSnafu,
-    InitMetadataSnafu, InitTimezoneSnafu, LoadLayeredConfigSnafu, OtherSnafu, Result,
-    ShutdownDatanodeSnafu, ShutdownFlownodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu,
-    StartFlownodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
+    BuildCacheRegistrySnafu, BuildWalOptionsAllocatorSnafu, CreateDirSnafu, IllegalConfigSnafu,
+    InitDdlManagerSnafu, InitMetadataSnafu, InitTimezoneSnafu, LoadLayeredConfigSnafu, OtherSnafu,
+    Result, ShutdownDatanodeSnafu, ShutdownFlownodeSnafu, ShutdownFrontendSnafu,
+    StartDatanodeSnafu, StartFlownodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
     StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
 };
 use crate::options::{GlobalOptions, GreptimeOptions};
@@ -145,6 +145,7 @@ pub struct StandaloneOptions {
     pub storage: StorageConfig,
     pub metadata_store: KvBackendConfig,
     pub procedure: ProcedureConfig,
+    pub flow: FlowConfig,
     pub logging: LoggingOptions,
     pub user_provider: Option<String>,
     /// Options for different store engines.
@@ -173,6 +174,7 @@ impl Default for StandaloneOptions {
             storage: StorageConfig::default(),
             metadata_store: KvBackendConfig::default(),
             procedure: ProcedureConfig::default(),
+            flow: FlowConfig::default(),
             logging: LoggingOptions::default(),
             export_metrics: ExportMetricsOption::default(),
             user_provider: None,
@@ -523,8 +525,12 @@ impl StartCommand {
             Self::create_table_metadata_manager(kv_backend.clone()).await?;
 
         let flow_metadata_manager = Arc::new(FlowMetadataManager::new(kv_backend.clone()));
+        let flownode_options = FlownodeOptions {
+            flow: opts.flow.clone(),
+            ..Default::default()
+        };
         let flow_builder = FlownodeBuilder::new(
-            Default::default(),
+            flownode_options,
             plugins.clone(),
             table_metadata_manager.clone(),
             catalog_manager.clone(),
@@ -563,10 +569,11 @@ impl StartCommand {
                 .step(10)
                 .build(),
         );
-        let wal_options_allocator = Arc::new(WalOptionsAllocator::new(
-            opts.wal.clone().into(),
-            kv_backend.clone(),
-        ));
+        let kafka_options = opts.wal.clone().into();
+        let wal_options_allocator = build_wal_options_allocator(&kafka_options, kv_backend.clone())
+            .await
+            .context(BuildWalOptionsAllocatorSnafu)?;
+        let wal_options_allocator = Arc::new(wal_options_allocator);
         let table_meta_allocator = Arc::new(TableMetadataAllocator::new(
             table_id_sequence,
             wal_options_allocator.clone(),

src/cmd/tests/load_config_test.rs

@@ -25,14 +25,16 @@ use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, DEFAULT_OTLP_E
 use common_wal::config::raft_engine::RaftEngineConfig;
 use common_wal::config::DatanodeWalConfig;
 use datanode::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
-use file_engine::config::EngineConfig;
+use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
 use meta_client::MetaClientOptions;
 use meta_srv::metasrv::MetasrvOptions;
 use meta_srv::selector::SelectorType;
+use metric_engine::config::EngineConfig as MetricEngineConfig;
 use mito2::config::MitoConfig;
 use servers::export_metrics::ExportMetricsOption;
 use servers::grpc::GrpcOptions;
+use servers::http::HttpOptions;
 
 #[allow(deprecated)]
 #[test]
@@ -72,7 +74,10 @@ fn test_load_datanode_example_config() {
                     write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                     ..Default::default()
                 }),
-                RegionEngineConfig::File(EngineConfig {}),
+                RegionEngineConfig::File(FileEngineConfig {}),
+                RegionEngineConfig::Metric(MetricEngineConfig {
+                    experimental_sparse_primary_key_encoding: false,
+                }),
             ],
             logging: LoggingOptions {
                 level: Some("info".to_string()),
@@ -140,6 +145,10 @@ fn test_load_frontend_example_config() {
                 ..Default::default()
             },
             grpc: GrpcOptions::default().with_hostname("127.0.0.1:4001"),
+            http: HttpOptions {
+                cors_allowed_origins: vec!["https://example.com".to_string()],
+                ..Default::default()
+            },
             ..Default::default()
         },
         ..Default::default()
@@ -210,7 +219,10 @@ fn test_load_standalone_example_config() {
                     write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                     ..Default::default()
                 }),
-                RegionEngineConfig::File(EngineConfig {}),
+                RegionEngineConfig::File(FileEngineConfig {}),
+                RegionEngineConfig::Metric(MetricEngineConfig {
+                    experimental_sparse_primary_key_encoding: false,
+                }),
             ],
             storage: StorageConfig {
                 data_home: "/tmp/greptimedb/".to_string(),
@@ -227,6 +239,10 @@ fn test_load_standalone_example_config() {
                 remote_write: Some(Default::default()),
                 ..Default::default()
             },
+            http: HttpOptions {
+                cors_allowed_origins: vec!["https://example.com".to_string()],
+                ..Default::default()
+            },
             ..Default::default()
         },
         ..Default::default()

src/common/base/src/range_read.rs

@@ -223,7 +223,6 @@ impl FileReader {
     }
 }
 
-#[cfg(any(test, feature = "testing"))]
 impl SizeAwareRangeReader for FileReader {
     fn with_file_size_hint(&mut self, _file_size_hint: u64) {
         // do nothing

src/common/catalog/src/consts.rs

@@ -109,6 +109,7 @@ pub const INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID: u32 = 35;
 pub const PG_CATALOG_PG_CLASS_TABLE_ID: u32 = 256;
 pub const PG_CATALOG_PG_TYPE_TABLE_ID: u32 = 257;
 pub const PG_CATALOG_PG_NAMESPACE_TABLE_ID: u32 = 258;
+pub const PG_CATALOG_PG_DATABASE_TABLE_ID: u32 = 259;
 
 // ----- End of pg_catalog tables -----
 

src/common/config/src/config.rs

@@ -73,14 +73,21 @@ pub trait Configurable: Serialize + DeserializeOwned + Default + Sized {
             layered_config = layered_config.add_source(File::new(config_file, FileFormat::Toml));
         }
 
-        let opts = layered_config
+        let mut opts: Self = layered_config
             .build()
             .and_then(|x| x.try_deserialize())
             .context(LoadLayeredConfigSnafu)?;
 
+        opts.validate_sanitize()?;
+
         Ok(opts)
     }
 
+    /// Validate(and possibly sanitize) the configuration.
+    fn validate_sanitize(&mut self) -> Result<()> {
+        Ok(())
+    }
+
     /// List of toml keys that should be parsed as a list.
     fn env_list_keys() -> Option<&'static [&'static str]> {
         None

src/common/datasource/Cargo.toml

@@ -31,7 +31,7 @@ derive_builder.workspace = true
 futures.workspace = true
 lazy_static.workspace = true
 object-store.workspace = true
-orc-rust = { git = "https://github.com/datafusion-contrib/datafusion-orc.git", rev = "502217315726314c4008808fe169764529640599", default-features = false, features = [
+orc-rust = { version = "0.5", default-features = false, features = [
     "async",
 ] }
 parquet.workspace = true

src/common/datasource/src/error.rs

@@ -180,7 +180,7 @@ pub enum Error {
 
     #[snafu(display("Failed to parse format {} with value: {}", key, value))]
     ParseFormat {
-        key: &'static str,
+        key: String,
         value: String,
         #[snafu(implicit)]
         location: Location,

src/common/datasource/src/file_format.rs

@@ -126,8 +126,7 @@ impl ArrowDecoder for arrow::csv::reader::Decoder {
     }
 }
 
-#[allow(deprecated)]
-impl ArrowDecoder for arrow::json::RawDecoder {
+impl ArrowDecoder for arrow::json::reader::Decoder {
     fn decode(&mut self, buf: &[u8]) -> result::Result<usize, ArrowError> {
         self.decode(buf)
     }

src/common/datasource/src/file_format/csv.rs

@@ -17,8 +17,7 @@ use std::str::FromStr;
 use std::sync::Arc;
 
 use arrow::csv;
-#[allow(deprecated)]
-use arrow::csv::reader::infer_reader_schema as infer_csv_schema;
+use arrow::csv::reader::Format;
 use arrow::record_batch::RecordBatch;
 use arrow_schema::{Schema, SchemaRef};
 use async_trait::async_trait;
@@ -161,7 +160,6 @@ impl FileOpener for CsvOpener {
     }
 }
 
-#[allow(deprecated)]
 #[async_trait]
 impl FileFormat for CsvFormat {
     async fn infer_schema(&self, store: &ObjectStore, path: &str) -> Result<Schema> {
@@ -188,9 +186,12 @@ impl FileFormat for CsvFormat {
         common_runtime::spawn_blocking_global(move || {
             let reader = SyncIoBridge::new(decoded);
 
-            let (schema, _records_read) =
-                infer_csv_schema(reader, delimiter, schema_infer_max_record, has_header)
-                    .context(error::InferSchemaSnafu)?;
+            let format = Format::default()
+                .with_delimiter(delimiter)
+                .with_header(has_header);
+            let (schema, _records_read) = format
+                .infer_schema(reader, schema_infer_max_record)
+                .context(error::InferSchemaSnafu)?;
             Ok(schema)
         })
         .await
@@ -253,7 +254,7 @@ mod tests {
                 "c7: Int64: NULL",
                 "c8: Int64: NULL",
                 "c9: Int64: NULL",
-                "c10: Int64: NULL",
+                "c10: Utf8: NULL",
                 "c11: Float64: NULL",
                 "c12: Float64: NULL",
                 "c13: Utf8: NULL"

src/common/datasource/src/file_format/json.rs

@@ -20,8 +20,7 @@ use std::sync::Arc;
 use arrow::datatypes::SchemaRef;
 use arrow::json::reader::{infer_json_schema_from_iterator, ValueIter};
 use arrow::json::writer::LineDelimited;
-#[allow(deprecated)]
-use arrow::json::{self, RawReaderBuilder};
+use arrow::json::{self, ReaderBuilder};
 use arrow::record_batch::RecordBatch;
 use arrow_schema::Schema;
 use async_trait::async_trait;
@@ -140,7 +139,6 @@ impl JsonOpener {
     }
 }
 
-#[allow(deprecated)]
 impl FileOpener for JsonOpener {
     fn open(&self, meta: FileMeta) -> DataFusionResult<FileOpenFuture> {
         open_with_decoder(
@@ -148,7 +146,7 @@ impl FileOpener for JsonOpener {
             meta.location().to_string(),
             self.compression_type,
             || {
-                RawReaderBuilder::new(self.projected_schema.clone())
+                ReaderBuilder::new(self.projected_schema.clone())
                     .with_batch_size(self.batch_size)
                     .build_decoder()
                     .map_err(DataFusionError::from)

src/common/datasource/src/file_format/tests.rs

@@ -42,7 +42,7 @@ struct Test<'a, T: FileOpener> {
     expected: Vec<&'a str>,
 }
 
-impl<'a, T: FileOpener> Test<'a, T> {
+impl<T: FileOpener> Test<'_, T> {
     pub async fn run(self) {
         let result = FileStream::new(
             &self.config,

src/common/decimal/src/decimal128.rs

@@ -164,7 +164,7 @@ impl FromStr for Decimal128 {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let len = s.as_bytes().len();
+        let len = s.len();
         if len <= BYTES_TO_OVERFLOW_RUST_DECIMAL {
             let rd = RustDecimal::from_str_exact(s).context(ParseRustDecimalStrSnafu { raw: s })?;
             Ok(Self::from(rd))

src/common/function/src/function_registry.rs

@@ -25,7 +25,6 @@ use crate::scalars::expression::ExpressionFunction;
 use crate::scalars::json::JsonFunction;
 use crate::scalars::matches::MatchesFunction;
 use crate::scalars::math::MathFunction;
-use crate::scalars::numpy::NumpyFunction;
 use crate::scalars::timestamp::TimestampFunction;
 use crate::scalars::vector::VectorFunction;
 use crate::system::SystemFunction;
@@ -103,7 +102,6 @@ pub static FUNCTION_REGISTRY: Lazy<Arc<FunctionRegistry>> = Lazy::new(|| {
 
     // Utility functions
     MathFunction::register(&function_registry);
-    NumpyFunction::register(&function_registry);
     TimestampFunction::register(&function_registry);
     DateFunction::register(&function_registry);
     ExpressionFunction::register(&function_registry);

src/common/function/src/scalars.rs

@@ -20,7 +20,6 @@ pub mod geo;
 pub mod json;
 pub mod matches;
 pub mod math;
-pub mod numpy;
 pub mod vector;
 
 #[cfg(test)]

src/common/function/src/scalars/date/date_add.rs

@@ -91,6 +91,7 @@ mod tests {
     use std::sync::Arc;
 
     use common_query::prelude::{TypeSignature, Volatility};
+    use datatypes::arrow::datatypes::IntervalDayTime;
     use datatypes::prelude::ConcreteDataType;
     use datatypes::value::Value;
     use datatypes::vectors::{
@@ -134,7 +135,12 @@ mod tests {
 
         let times = vec![Some(123), None, Some(42), None];
         // Intervals in milliseconds
-        let intervals = vec![1000, 2000, 3000, 1000];
+        let intervals = vec![
+            IntervalDayTime::new(0, 1000),
+            IntervalDayTime::new(0, 2000),
+            IntervalDayTime::new(0, 3000),
+            IntervalDayTime::new(0, 1000),
+        ];
         let results = [Some(124), None, Some(45), None];
 
         let time_vector = TimestampSecondVector::from(times.clone());

src/common/function/src/scalars/date/date_sub.rs

@@ -91,6 +91,7 @@ mod tests {
     use std::sync::Arc;
 
     use common_query::prelude::{TypeSignature, Volatility};
+    use datatypes::arrow::datatypes::IntervalDayTime;
     use datatypes::prelude::ConcreteDataType;
     use datatypes::value::Value;
     use datatypes::vectors::{
@@ -139,7 +140,12 @@ mod tests {
 
         let times = vec![Some(123), None, Some(42), None];
         // Intervals in milliseconds
-        let intervals = vec![1000, 2000, 3000, 1000];
+        let intervals = vec![
+            IntervalDayTime::new(0, 1000),
+            IntervalDayTime::new(0, 2000),
+            IntervalDayTime::new(0, 3000),
+            IntervalDayTime::new(0, 1000),
+        ];
         let results = [Some(122), None, Some(39), None];
 
         let time_vector = TimestampSecondVector::from(times.clone());

src/common/function/src/scalars/matches.rs

@@ -21,10 +21,9 @@ use common_query::error::{
 };
 use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeIterator, TreeNodeRecursion};
 use datafusion::common::{DFSchema, Result as DfResult};
-use datafusion::execution::context::SessionState;
+use datafusion::execution::SessionStateBuilder;
 use datafusion::logical_expr::{self, Expr, Volatility};
 use datafusion::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner};
-use datafusion::prelude::SessionConfig;
 use datatypes::arrow::array::RecordBatch;
 use datatypes::arrow::datatypes::{DataType, Field};
 use datatypes::prelude::VectorRef;
@@ -104,8 +103,7 @@ impl MatchesFunction {
         let like_expr = ast.into_like_expr(col_name);
 
         let input_schema = Self::input_schema();
-        let session_state =
-            SessionState::new_with_config_rt(SessionConfig::default(), Arc::default());
+        let session_state = SessionStateBuilder::new().with_default_features().build();
         let planner = DefaultPhysicalPlanner::default();
         let physical_expr = planner
             .create_physical_expr(&like_expr, &input_schema, &session_state)
@@ -131,7 +129,7 @@ impl MatchesFunction {
     }
 
     fn input_schema() -> DFSchema {
-        DFSchema::from_unqualifed_fields(
+        DFSchema::from_unqualified_fields(
             [Arc::new(Field::new("data", DataType::Utf8, true))].into(),
             HashMap::new(),
         )
@@ -725,7 +723,8 @@ struct Tokenizer {
 impl Tokenizer {
     pub fn tokenize(mut self, pattern: &str) -> Result<Vec<Token>> {
         let mut tokens = vec![];
-        while self.cursor < pattern.len() {
+        let char_len = pattern.chars().count();
+        while self.cursor < char_len {
             // TODO: collect pattern into Vec<char> if this tokenizer is bottleneck in the future
             let c = pattern.chars().nth(self.cursor).unwrap();
             match c {
@@ -794,7 +793,8 @@ impl Tokenizer {
         let mut phase = String::new();
         let mut is_quote_present = false;
 
-        while self.cursor < pattern.len() {
+        let char_len = pattern.chars().count();
+        while self.cursor < char_len {
             let mut c = pattern.chars().nth(self.cursor).unwrap();
 
             match c {
@@ -899,6 +899,26 @@ mod test {
                     Phase("c".to_string()),
                 ],
             ),
+            (
+                r#"中文 测试"#,
+                vec![Phase("中文".to_string()), Phase("测试".to_string())],
+            ),
+            (
+                r#"中文 AND 测试"#,
+                vec![Phase("中文".to_string()), And, Phase("测试".to_string())],
+            ),
+            (
+                r#"中文 +测试"#,
+                vec![Phase("中文".to_string()), Must, Phase("测试".to_string())],
+            ),
+            (
+                r#"中文 -测试"#,
+                vec![
+                    Phase("中文".to_string()),
+                    Negative,
+                    Phase("测试".to_string()),
+                ],
+            ),
         ];
 
         for (query, expected) in cases {
@@ -1030,6 +1050,61 @@ mod test {
                     ],
                 },
             ),
+            (
+                r#"中文 测试"#,
+                PatternAst::Binary {
+                    op: BinaryOp::Or,
+                    children: vec![
+                        PatternAst::Literal {
+                            op: UnaryOp::Optional,
+                            pattern: "中文".to_string(),
+                        },
+                        PatternAst::Literal {
+                            op: UnaryOp::Optional,
+                            pattern: "测试".to_string(),
+                        },
+                    ],
+                },
+            ),
+            (
+                r#"中文 AND 测试"#,
+                PatternAst::Binary {
+                    op: BinaryOp::And,
+                    children: vec![
+                        PatternAst::Literal {
+                            op: UnaryOp::Optional,
+                            pattern: "中文".to_string(),
+                        },
+                        PatternAst::Literal {
+                            op: UnaryOp::Optional,
+                            pattern: "测试".to_string(),
+                        },
+                    ],
+                },
+            ),
+            (
+                r#"中文 +测试"#,
+                PatternAst::Literal {
+                    op: UnaryOp::Must,
+                    pattern: "测试".to_string(),
+                },
+            ),
+            (
+                r#"中文 -测试"#,
+                PatternAst::Binary {
+                    op: BinaryOp::And,
+                    children: vec![
+                        PatternAst::Literal {
+                            op: UnaryOp::Negative,
+                            pattern: "测试".to_string(),
+                        },
+                        PatternAst::Literal {
+                            op: UnaryOp::Optional,
+                            pattern: "中文".to_string(),
+                        },
+                    ],
+                },
+            ),
         ];
 
         for (query, expected) in cases {

src/common/function/src/scalars/numpy/clip.rs

@@ -1,298 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::fmt;
-use std::sync::Arc;
-
-use common_query::error::Result;
-use common_query::prelude::{Signature, Volatility};
-use datatypes::arrow::compute;
-use datatypes::arrow::datatypes::ArrowPrimitiveType;
-use datatypes::data_type::ConcreteDataType;
-use datatypes::prelude::*;
-use datatypes::vectors::PrimitiveVector;
-use paste::paste;
-
-use crate::function::{Function, FunctionContext};
-use crate::scalars::expression::{scalar_binary_op, EvalContext};
-
-/// numpy.clip function, <https://numpy.org/doc/stable/reference/generated/numpy.clip.html>
-#[derive(Clone, Debug, Default)]
-pub struct ClipFunction;
-
-macro_rules! define_eval {
-    ($O: ident) => {
-        paste! {
-            fn [<eval_ $O>](columns: &[VectorRef]) -> Result<VectorRef> {
-                fn cast_vector(input: &VectorRef) -> VectorRef {
-                    Arc::new(PrimitiveVector::<<$O as WrapperType>::LogicalType>::try_from_arrow_array(
-                        compute::cast(&input.to_arrow_array(), &<<<$O as WrapperType>::LogicalType as LogicalPrimitiveType>::ArrowPrimitive as ArrowPrimitiveType>::DATA_TYPE).unwrap()
-                    ).unwrap()) as _
-                }
-                let operator_1 = cast_vector(&columns[0]);
-                let operator_2 = cast_vector(&columns[1]);
-                let operator_3 = cast_vector(&columns[2]);
-
-                // clip(a, min, max) is equals to min(max(a, min), max)
-                let col: VectorRef = Arc::new(scalar_binary_op::<$O, $O, $O, _>(
-                    &operator_1,
-                    &operator_2,
-                    scalar_max,
-                    &mut EvalContext::default(),
-                )?);
-                let col = scalar_binary_op::<$O, $O, $O, _>(
-                    &col,
-                    &operator_3,
-                    scalar_min,
-                    &mut EvalContext::default(),
-                )?;
-                Ok(Arc::new(col))
-            }
-        }
-    };
-}
-
-define_eval!(i64);
-define_eval!(u64);
-define_eval!(f64);
-
-impl Function for ClipFunction {
-    fn name(&self) -> &str {
-        "clip"
-    }
-
-    fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        if input_types.iter().all(ConcreteDataType::is_signed) {
-            Ok(ConcreteDataType::int64_datatype())
-        } else if input_types.iter().all(ConcreteDataType::is_unsigned) {
-            Ok(ConcreteDataType::uint64_datatype())
-        } else {
-            Ok(ConcreteDataType::float64_datatype())
-        }
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::uniform(3, ConcreteDataType::numerics(), Volatility::Immutable)
-    }
-
-    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        if columns.iter().all(|v| v.data_type().is_signed()) {
-            eval_i64(columns)
-        } else if columns.iter().all(|v| v.data_type().is_unsigned()) {
-            eval_u64(columns)
-        } else {
-            eval_f64(columns)
-        }
-    }
-}
-
-#[inline]
-pub fn min<T: PartialOrd>(input: T, min: T) -> T {
-    if input < min {
-        input
-    } else {
-        min
-    }
-}
-
-#[inline]
-pub fn max<T: PartialOrd>(input: T, max: T) -> T {
-    if input > max {
-        input
-    } else {
-        max
-    }
-}
-
-#[inline]
-fn scalar_min<O>(left: Option<O>, right: Option<O>, _ctx: &mut EvalContext) -> Option<O>
-where
-    O: Scalar + Copy + PartialOrd,
-{
-    match (left, right) {
-        (Some(left), Some(right)) => Some(min(left, right)),
-        _ => None,
-    }
-}
-
-#[inline]
-fn scalar_max<O>(left: Option<O>, right: Option<O>, _ctx: &mut EvalContext) -> Option<O>
-where
-    O: Scalar + Copy + PartialOrd,
-{
-    match (left, right) {
-        (Some(left), Some(right)) => Some(max(left, right)),
-        _ => None,
-    }
-}
-
-impl fmt::Display for ClipFunction {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "CLIP")
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use common_query::prelude::TypeSignature;
-    use datatypes::value::Value;
-    use datatypes::vectors::{
-        ConstantVector, Float32Vector, Int16Vector, Int32Vector, Int8Vector, UInt16Vector,
-        UInt32Vector, UInt8Vector,
-    };
-
-    use super::*;
-
-    #[test]
-    fn test_clip_signature() {
-        let clip = ClipFunction;
-
-        assert_eq!("clip", clip.name());
-        assert_eq!(
-            ConcreteDataType::int64_datatype(),
-            clip.return_type(&[]).unwrap()
-        );
-
-        assert_eq!(
-            ConcreteDataType::int64_datatype(),
-            clip.return_type(&[
-                ConcreteDataType::int16_datatype(),
-                ConcreteDataType::int64_datatype(),
-                ConcreteDataType::int8_datatype()
-            ])
-            .unwrap()
-        );
-        assert_eq!(
-            ConcreteDataType::uint64_datatype(),
-            clip.return_type(&[
-                ConcreteDataType::uint16_datatype(),
-                ConcreteDataType::uint64_datatype(),
-                ConcreteDataType::uint8_datatype()
-            ])
-            .unwrap()
-        );
-        assert_eq!(
-            ConcreteDataType::float64_datatype(),
-            clip.return_type(&[
-                ConcreteDataType::uint16_datatype(),
-                ConcreteDataType::int64_datatype(),
-                ConcreteDataType::uint8_datatype()
-            ])
-            .unwrap()
-        );
-
-        assert!(matches!(clip.signature(),
-                         Signature {
-                             type_signature: TypeSignature::Uniform(3, valid_types),
-                             volatility: Volatility::Immutable
-                         } if  valid_types == ConcreteDataType::numerics()
-        ));
-    }
-
-    #[test]
-    fn test_clip_fn_signed() {
-        // eval with signed integers
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from_values(0..10)),
-            Arc::new(ConstantVector::new(
-                Arc::new(Int8Vector::from_vec(vec![3])),
-                10,
-            )),
-            Arc::new(ConstantVector::new(
-                Arc::new(Int16Vector::from_vec(vec![6])),
-                10,
-            )),
-        ];
-
-        let vector = ClipFunction
-            .eval(FunctionContext::default(), &args)
-            .unwrap();
-        assert_eq!(10, vector.len());
-
-        // clip([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 3, 6) = [3, 3, 3, 3, 4, 5, 6, 6, 6, 6]
-        for i in 0..10 {
-            if i <= 3 {
-                assert!(matches!(vector.get(i), Value::Int64(v) if v == 3));
-            } else if i <= 6 {
-                assert!(matches!(vector.get(i), Value::Int64(v) if v == (i as i64)));
-            } else {
-                assert!(matches!(vector.get(i), Value::Int64(v) if v == 6));
-            }
-        }
-    }
-
-    #[test]
-    fn test_clip_fn_unsigned() {
-        // eval with unsigned integers
-        let args: Vec<VectorRef> = vec![
-            Arc::new(UInt8Vector::from_values(0..10)),
-            Arc::new(ConstantVector::new(
-                Arc::new(UInt32Vector::from_vec(vec![3])),
-                10,
-            )),
-            Arc::new(ConstantVector::new(
-                Arc::new(UInt16Vector::from_vec(vec![6])),
-                10,
-            )),
-        ];
-
-        let vector = ClipFunction
-            .eval(FunctionContext::default(), &args)
-            .unwrap();
-        assert_eq!(10, vector.len());
-
-        // clip([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 3, 6) = [3, 3, 3, 3, 4, 5, 6, 6, 6, 6]
-        for i in 0..10 {
-            if i <= 3 {
-                assert!(matches!(vector.get(i), Value::UInt64(v) if v == 3));
-            } else if i <= 6 {
-                assert!(matches!(vector.get(i), Value::UInt64(v) if v == (i as u64)));
-            } else {
-                assert!(matches!(vector.get(i), Value::UInt64(v) if v == 6));
-            }
-        }
-    }
-
-    #[test]
-    fn test_clip_fn_float() {
-        // eval with floats
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Int8Vector::from_values(0..10)),
-            Arc::new(ConstantVector::new(
-                Arc::new(UInt32Vector::from_vec(vec![3])),
-                10,
-            )),
-            Arc::new(ConstantVector::new(
-                Arc::new(Float32Vector::from_vec(vec![6f32])),
-                10,
-            )),
-        ];
-
-        let vector = ClipFunction
-            .eval(FunctionContext::default(), &args)
-            .unwrap();
-        assert_eq!(10, vector.len());
-
-        // clip([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 3, 6) = [3, 3, 3, 3, 4, 5, 6, 6, 6, 6]
-        for i in 0..10 {
-            if i <= 3 {
-                assert!(matches!(vector.get(i), Value::Float64(v) if v == 3.0));
-            } else if i <= 6 {
-                assert!(matches!(vector.get(i), Value::Float64(v) if v == (i as f64)));
-            } else {
-                assert!(matches!(vector.get(i), Value::Float64(v) if v == 6.0));
-            }
-        }
-    }
-}

src/common/function/src/scalars/numpy/interp.rs

@@ -1,360 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use common_query::error::{self, Result};
-use datatypes::arrow::compute::cast;
-use datatypes::arrow::datatypes::DataType as ArrowDataType;
-use datatypes::data_type::DataType;
-use datatypes::prelude::ScalarVector;
-use datatypes::value::Value;
-use datatypes::vectors::{Float64Vector, Vector, VectorRef};
-use datatypes::with_match_primitive_type_id;
-use snafu::{ensure, ResultExt};
-
-/// search the biggest number that smaller than x in xp
-fn linear_search_ascending_vector(x: Value, xp: &Float64Vector) -> usize {
-    for i in 0..xp.len() {
-        if x < xp.get(i) {
-            return i - 1;
-        }
-    }
-    xp.len() - 1
-}
-
-/// search the biggest number that smaller than x in xp
-fn binary_search_ascending_vector(key: Value, xp: &Float64Vector) -> usize {
-    let mut left = 0;
-    let mut right = xp.len();
-    /* If len <= 4 use linear search. */
-    if xp.len() <= 4 {
-        return linear_search_ascending_vector(key, xp);
-    }
-    /* find index by bisection */
-    while left < right {
-        let mid = left + ((right - left) >> 1);
-        if key >= xp.get(mid) {
-            left = mid + 1;
-        } else {
-            right = mid;
-        }
-    }
-    left - 1
-}
-
-fn concrete_type_to_primitive_vector(arg: &VectorRef) -> Result<Float64Vector> {
-    with_match_primitive_type_id!(arg.data_type().logical_type_id(), |$S| {
-        let tmp = arg.to_arrow_array();
-        let array = cast(&tmp, &ArrowDataType::Float64).context(error::TypeCastSnafu {
-            typ: ArrowDataType::Float64,
-        })?;
-        // Safety: array has been cast to Float64Array.
-        Ok(Float64Vector::try_from_arrow_array(array).unwrap())
-    },{
-        unreachable!()
-    })
-}
-
-/// One-dimensional linear interpolation for monotonically increasing sample points. Refers to
-/// <https://github.com/numpy/numpy/blob/b101756ac02e390d605b2febcded30a1da50cc2c/numpy/core/src/multiarray/compiled_base.c#L491>
-#[allow(unused)]
-pub fn interp(args: &[VectorRef]) -> Result<VectorRef> {
-    let mut left = None;
-    let mut right = None;
-
-    ensure!(
-        args.len() >= 3,
-        error::InvalidFuncArgsSnafu {
-            err_msg: format!(
-                "The length of the args is not enough, expect at least: {}, have: {}",
-                3,
-                args.len()
-            ),
-        }
-    );
-
-    let x = concrete_type_to_primitive_vector(&args[0])?;
-    let xp = concrete_type_to_primitive_vector(&args[1])?;
-    let fp = concrete_type_to_primitive_vector(&args[2])?;
-
-    // make sure the args.len() is 3 or 5
-    if args.len() > 3 {
-        ensure!(
-            args.len() == 5,
-            error::InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not enough, expect at least: {}, have: {}",
-                    5,
-                    args.len()
-                ),
-            }
-        );
-
-        left = concrete_type_to_primitive_vector(&args[3])
-            .unwrap()
-            .get_data(0);
-        right = concrete_type_to_primitive_vector(&args[4])
-            .unwrap()
-            .get_data(0);
-    }
-
-    ensure!(
-        x.len() != 0,
-        error::InvalidFuncArgsSnafu {
-            err_msg: "The sample x is empty",
-        }
-    );
-    ensure!(
-        xp.len() != 0,
-        error::InvalidFuncArgsSnafu {
-            err_msg: "The sample xp is empty",
-        }
-    );
-    ensure!(
-        fp.len() != 0,
-        error::InvalidFuncArgsSnafu {
-            err_msg: "The sample fp is empty",
-        }
-    );
-    ensure!(
-        xp.len() == fp.len(),
-        error::InvalidFuncArgsSnafu {
-            err_msg: format!(
-                "The length of the len1: {} don't match the length of the len2: {}",
-                xp.len(),
-                fp.len()
-            ),
-        }
-    );
-
-    /* Get left and right fill values. */
-    let left = match left {
-        Some(left) => Some(left),
-        _ => fp.get_data(0),
-    };
-
-    let right = match right {
-        Some(right) => Some(right),
-        _ => fp.get_data(fp.len() - 1),
-    };
-
-    let res;
-    if xp.len() == 1 {
-        let data = x
-            .iter_data()
-            .map(|x| {
-                if Value::from(x) < xp.get(0) {
-                    left
-                } else if Value::from(x) > xp.get(xp.len() - 1) {
-                    right
-                } else {
-                    fp.get_data(0)
-                }
-            })
-            .collect::<Vec<_>>();
-        res = Float64Vector::from(data);
-    } else {
-        let mut j = 0;
-        /* only pre-calculate slopes if there are relatively few of them. */
-        let mut slopes: Option<Vec<_>> = None;
-        if x.len() >= xp.len() {
-            let mut slopes_tmp = Vec::with_capacity(xp.len() - 1);
-            for i in 0..xp.len() - 1 {
-                let slope = match (
-                    fp.get_data(i + 1),
-                    fp.get_data(i),
-                    xp.get_data(i + 1),
-                    xp.get_data(i),
-                ) {
-                    (Some(fp1), Some(fp2), Some(xp1), Some(xp2)) => {
-                        if xp1 == xp2 {
-                            None
-                        } else {
-                            Some((fp1 - fp2) / (xp1 - xp2))
-                        }
-                    }
-                    _ => None,
-                };
-                slopes_tmp.push(slope);
-            }
-            slopes = Some(slopes_tmp);
-        }
-        let data = x
-            .iter_data()
-            .map(|x| match x {
-                Some(xi) => {
-                    if Value::from(xi) > xp.get(xp.len() - 1) {
-                        right
-                    } else if Value::from(xi) < xp.get(0) {
-                        left
-                    } else {
-                        j = binary_search_ascending_vector(Value::from(xi), &xp);
-                        if j == xp.len() - 1 || xp.get(j) == Value::from(xi) {
-                            fp.get_data(j)
-                        } else {
-                            let slope = match &slopes {
-                                Some(slopes) => slopes[j],
-                                _ => match (
-                                    fp.get_data(j + 1),
-                                    fp.get_data(j),
-                                    xp.get_data(j + 1),
-                                    xp.get_data(j),
-                                ) {
-                                    (Some(fp1), Some(fp2), Some(xp1), Some(xp2)) => {
-                                        if xp1 == xp2 {
-                                            None
-                                        } else {
-                                            Some((fp1 - fp2) / (xp1 - xp2))
-                                        }
-                                    }
-                                    _ => None,
-                                },
-                            };
-
-                            /* If we get nan in one direction, try the other */
-                            let ans = match (slope, xp.get_data(j), fp.get_data(j)) {
-                                (Some(slope), Some(xp), Some(fp)) => Some(slope * (xi - xp) + fp),
-                                _ => None,
-                            };
-
-                            let ans = match ans {
-                                Some(ans) => Some(ans),
-                                _ => match (slope, xp.get_data(j + 1), fp.get_data(j + 1)) {
-                                    (Some(slope), Some(xp), Some(fp)) => {
-                                        Some(slope * (xi - xp) + fp)
-                                    }
-                                    _ => None,
-                                },
-                            };
-                            let ans = match ans {
-                                Some(ans) => Some(ans),
-                                _ => {
-                                    if fp.get_data(j) == fp.get_data(j + 1) {
-                                        fp.get_data(j)
-                                    } else {
-                                        None
-                                    }
-                                }
-                            };
-                            ans
-                        }
-                    }
-                }
-                _ => None,
-            })
-            .collect::<Vec<_>>();
-        res = Float64Vector::from(data);
-    }
-    Ok(Arc::new(res) as _)
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use datatypes::vectors::{Int32Vector, Int64Vector};
-
-    use super::*;
-    #[test]
-    fn test_basic_interp() {
-        // x xp fp
-        let x = 2.5;
-        let xp = vec![1i32, 2i32, 3i32];
-        let fp = vec![3i64, 2i64, 0i64];
-
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Float64Vector::from_vec(vec![x])),
-            Arc::new(Int32Vector::from_vec(xp.clone())),
-            Arc::new(Int64Vector::from_vec(fp.clone())),
-        ];
-        let vector = interp(&args).unwrap();
-        assert_eq!(vector.len(), 1);
-
-        assert!(matches!(vector.get(0), Value::Float64(v) if v==1.0));
-
-        let x = vec![0.0, 1.0, 1.5, 3.2];
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Float64Vector::from_vec(x)),
-            Arc::new(Int32Vector::from_vec(xp)),
-            Arc::new(Int64Vector::from_vec(fp)),
-        ];
-        let vector = interp(&args).unwrap();
-        assert_eq!(4, vector.len());
-        let res = [3.0, 3.0, 2.5, 0.0];
-        for (i, item) in res.iter().enumerate().take(vector.len()) {
-            assert!(matches!(vector.get(i),Value::Float64(v) if v==*item));
-        }
-    }
-
-    #[test]
-    fn test_left_right() {
-        let x = vec![0.0, 1.0, 1.5, 2.0, 3.0, 4.0];
-        let xp = vec![1i32, 2i32, 3i32];
-        let fp = vec![3i64, 2i64, 0i64];
-        let left = vec![-1];
-        let right = vec![2];
-
-        let expect = [-1.0, 3.0, 2.5, 2.0, 0.0, 2.0];
-
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Float64Vector::from_vec(x)),
-            Arc::new(Int32Vector::from_vec(xp)),
-            Arc::new(Int64Vector::from_vec(fp)),
-            Arc::new(Int32Vector::from_vec(left)),
-            Arc::new(Int32Vector::from_vec(right)),
-        ];
-        let vector = interp(&args).unwrap();
-
-        for (i, item) in expect.iter().enumerate().take(vector.len()) {
-            assert!(matches!(vector.get(i),Value::Float64(v) if v==*item));
-        }
-    }
-
-    #[test]
-    fn test_scalar_interpolation_point() {
-        // x=0 output:0
-        let x = vec![0];
-        let xp = vec![0, 1, 5];
-        let fp = vec![0, 1, 5];
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Int64Vector::from_vec(x.clone())),
-            Arc::new(Int64Vector::from_vec(xp.clone())),
-            Arc::new(Int64Vector::from_vec(fp.clone())),
-        ];
-        let vector = interp(&args).unwrap();
-        assert!(matches!(vector.get(0), Value::Float64(v) if v==x[0] as f64));
-
-        // x=0.3 output:0.3
-        let x = vec![0.3];
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Float64Vector::from_vec(x.clone())),
-            Arc::new(Int64Vector::from_vec(xp.clone())),
-            Arc::new(Int64Vector::from_vec(fp.clone())),
-        ];
-        let vector = interp(&args).unwrap();
-        assert!(matches!(vector.get(0), Value::Float64(v) if v == x[0]));
-
-        // x=None output:Null
-        let input = vec![None, Some(0.0), Some(0.3)];
-        let x = Float64Vector::from(input);
-        let args: Vec<VectorRef> = vec![
-            Arc::new(x),
-            Arc::new(Int64Vector::from_vec(xp)),
-            Arc::new(Int64Vector::from_vec(fp)),
-        ];
-        let vector = interp(&args).unwrap();
-        assert!(matches!(vector.get(0), Value::Null));
-    }
-}

src/common/function/src/system/build.rs

@@ -42,7 +42,7 @@ impl Function for BuildFunction {
     }
 
     fn signature(&self) -> Signature {
-        Signature::uniform(0, vec![], Volatility::Immutable)
+        Signature::nullary(Volatility::Immutable)
     }
 
     fn eval(&self, _func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
@@ -56,8 +56,6 @@ impl Function for BuildFunction {
 mod tests {
     use std::sync::Arc;
 
-    use common_query::prelude::TypeSignature;
-
     use super::*;
     #[test]
     fn test_build_function() {
@@ -67,12 +65,7 @@ mod tests {
             ConcreteDataType::string_datatype(),
             build.return_type(&[]).unwrap()
         );
-        assert!(matches!(build.signature(),
-                         Signature {
-                             type_signature: TypeSignature::Uniform(0, valid_types),
-                             volatility: Volatility::Immutable
-                         } if  valid_types.is_empty()
-        ));
+        assert_eq!(build.signature(), Signature::nullary(Volatility::Immutable));
         let build_info = common_version::build_info().to_string();
         let vector = build.eval(FunctionContext::default(), &[]).unwrap();
         let expect: VectorRef = Arc::new(StringVector::from(vec![build_info]));

src/common/function/src/system/database.rs

@@ -44,7 +44,7 @@ impl Function for DatabaseFunction {
     }
 
     fn signature(&self) -> Signature {
-        Signature::uniform(0, vec![], Volatility::Immutable)
+        Signature::nullary(Volatility::Immutable)
     }
 
     fn eval(&self, func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
@@ -116,7 +116,6 @@ impl fmt::Display for SessionUserFunction {
 mod tests {
     use std::sync::Arc;
 
-    use common_query::prelude::TypeSignature;
     use session::context::QueryContextBuilder;
 
     use super::*;
@@ -128,12 +127,7 @@ mod tests {
             ConcreteDataType::string_datatype(),
             build.return_type(&[]).unwrap()
         );
-        assert!(matches!(build.signature(),
-                         Signature {
-                             type_signature: TypeSignature::Uniform(0, valid_types),
-                             volatility: Volatility::Immutable
-                         } if  valid_types == vec![]
-        ));
+        assert_eq!(build.signature(), Signature::nullary(Volatility::Immutable));
 
         let query_ctx = QueryContextBuilder::default()
             .current_schema("test_db".to_string())

src/common/function/src/system/timezone.rs

@@ -38,7 +38,7 @@ impl Function for TimezoneFunction {
     }
 
     fn signature(&self) -> Signature {
-        Signature::uniform(0, vec![], Volatility::Immutable)
+        Signature::nullary(Volatility::Immutable)
     }
 
     fn eval(&self, func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
@@ -58,7 +58,6 @@ impl fmt::Display for TimezoneFunction {
 mod tests {
     use std::sync::Arc;
 
-    use common_query::prelude::TypeSignature;
     use session::context::QueryContextBuilder;
 
     use super::*;
@@ -70,12 +69,7 @@ mod tests {
             ConcreteDataType::string_datatype(),
             build.return_type(&[]).unwrap()
         );
-        assert!(matches!(build.signature(),
-                         Signature {
-                             type_signature: TypeSignature::Uniform(0, valid_types),
-                             volatility: Volatility::Immutable
-                         } if  valid_types == vec![]
-        ));
+        assert_eq!(build.signature(), Signature::nullary(Volatility::Immutable));
 
         let query_ctx = QueryContextBuilder::default().build().into();
 

src/common/function/src/system/version.rs

@@ -42,7 +42,7 @@ impl Function for VersionFunction {
     }
 
     fn signature(&self) -> Signature {
-        Signature::exact(vec![], Volatility::Immutable)
+        Signature::nullary(Volatility::Immutable)
     }
 
     fn eval(&self, func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {

src/common/grpc-expr/src/alter.rs

@@ -25,12 +25,15 @@ use datatypes::schema::{ColumnSchema, FulltextOptions, RawSchema};
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::region_request::{SetRegionOption, UnsetRegionOption};
 use table::metadata::TableId;
-use table::requests::{AddColumnRequest, AlterKind, AlterTableRequest, ModifyColumnTypeRequest};
+use table::requests::{
+    AddColumnRequest, AlterKind, AlterTableRequest, ModifyColumnTypeRequest, SetIndexOptions,
+    UnsetIndexOptions,
+};
 
 use crate::error::{
     InvalidColumnDefSnafu, InvalidSetFulltextOptionRequestSnafu, InvalidSetTableOptionRequestSnafu,
-    InvalidUnsetTableOptionRequestSnafu, MissingFieldSnafu, MissingTimestampColumnSnafu, Result,
-    UnknownLocationTypeSnafu,
+    InvalidUnsetTableOptionRequestSnafu, MissingAlterIndexOptionSnafu, MissingFieldSnafu,
+    MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu,
 };
 
 const LOCATION_TYPE_FIRST: i32 = LocationType::First as i32;
@@ -114,18 +117,43 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterTableExpr) -> Result<
                     .context(InvalidUnsetTableOptionRequestSnafu)?,
             }
         }
-        Kind::SetColumnFulltext(c) => AlterKind::SetColumnFulltext {
-            column_name: c.column_name,
-            options: FulltextOptions {
-                enable: c.enable,
-                analyzer: as_fulltext_option(
-                    Analyzer::try_from(c.analyzer).context(InvalidSetFulltextOptionRequestSnafu)?,
-                ),
-                case_sensitive: c.case_sensitive,
+        Kind::SetIndex(o) => match o.options {
+            Some(opt) => match opt {
+                api::v1::set_index::Options::Fulltext(f) => AlterKind::SetIndex {
+                    options: SetIndexOptions::Fulltext {
+                        column_name: f.column_name.clone(),
+                        options: FulltextOptions {
+                            enable: f.enable,
+                            analyzer: as_fulltext_option(
+                                Analyzer::try_from(f.analyzer)
+                                    .context(InvalidSetFulltextOptionRequestSnafu)?,
+                            ),
+                            case_sensitive: f.case_sensitive,
+                        },
+                    },
+                },
+                api::v1::set_index::Options::Inverted(i) => AlterKind::SetIndex {
+                    options: SetIndexOptions::Inverted {
+                        column_name: i.column_name,
+                    },
+                },
             },
+            None => return MissingAlterIndexOptionSnafu.fail(),
         },
-        Kind::UnsetColumnFulltext(c) => AlterKind::UnsetColumnFulltext {
-            column_name: c.column_name,
+        Kind::UnsetIndex(o) => match o.options {
+            Some(opt) => match opt {
+                api::v1::unset_index::Options::Fulltext(f) => AlterKind::UnsetIndex {
+                    options: UnsetIndexOptions::Fulltext {
+                        column_name: f.column_name,
+                    },
+                },
+                api::v1::unset_index::Options::Inverted(i) => AlterKind::UnsetIndex {
+                    options: UnsetIndexOptions::Inverted {
+                        column_name: i.column_name,
+                    },
+                },
+            },
+            None => return MissingAlterIndexOptionSnafu.fail(),
         },
     };
 

src/common/grpc-expr/src/error.rs

@@ -107,7 +107,7 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
         #[snafu(source)]
-        error: prost::DecodeError,
+        error: prost::UnknownEnumValue,
     },
 
     #[snafu(display(
@@ -137,7 +137,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
         #[snafu(source)]
-        error: prost::DecodeError,
+        error: prost::UnknownEnumValue,
+    },
+
+    #[snafu(display("Missing alter index options"))]
+    MissingAlterIndexOption {
+        #[snafu(implicit)]
+        location: Location,
     },
 }
 
@@ -164,7 +170,8 @@ impl ErrorExt for Error {
             }
             Error::InvalidSetTableOptionRequest { .. }
             | Error::InvalidUnsetTableOptionRequest { .. }
-            | Error::InvalidSetFulltextOptionRequest { .. } => StatusCode::InvalidArguments,
+            | Error::InvalidSetFulltextOptionRequest { .. }
+            | Error::MissingAlterIndexOption { .. } => StatusCode::InvalidArguments,
         }
     }
 

src/common/grpc-expr/src/insert.rs

@@ -255,7 +255,7 @@ mod tests {
             ConcreteDataType::from(
                 ColumnDataTypeWrapper::try_new(
                     decimal_column.data_type,
-                    decimal_column.datatype_extension.clone(),
+                    decimal_column.datatype_extension,
                 )
                 .unwrap()
             )
@@ -351,7 +351,6 @@ mod tests {
                         .as_ref()
                         .unwrap()
                         .datatype_extension
-                        .clone()
                 )
                 .unwrap()
             )

src/common/grpc-expr/src/util.rs

@@ -119,29 +119,30 @@ pub fn build_create_table_expr(
     }
 
     let mut column_defs = Vec::with_capacity(column_exprs.len());
-    let mut primary_keys = Vec::default();
+    let mut primary_keys = Vec::with_capacity(column_exprs.len());
     let mut time_index = None;
 
-    for ColumnExpr {
-        column_name,
-        datatype,
-        semantic_type,
-        datatype_extension,
-        options,
-    } in column_exprs
-    {
+    for expr in column_exprs {
+        let ColumnExpr {
+            column_name,
+            datatype,
+            semantic_type,
+            datatype_extension,
+            options,
+        } = expr;
+
         let mut is_nullable = true;
         match semantic_type {
-            v if v == SemanticType::Tag as i32 => primary_keys.push(column_name.to_string()),
+            v if v == SemanticType::Tag as i32 => primary_keys.push(column_name.to_owned()),
             v if v == SemanticType::Timestamp as i32 => {
                 ensure!(
                     time_index.is_none(),
                     DuplicatedTimestampColumnSnafu {
-                        exists: time_index.unwrap(),
+                        exists: time_index.as_ref().unwrap(),
                         duplicated: column_name,
                     }
                 );
-                time_index = Some(column_name.to_string());
+                time_index = Some(column_name.to_owned());
                 // Timestamp column must not be null.
                 is_nullable = false;
             }
@@ -158,24 +159,23 @@ pub fn build_create_table_expr(
             }
         );
 
-        let column_def = ColumnDef {
-            name: column_name.to_string(),
+        column_defs.push(ColumnDef {
+            name: column_name.to_owned(),
             data_type: datatype,
             is_nullable,
             default_constraint: vec![],
             semantic_type,
             comment: String::new(),
-            datatype_extension: datatype_extension.clone(),
+            datatype_extension: *datatype_extension,
             options: options.clone(),
-        };
-        column_defs.push(column_def);
+        });
     }
 
     let time_index = time_index.context(MissingTimestampColumnSnafu {
         msg: format!("table is {}", table_name.table),
     })?;
 
-    let expr = CreateTableExpr {
+    Ok(CreateTableExpr {
         catalog_name: table_name.catalog.to_string(),
         schema_name: table_name.schema.to_string(),
         table_name: table_name.table.to_string(),
@@ -187,9 +187,7 @@ pub fn build_create_table_expr(
         table_options: Default::default(),
         table_id: table_id.map(|id| api::v1::TableId { id }),
         engine: engine.to_string(),
-    };
-
-    Ok(expr)
+    })
 }
 
 /// Find columns that are not present in the schema and return them as `AddColumns`
@@ -210,7 +208,7 @@ pub fn extract_new_columns(
                 default_constraint: vec![],
                 semantic_type: expr.semantic_type,
                 comment: String::new(),
-                datatype_extension: expr.datatype_extension.clone(),
+                datatype_extension: *expr.datatype_extension,
                 options: expr.options.clone(),
             });
             AddColumn {

src/common/grpc/Cargo.toml

@@ -19,7 +19,8 @@ common-telemetry.workspace = true
 common-time.workspace = true
 dashmap.workspace = true
 datatypes.workspace = true
-flatbuffers = "23.1"
+flatbuffers = "24"
+hyper.workspace = true
 lazy_static.workspace = true
 prost.workspace = true
 snafu.workspace = true
@@ -29,6 +30,7 @@ tower.workspace = true
 
 [dev-dependencies]
 criterion = "0.4"
+hyper-util = { workspace = true, features = ["tokio"] }
 rand.workspace = true
 
 [[bench]]

src/common/grpc/src/channel_manager.rs

@@ -25,7 +25,7 @@ use snafu::{OptionExt, ResultExt};
 use tonic::transport::{
     Certificate, Channel as InnerChannel, ClientTlsConfig, Endpoint, Identity, Uri,
 };
-use tower::make::MakeConnection;
+use tower::Service;
 
 use crate::error::{CreateChannelSnafu, InvalidConfigFilePathSnafu, InvalidTlsConfigSnafu, Result};
 
@@ -137,8 +137,8 @@ impl ChannelManager {
         connector: C,
     ) -> Result<InnerChannel>
     where
-        C: MakeConnection<Uri> + Send + 'static,
-        C::Connection: Unpin + Send + 'static,
+        C: Service<Uri> + Send + 'static,
+        C::Response: hyper::rt::Read + hyper::rt::Write + Send + Unpin,
         C::Future: Send + 'static,
         Box<dyn std::error::Error + Send + Sync>: From<C::Error> + Send + 'static,
     {
@@ -607,7 +607,7 @@ mod tests {
         });
 
         let (client, _) = tokio::io::duplex(1024);
-        let mut client = Some(client);
+        let mut client = Some(hyper_util::rt::TokioIo::new(client));
         let res = mgr.reset_with_connector(
             addr,
             service_fn(move |_| {

src/common/grpc/src/select.rs

@@ -205,7 +205,7 @@ pub fn values(arrays: &[VectorRef]) -> Result<Values> {
             ConcreteDataType::Interval(IntervalType::DayTime(_)),
             IntervalDayTimeVector,
             interval_day_time_values,
-            |x| { x.into_native() }
+            |x| { x.to_i64() }
         ),
         (
             ConcreteDataType::Interval(IntervalType::MonthDayNano(_)),
@@ -232,6 +232,8 @@ pub fn values(arrays: &[VectorRef]) -> Result<Values> {
 mod tests {
     use std::sync::Arc;
 
+    use datatypes::arrow::datatypes::{IntervalDayTime, IntervalMonthDayNano};
+
     use super::*;
 
     #[test]
@@ -266,7 +268,12 @@ mod tests {
 
     #[test]
     fn test_convert_arrow_array_interval_day_time() {
-        let array = IntervalDayTimeVector::from(vec![Some(1), Some(2), None, Some(3)]);
+        let array = IntervalDayTimeVector::from(vec![
+            Some(IntervalDayTime::new(0, 1)),
+            Some(IntervalDayTime::new(0, 2)),
+            None,
+            Some(IntervalDayTime::new(0, 3)),
+        ]);
         let array: VectorRef = Arc::new(array);
 
         let values = values(&[array]).unwrap();
@@ -276,7 +283,12 @@ mod tests {
 
     #[test]
     fn test_convert_arrow_array_interval_month_day_nano() {
-        let array = IntervalMonthDayNanoVector::from(vec![Some(1), Some(2), None, Some(3)]);
+        let array = IntervalMonthDayNanoVector::from(vec![
+            Some(IntervalMonthDayNano::new(0, 0, 1)),
+            Some(IntervalMonthDayNano::new(0, 0, 2)),
+            None,
+            Some(IntervalMonthDayNano::new(0, 0, 3)),
+        ]);
         let array: VectorRef = Arc::new(array);
 
         let values = values(&[array]).unwrap();

src/common/macro/src/range_fn.rs

@@ -138,7 +138,7 @@ fn build_struct(
                 datafusion_expr::create_udf(
                     Self::name(),
                     Self::input_type(),
-                    Arc::new(Self::return_type()),
+                    Self::return_type(),
                     Volatility::Immutable,
                     Arc::new(Self::calc) as _,
                 )

src/common/meta/src/ddl/alter_table/region_request.rs

@@ -133,10 +133,8 @@ fn create_proto_alter_kind(
         Kind::RenameTable(_) => Ok(None),
         Kind::SetTableOptions(v) => Ok(Some(alter_request::Kind::SetTableOptions(v.clone()))),
         Kind::UnsetTableOptions(v) => Ok(Some(alter_request::Kind::UnsetTableOptions(v.clone()))),
-        Kind::SetColumnFulltext(v) => Ok(Some(alter_request::Kind::SetColumnFulltext(v.clone()))),
-        Kind::UnsetColumnFulltext(v) => {
-            Ok(Some(alter_request::Kind::UnsetColumnFulltext(v.clone())))
-        }
+        Kind::SetIndex(v) => Ok(Some(alter_request::Kind::SetIndex(v.clone()))),
+        Kind::UnsetIndex(v) => Ok(Some(alter_request::Kind::UnsetIndex(v.clone()))),
     }
 }
 

src/common/meta/src/ddl/alter_table/update_metadata.rs

@@ -60,8 +60,8 @@ impl AlterTableProcedure {
             | AlterKind::ModifyColumnTypes { .. }
             | AlterKind::SetTableOptions { .. }
             | AlterKind::UnsetTableOptions { .. }
-            | AlterKind::SetColumnFulltext { .. }
-            | AlterKind::UnsetColumnFulltext { .. } => {}
+            | AlterKind::SetIndex { .. }
+            | AlterKind::UnsetIndex { .. } => {}
         }
 
         Ok(new_info)

src/common/meta/src/ddl/create_logical_tables.rs

@@ -21,7 +21,7 @@ use api::v1::CreateTableExpr;
 use async_trait::async_trait;
 use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
 use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status};
-use common_telemetry::warn;
+use common_telemetry::{debug, warn};
 use futures_util::future::join_all;
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, ResultExt};
@@ -143,7 +143,12 @@ impl CreateLogicalTablesProcedure {
 
         for peer in leaders {
             let requester = self.context.node_manager.datanode(&peer).await;
-            let request = self.make_request(&peer, region_routes)?;
+            let Some(request) = self.make_request(&peer, region_routes)? else {
+                debug!("no region request to send to datanode {}", peer);
+                // We can skip the rest of the datanodes,
+                // the rest of the datanodes should have the same result.
+                break;
+            };
 
             create_region_tasks.push(async move {
                 requester

src/common/meta/src/ddl/create_logical_tables/check.rs

@@ -25,7 +25,7 @@ impl CreateLogicalTablesProcedure {
         Ok(())
     }
 
-    pub(crate) async fn check_tables_already_exist(&mut self) -> Result<()> {
+    pub async fn check_tables_already_exist(&mut self) -> Result<()> {
         let table_name_keys = self
             .data
             .all_create_table_exprs()

src/common/meta/src/ddl/create_logical_tables/region_request.rs

@@ -15,6 +15,7 @@
 use std::collections::HashMap;
 
 use api::v1::region::{region_request, CreateRequests, RegionRequest, RegionRequestHeader};
+use common_telemetry::debug;
 use common_telemetry::tracing_context::TracingContext;
 use store_api::storage::RegionId;
 
@@ -31,11 +32,15 @@ impl CreateLogicalTablesProcedure {
         &self,
         peer: &Peer,
         region_routes: &[RegionRoute],
-    ) -> Result<RegionRequest> {
+    ) -> Result<Option<RegionRequest>> {
         let tasks = &self.data.tasks;
+        let table_ids_already_exists = &self.data.table_ids_already_exists;
         let regions_on_this_peer = find_leader_regions(region_routes, peer);
         let mut requests = Vec::with_capacity(tasks.len() * regions_on_this_peer.len());
-        for task in tasks {
+        for (task, table_id_already_exists) in tasks.iter().zip(table_ids_already_exists) {
+            if table_id_already_exists.is_some() {
+                continue;
+            }
             let create_table_expr = &task.create_table;
             let catalog = &create_table_expr.catalog_name;
             let schema = &create_table_expr.schema_name;
@@ -51,13 +56,18 @@ impl CreateLogicalTablesProcedure {
             }
         }
 
-        Ok(RegionRequest {
+        if requests.is_empty() {
+            debug!("no region request to send to datanodes");
+            return Ok(None);
+        }
+
+        Ok(Some(RegionRequest {
             header: Some(RegionRequestHeader {
                 tracing_context: TracingContext::from_current_span().to_w3c(),
                 ..Default::default()
             }),
             body: Some(region_request::Body::Creates(CreateRequests { requests })),
-        })
+        }))
     }
 
     fn create_region_request_builder(

src/common/meta/src/ddl/create_table_template.rs

@@ -47,7 +47,7 @@ pub(crate) fn build_template(create_table_expr: &CreateTableExpr) -> Result<Crea
                     default_constraint: c.default_constraint.clone(),
                     semantic_type: semantic_type as i32,
                     comment: String::new(),
-                    datatype_extension: c.datatype_extension.clone(),
+                    datatype_extension: c.datatype_extension,
                     options: c.options.clone(),
                 }),
                 column_id: i as u32,

src/common/meta/src/ddl/drop_table.rs

@@ -78,7 +78,7 @@ impl DropTableProcedure {
         })
     }
 
-    pub(crate) async fn on_prepare<'a>(&mut self) -> Result<Status> {
+    pub(crate) async fn on_prepare(&mut self) -> Result<Status> {
         if self.executor.on_prepare(&self.context).await?.stop() {
             return Ok(Status::done());
         }

src/common/meta/src/error.rs

@@ -703,6 +703,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Invalid topic name prefix: {}", prefix))]
+    InvalidTopicNamePrefix {
+        prefix: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -770,7 +777,8 @@ impl ErrorExt for Error {
             | MismatchPrefix { .. }
             | TlsConfig { .. }
             | InvalidSetDatabaseOption { .. }
-            | InvalidUnsetDatabaseOption { .. } => StatusCode::InvalidArguments,
+            | InvalidUnsetDatabaseOption { .. }
+            | InvalidTopicNamePrefix { .. } => StatusCode::InvalidArguments,
 
             FlowNotFound { .. } => StatusCode::FlowNotFound,
             FlowRouteNotFound { .. } => StatusCode::Unexpected,