fix: choose frontend randomly

* feat: set read-preference for grpc client

* todo

* address PR comments

* fix ci

.github/scripts/create-version.sh

@@ -10,17 +10,17 @@ set -e
 function create_version() {
   # Read from envrionment variables.
   if [ -z "$GITHUB_EVENT_NAME" ]; then
-      echo "GITHUB_EVENT_NAME is empty"
+      echo "GITHUB_EVENT_NAME is empty" >&2
       exit 1
   fi
 
   if [ -z "$NEXT_RELEASE_VERSION" ]; then
-      echo "NEXT_RELEASE_VERSION is empty"
-      exit 1
+      echo "NEXT_RELEASE_VERSION is empty, use version from Cargo.toml" >&2
+      export NEXT_RELEASE_VERSION=$(grep '^version = ' Cargo.toml | cut -d '"' -f 2 | head -n 1)
   fi
 
   if [ -z "$NIGHTLY_RELEASE_PREFIX" ]; then
-      echo "NIGHTLY_RELEASE_PREFIX is empty"
+      echo "NIGHTLY_RELEASE_PREFIX is empty" >&2
       exit 1
   fi
 
@@ -35,7 +35,7 @@ function create_version() {
   # It will be like 'dev-2023080819-f0e7216c'.
   if [ "$NEXT_RELEASE_VERSION" = dev ]; then
     if [ -z "$COMMIT_SHA" ]; then
-      echo "COMMIT_SHA is empty in dev build"
+      echo "COMMIT_SHA is empty in dev build" >&2
       exit 1
     fi
     echo "dev-$(date "+%Y%m%d-%s")-$(echo "$COMMIT_SHA" | cut -c1-8)"
@@ -45,7 +45,7 @@ function create_version() {
   # Note: Only output 'version=xxx' to stdout when everything is ok, so that it can be used in GitHub Actions Outputs.
   if [ "$GITHUB_EVENT_NAME" = push ]; then
     if [ -z "$GITHUB_REF_NAME" ]; then
-      echo "GITHUB_REF_NAME is empty in push event"
+      echo "GITHUB_REF_NAME is empty in push event" >&2
       exit 1
     fi
     echo "$GITHUB_REF_NAME"
@@ -54,7 +54,7 @@ function create_version() {
   elif [ "$GITHUB_EVENT_NAME" = schedule ]; then
     echo "$NEXT_RELEASE_VERSION-$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")"
   else
-    echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME"
+    echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME" >&2
     exit 1
   fi
 }

.github/workflows/develop.yml

@@ -22,6 +22,7 @@ concurrency:
 
 jobs:
   check-typos-and-docs:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Check typos and docs
     runs-on: ubuntu-latest
     steps:
@@ -36,6 +37,7 @@ jobs:
           || (echo "'config/config.md' is not up-to-date, please run 'make config-docs'." && exit 1)
 
   license-header-check:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: ubuntu-latest
     name: Check License Header
     steps:
@@ -45,6 +47,7 @@ jobs:
       - uses: korandoru/hawkeye@v5
 
   check:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Check
     runs-on: ${{ matrix.os }}
     strategy:
@@ -71,6 +74,7 @@ jobs:
         run: cargo check --locked --workspace --all-targets
 
   toml:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Toml Check
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -85,6 +89,7 @@ jobs:
         run: taplo format --check
 
   build:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Build GreptimeDB binaries
     runs-on: ${{ matrix.os }}
     strategy:
@@ -127,6 +132,7 @@ jobs:
           version: current
 
   fuzztest:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Fuzz Test
     needs: build
     runs-on: ubuntu-latest
@@ -183,6 +189,7 @@ jobs:
           max-total-time: 120
 
   unstable-fuzztest:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Unstable Fuzz Test
     needs: build-greptime-ci
     runs-on: ubuntu-latest
@@ -244,6 +251,7 @@ jobs:
           retention-days: 3
 
   build-greptime-ci:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Build GreptimeDB binary (profile-CI)
     runs-on: ${{ matrix.os }}
     strategy:
@@ -285,6 +293,7 @@ jobs:
           version: current
 
   distributed-fuzztest:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Fuzz Test (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
     runs-on: ubuntu-latest
     needs:  build-greptime-ci
@@ -416,6 +425,7 @@ jobs:
           docker system prune -f
 
   distributed-fuzztest-with-chaos:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Fuzz Test with Chaos (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
     runs-on: ubuntu-latest
     needs:  build-greptime-ci
@@ -563,6 +573,7 @@ jobs:
           docker system prune -f
 
   sqlness:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Sqlness Test (${{ matrix.mode.name }})
     needs: build
     runs-on: ${{ matrix.os }}
@@ -609,6 +620,7 @@ jobs:
           retention-days: 3
 
   fmt:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Rustfmt
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -626,6 +638,7 @@ jobs:
         run: make fmt-check
 
   clippy:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Clippy
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -651,6 +664,7 @@ jobs:
         run: make clippy
 
   conflict-check:
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Check for conflict
     runs-on: ubuntu-latest
     steps:
@@ -661,7 +675,7 @@ jobs:
         uses: olivernybroe/action-conflict-finder@v4.0
 
   test:
-    if: github.event_name != 'merge_group'
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'merge_group' }}
     runs-on: ubuntu-22.04-arm
     timeout-minutes: 60
     needs:  [conflict-check, clippy, fmt]
@@ -713,7 +727,7 @@ jobs:
           UNITTEST_LOG_DIR: "__unittest_logs"
 
   coverage:
-    if: github.event_name == 'merge_group'
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && github.event_name == 'merge_group' }}
     runs-on: ubuntu-22.04-8-cores
     timeout-minutes: 60
     steps:
@@ -773,6 +787,7 @@ jobs:
           verbose: true
 
   # compat:
+  #   if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
   #   name: Compatibility Test
   #   needs: build
   #   runs-on: ubuntu-22.04

.github/workflows/nightly-ci.yml

@@ -117,16 +117,16 @@ jobs:
     name: Run clean build on Linux
     runs-on: ubuntu-latest
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    timeout-minutes: 60
+    timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           persist-credentials: false
-      - uses: cachix/install-nix-action@v27
+      - uses: cachix/install-nix-action@v31
         with:
           nix_path: nixpkgs=channel:nixos-24.11
-      - run: nix develop --command cargo build
+      - run: nix develop --command cargo build --bin greptime
 
   check-status:
     name: Check status

.github/workflows/release.yml

@@ -90,8 +90,6 @@ env:
 
   # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
-  # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.14.0
 
 jobs:
   allocate-runners:
@@ -135,7 +133,6 @@ jobs:
         env:
           GITHUB_EVENT_NAME: ${{ github.event_name }}
           GITHUB_REF_NAME: ${{ github.ref_name }}
-          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
           NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
 
       - name: Allocate linux-amd64 runner
@@ -467,6 +464,29 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DOCS_REPO_TOKEN: ${{ secrets.DOCS_REPO_TOKEN }}
 
+  bump-website-version:
+    name: Bump website version
+    if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [allocate-runners]
+    runs-on: ubuntu-latest
+    # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+    permissions:
+      issues: write # Allows the action to create issues for cyborg.
+      contents: write # Allows the action to create a release.
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: ./.github/actions/setup-cyborg
+      - name: Bump website version
+        working-directory: cyborg
+        run: pnpm tsx bin/bump-website-version.ts
+        env:
+          VERSION: ${{ needs.allocate-runners.outputs.version }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WEBSITE_REPO_TOKEN: ${{ secrets.WEBSITE_REPO_TOKEN }}
+
   notification:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || github.event_name == 'schedule') && always() }}
     name: Send notification to Greptime team

.gitignore

@@ -28,6 +28,7 @@ debug/
 # Logs
 **/__unittest_logs
 logs/
+!grafana/dashboards/logs/
 
 # cpython's generated python byte code
 **/__pycache__/

Cargo.toml

@@ -68,15 +68,16 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.14.3"
+version = "0.15.0"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.lints]
-clippy.print_stdout = "warn"
-clippy.print_stderr = "warn"
 clippy.dbg_macro = "warn"
 clippy.implicit_clone = "warn"
+clippy.result_large_err = "allow"
+clippy.large_enum_variant = "allow"
+clippy.doc_overindented_list_items = "allow"
 rust.unknown_lints = "deny"
 rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
 
@@ -129,7 +130,7 @@ etcd-client = "0.14"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "4d4136692fe7fbbd509ebc8c902f6afcc0ce61e4" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "17a3550751c8b1e02ec16be40101d5f24dc255c3" }
 hex = "0.4"
 http = "1"
 humantime = "2.1"
@@ -161,9 +162,7 @@ parquet = { version = "54.2", default-features = false, features = ["arrow", "as
 paste = "1.0"
 pin-project = "1.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-promql-parser = { git = "https://github.com/GreptimeTeam/promql-parser.git", rev = "0410e8b459dda7cb222ce9596f8bf3971bd07bd2", features = [
-    "ser",
-] }
+promql-parser = { version = "0.5.1", features = ["ser"] }
 prost = { version = "0.13", features = ["no-recursion-limit"] }
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.9"

Makefile

@@ -8,7 +8,7 @@ CARGO_BUILD_OPTS := --locked
 IMAGE_REGISTRY ?= docker.io
 IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
-DEV_BUILDER_IMAGE_TAG ?= 2024-12-25-a71b93dd-20250305072908
+DEV_BUILDER_IMAGE_TAG ?= 2025-04-15-1a517ec8-20250428023155
 BUILDX_MULTI_PLATFORM_BUILD ?= false
 BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu

README.md

@@ -8,6 +8,8 @@
 
 <h2 align="center">Real-Time & Cloud-Native Observability  Database<br/>for metrics, logs, and traces</h2>
 
+>  Delivers sub-second querying at PB scale and exceptional cost efficiency from edge to cloud.
+
 <div align="center">
 <h3 align="center">
   <a href="https://greptime.com/product/cloud">GreptimeCloud</a> |
@@ -49,74 +51,77 @@
 </div>
 
 - [Introduction](#introduction)
-- [**Features: Why GreptimeDB**](#why-greptimedb)
-- [Architecture](https://docs.greptime.com/contributor-guide/overview/#architecture)
-- [Try it for free](#try-greptimedb)
+- [⭐ Key Features](#features)
+- [Quick Comparison](#quick-comparison)
+- [Architecture](#architecture)
+- [Try GreptimeDB](#try-greptimedb)
 - [Getting Started](#getting-started)
-- [Project Status](#project-status)
-- [Join the community](#community)
-  - [Contributing](#contributing)
+- [Build From Source](#build-from-source)
 - [Tools & Extensions](#tools--extensions)
+- [Project Status](#project-status)
+- [Community](#community)
 - [License](#license)
+- [Commercial Support](#commercial-support)
+- [Contributing](#contributing)
 - [Acknowledgement](#acknowledgement)
 
 ## Introduction
 
-**GreptimeDB** is an open-source, cloud-native, unified & cost-effective observability database for **Metrics**, **Logs**, and **Traces**. You can gain real-time insights from Edge to Cloud at Any Scale.
+**GreptimeDB** is an open-source, cloud-native database purpose-built for the unified collection and analysis of observability data (metrics, logs, and traces). Whether you’re operating on the edge, in the cloud, or across hybrid environments, GreptimeDB empowers real-time insights at massive scale — all in one system.
 
-## News
+## Features
 
-**[GreptimeDB tops JSONBench's billion-record cold run test!](https://greptime.com/blogs/2025-03-18-jsonbench-greptimedb-performance)**
+|   Feature  | Description |
+| --------- | ----------- |
+| [Unified Observability Data](https://docs.greptime.com/user-guide/concepts/why-greptimedb) | Store metrics, logs, and traces as timestamped, contextual wide events. Query via [SQL](https://docs.greptime.com/user-guide/query-data/sql), [PromQL](https://docs.greptime.com/user-guide/query-data/promql), and [streaming](https://docs.greptime.com/user-guide/flow-computation/overview). |
+| [High Performance & Cost Effective](https://docs.greptime.com/user-guide/manage-data/data-index) | Written in Rust, with a distributed query engine, [rich indexing](https://docs.greptime.com/user-guide/manage-data/data-index), and optimized columnar storage, delivering sub-second responses at PB scale. |
+| [Cloud-Native Architecture](https://docs.greptime.com/user-guide/concepts/architecture) | Designed for [Kubernetes](https://docs.greptime.com/user-guide/deployments/deploy-on-kubernetes/greptimedb-operator-management), with compute/storage separation, native object storage (AWS S3, Azure Blob, etc.) and seamless cross-cloud access. |
+| [Developer-Friendly](https://docs.greptime.com/user-guide/protocols/overview) | Access via SQL/PromQL interfaces, REST API, MySQL/PostgreSQL protocols, and popular ingestion [protocols](https://docs.greptime.com/user-guide/protocols/overview). |
+| [Flexible Deployment](https://docs.greptime.com/user-guide/deployments/overview) | Deploy anywhere: edge (including ARM/[Android](https://docs.greptime.com/user-guide/deployments/run-on-android)) or cloud, with unified APIs and efficient data sync. |
 
-## Why GreptimeDB
+Learn more in [Why GreptimeDB](https://docs.greptime.com/user-guide/concepts/why-greptimedb) and [Observability 2.0 and the Database for It](https://greptime.com/blogs/2025-04-25-greptimedb-observability2-new-database).
 
-Our core developers have been building observability data platforms for years. Based on our best practices, GreptimeDB was born to give you:
+## Quick Comparison
 
-* **Unified Processing of Observability Data**
+| Feature                         | GreptimeDB            | Traditional TSDB   | Log Stores      |
+|----------------------------------|-----------------------|--------------------|-----------------|
+| Data Types                      | Metrics, Logs, Traces | Metrics only       | Logs only       |
+| Query Language                  | SQL, PromQL, Streaming|  Custom/PromQL     | Custom/DSL      |
+| Deployment                      | Edge + Cloud          | Cloud/On-prem      | Mostly central  |
+| Indexing & Performance          | PB-Scale, Sub-second  | Varies             | Varies          |
+| Integration                     | REST, SQL, Common protocols | Varies     | Varies          |
 
-  A unified database that treats metrics, logs, and traces as timestamped wide events with context, supporting [SQL](https://docs.greptime.com/user-guide/query-data/sql)/[PromQL](https://docs.greptime.com/user-guide/query-data/promql) queries and [stream processing](https://docs.greptime.com/user-guide/flow-computation/overview) to simplify complex data stacks.
+**Performance:**
+* [GreptimeDB tops JSONBench's billion-record cold run test!](https://greptime.com/blogs/2025-03-18-jsonbench-greptimedb-performance)
+* [TSBS Benchmark](https://github.com/GreptimeTeam/greptimedb/tree/main/docs/benchmarks/tsbs)
 
-* **High Performance and Cost-effective**
+Read [more benchmark reports](https://docs.greptime.com/user-guide/concepts/features-that-you-concern#how-is-greptimedbs-performance-compared-to-other-solutions).
 
-   Written in Rust, combines a distributed query engine with [rich indexing](https://docs.greptime.com/user-guide/manage-data/data-index) (inverted, fulltext, skip data, and vector) and optimized columnar storage to deliver sub-second responses on petabyte-scale data and high-cost efficiency.
+## Architecture
 
-* **Cloud-native Distributed Database**
-
-  Built for [Kubernetes](https://docs.greptime.com/user-guide/deployments/deploy-on-kubernetes/greptimedb-operator-management). GreptimeDB achieves seamless scalability with its [cloud-native architecture](https://docs.greptime.com/user-guide/concepts/architecture) of separated compute and storage, built on object storage (AWS S3, Azure Blob Storage, etc.) while enabling cross-cloud deployment through a unified data access layer.
-
-* **Developer-Friendly**
-
-  Access standardized SQL/PromQL interfaces through built-in web dashboard, REST API, and MySQL/PostgreSQL protocols. Supports widely adopted data ingestion [protocols](https://docs.greptime.com/user-guide/protocols/overview) for seamless migration and integration.
-
-* **Flexible Deployment Options**
-
-  Deploy GreptimeDB anywhere from ARM-based edge devices to cloud environments with unified APIs and bandwidth-efficient data synchronization. Query edge and cloud data seamlessly through identical APIs. [Learn how to run on Android](https://docs.greptime.com/user-guide/deployments/run-on-android/).
-
-For more detailed info please read  [Why GreptimeDB](https://docs.greptime.com/user-guide/concepts/why-greptimedb).
+* Read the [architecture](https://docs.greptime.com/contributor-guide/overview/#architecture) document.
+* [DeepWiki](https://deepwiki.com/GreptimeTeam/greptimedb/1-overview) provides an in-depth look at GreptimeDB:
+  <img alt="GreptimeDB System Overview" src="docs/architecture.png">
 
 ## Try GreptimeDB
 
 ### 1. [Live Demo](https://greptime.com/playground)
 
-Try out the features of GreptimeDB right from your browser.
+Experience GreptimeDB directly in your browser.
 
 ### 2. [GreptimeCloud](https://console.greptime.cloud/)
 
 Start instantly with a free cluster.
 
-### 3. Docker Image
-
-To install GreptimeDB locally, the recommended way is via Docker:
+### 3. Docker (Local Quickstart)
 
 ```shell
 docker pull greptime/greptimedb
 ```
 
-Start a GreptimeDB container with:
-
 ```shell
 docker run -p 127.0.0.1:4000-4003:4000-4003 \
-  -v "$(pwd)/greptimedb:./greptimedb_data" \
+  -v "$(pwd)/greptimedb:/greptimedb_data" \
   --name greptime --rm \
   greptime/greptimedb:latest standalone start \
   --http-addr 0.0.0.0:4000 \
@@ -124,114 +129,90 @@ docker run -p 127.0.0.1:4000-4003:4000-4003 \
   --mysql-addr 0.0.0.0:4002 \
   --postgres-addr 0.0.0.0:4003
 ```
+Dashboard: [http://localhost:4000/dashboard](http://localhost:4000/dashboard)  
+[Full Install Guide](https://docs.greptime.com/getting-started/installation/overview)
 
-Access the dashboard via `http://localhost:4000/dashboard`.
-
-Read more about [Installation](https://docs.greptime.com/getting-started/installation/overview) on docs.
+**Troubleshooting:**
+* Cannot connect to the database? Ensure that ports `4000`, `4001`, `4002`, and `4003` are not blocked by a firewall or used by other services.
+* Failed to start? Check the container logs with `docker logs greptime` for further details.
 
 ## Getting Started
 
-* [Quickstart](https://docs.greptime.com/getting-started/quick-start)
-* [User Guide](https://docs.greptime.com/user-guide/overview)
-* [Demos](https://github.com/GreptimeTeam/demo-scene)
-* [FAQ](https://docs.greptime.com/faq-and-others/faq)
+- [Quickstart](https://docs.greptime.com/getting-started/quick-start)
+- [User Guide](https://docs.greptime.com/user-guide/overview)
+- [Demo Scenes](https://github.com/GreptimeTeam/demo-scene)
+- [FAQ](https://docs.greptime.com/faq-and-others/faq)
 
-## Build
-
-Check the prerequisite:
+## Build From Source
 
+**Prerequisites:**
 * [Rust toolchain](https://www.rust-lang.org/tools/install) (nightly)
 * [Protobuf compiler](https://grpc.io/docs/protoc-installation/) (>= 3.15)
 * C/C++ building essentials, including `gcc`/`g++`/`autoconf` and glibc library (eg. `libc6-dev` on Ubuntu and `glibc-devel` on Fedora)
 * Python toolchain (optional): Required only if using some test scripts.
 
-Build GreptimeDB binary:
-
-```shell
+**Build and Run:**
+```bash
 make
-```
-
-Run a standalone server:
-
-```shell
 cargo run -- standalone start
 ```
 
 ## Tools & Extensions
 
-### Kubernetes
-
-- [GreptimeDB Operator](https://github.com/GrepTimeTeam/greptimedb-operator)
-
-### Dashboard
-
-- [The dashboard UI for GreptimeDB](https://github.com/GreptimeTeam/dashboard)
-
-### SDK
-
-- [GreptimeDB Go Ingester](https://github.com/GreptimeTeam/greptimedb-ingester-go)
-- [GreptimeDB Java Ingester](https://github.com/GreptimeTeam/greptimedb-ingester-java)
-- [GreptimeDB C++ Ingester](https://github.com/GreptimeTeam/greptimedb-ingester-cpp)
-- [GreptimeDB Erlang Ingester](https://github.com/GreptimeTeam/greptimedb-ingester-erl)
-- [GreptimeDB Rust Ingester](https://github.com/GreptimeTeam/greptimedb-ingester-rust)
-- [GreptimeDB JavaScript Ingester](https://github.com/GreptimeTeam/greptimedb-ingester-js)
-
-### Grafana Dashboard
-
-Our official Grafana dashboard for monitoring GreptimeDB is available at [grafana](grafana/README.md) directory.
+- **Kubernetes:** [GreptimeDB Operator](https://github.com/GrepTimeTeam/greptimedb-operator)
+- **Helm Charts:** [Greptime Helm Charts](https://github.com/GreptimeTeam/helm-charts)
+- **Dashboard:** [Web UI](https://github.com/GreptimeTeam/dashboard)
+- **SDKs/Ingester:** [Go](https://github.com/GreptimeTeam/greptimedb-ingester-go), [Java](https://github.com/GreptimeTeam/greptimedb-ingester-java), [C++](https://github.com/GreptimeTeam/greptimedb-ingester-cpp), [Erlang](https://github.com/GreptimeTeam/greptimedb-ingester-erl), [Rust](https://github.com/GreptimeTeam/greptimedb-ingester-rust), [JS](https://github.com/GreptimeTeam/greptimedb-ingester-js)
+- **Grafana**: [Official Dashboard](https://github.com/GreptimeTeam/greptimedb/blob/main/grafana/README.md)
 
 ## Project Status
 
-GreptimeDB is currently in Beta. We are targeting GA (General Availability) with v1.0 release by Early 2025.
+> **Status:** Beta.  
+> **GA (v1.0):** Targeted for mid 2025.
 
-While in Beta, GreptimeDB is already:
-
-* Being used in production by early adopters
-* Actively maintained with regular releases, [about version number](https://docs.greptime.com/nightly/reference/about-greptimedb-version)
-* Suitable for testing and evaluation
+- Being used in production by early adopters
+- Stable, actively maintained, with regular releases ([version info](https://docs.greptime.com/nightly/reference/about-greptimedb-version))
+- Suitable for evaluation and pilot deployments
 
 For production use, we recommend using the latest stable release.
+[![Star History Chart](https://api.star-history.com/svg?repos=GreptimeTeam/GreptimeDB&type=Date)](https://www.star-history.com/#GreptimeTeam/GreptimeDB&Date)
+
+If you find this project useful, a ⭐ would mean a lot to us!
+<img alt="Known Users" src="https://greptime.com/logo/img/users.png"/>
 
 ## Community
 
-Our core team is thrilled to see you participate in any ways you like. When you are stuck, try to
-ask for help by filling an issue with a detailed description of what you were trying to do
-and what went wrong. If you have any questions or if you would like to get involved in our
-community, please check out:
+We invite you to engage and contribute!
 
-- GreptimeDB Community on [Slack](https://greptime.com/slack)
-- GreptimeDB [GitHub Discussions forum](https://github.com/GreptimeTeam/greptimedb/discussions)
-- Greptime official [website](https://greptime.com)
-
-In addition, you may:
-
-- View our official [Blog](https://greptime.com/blogs/)
-- Connect us with [Linkedin](https://www.linkedin.com/company/greptime/)
-- Follow us on [Twitter](https://twitter.com/greptime)
-
-## Commercial Support
-
-If you are running GreptimeDB OSS in your organization, we offer additional
-enterprise add-ons, installation services, training, and consulting. [Contact
-us](https://greptime.com/contactus) and we will reach out to you with more
-detail of our commercial license.
+- [Slack](https://greptime.com/slack)
+- [Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
+- [Official Website](https://greptime.com/)
+- [Blog](https://greptime.com/blogs/)
+- [LinkedIn](https://www.linkedin.com/company/greptime/)
+- [Twitter](https://twitter.com/greptime)
 
 ## License
 
-GreptimeDB uses the [Apache License 2.0](https://apache.org/licenses/LICENSE-2.0.txt) to strike a balance between
-open contributions and allowing you to use the software however you want.
+GreptimeDB is licensed under the [Apache License 2.0](https://apache.org/licenses/LICENSE-2.0.txt).
+
+## Commercial Support
+
+Running GreptimeDB in your organization?  
+We offer enterprise add-ons, services, training, and consulting.  
+[Contact us](https://greptime.com/contactus) for details.
 
 ## Contributing
 
-Please refer to [contribution guidelines](CONTRIBUTING.md) and [internal concepts docs](https://docs.greptime.com/contributor-guide/overview.html) for more information.
+- Read our [Contribution Guidelines](https://github.com/GreptimeTeam/greptimedb/blob/main/CONTRIBUTING.md).
+- Explore [Internal Concepts](https://docs.greptime.com/contributor-guide/overview.html) and [DeepWiki](https://deepwiki.com/GreptimeTeam/greptimedb).
+- Pick up a [good first issue](https://github.com/GreptimeTeam/greptimedb/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) and join the #contributors [Slack](https://greptime.com/slack) channel.
 
 ## Acknowledgement
 
-Special thanks to all the contributors who have propelled GreptimeDB forward. For a complete list of contributors, please refer to [AUTHOR.md](AUTHOR.md).
+Special thanks to all contributors! See [AUTHORS.md](https://github.com/GreptimeTeam/greptimedb/blob/main/AUTHOR.md).
 
-- GreptimeDB uses [Apache Arrow™](https://arrow.apache.org/) as the memory model and [Apache Parquet™](https://parquet.apache.org/) as the persistent file format.
-- GreptimeDB's query engine is powered by [Apache Arrow DataFusion™](https://arrow.apache.org/datafusion/).
-- [Apache OpenDAL™](https://opendal.apache.org) gives GreptimeDB a very general and elegant data access abstraction layer.
-- GreptimeDB's meta service is based on [etcd](https://etcd.io/).
-
-<img alt="Known Users" src="https://greptime.com/logo/img/users.png"/>
+- Uses [Apache Arrow™](https://arrow.apache.org/) (memory model)
+- [Apache Parquet™](https://parquet.apache.org/) (file storage)
+- [Apache Arrow DataFusion™](https://arrow.apache.org/datafusion/) (query engine)
+- [Apache OpenDAL™](https://opendal.apache.org/) (data access abstraction)
+- [etcd](https://etcd.io/) (meta service)

cyborg/bin/bump-website-version.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2023 Greptime Team
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as core from "@actions/core";
+import {obtainClient} from "@/common";
+
+async function triggerWorkflow(workflowId: string, version: string) {
+  const websiteClient = obtainClient("WEBSITE_REPO_TOKEN")
+  try {
+    await websiteClient.rest.actions.createWorkflowDispatch({
+      owner: "GreptimeTeam",
+      repo: "website",
+      workflow_id: workflowId,
+      ref: "main",
+      inputs: {
+        version,
+      },
+    });
+    console.log(`Successfully triggered ${workflowId} workflow with version ${version}`);
+  } catch (error) {
+    core.setFailed(`Failed to trigger workflow: ${error.message}`);
+  }
+}
+
+const version = process.env.VERSION;
+if (!version) {
+  core.setFailed("VERSION environment variable is required");
+  process.exit(1);
+}
+
+// Remove 'v' prefix if exists
+const cleanVersion = version.startsWith('v') ? version.slice(1) : version;
+
+if (cleanVersion.includes('nightly')) {
+  console.log('Nightly version detected, skipping workflow trigger.');
+  process.exit(0);
+}
+
+try {
+  triggerWorkflow('bump-patch-version.yml', cleanVersion);
+} catch (error) {
+  core.setFailed(`Error processing version: ${error.message}`);
+  process.exit(1);
+}

flake.lock

@@ -8,11 +8,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1737613896,
-        "narHash": "sha256-ldqXIglq74C7yKMFUzrS9xMT/EVs26vZpOD68Sh7OcU=",
+        "lastModified": 1745735608,
+        "narHash": "sha256-L0jzm815XBFfF2wCFmR+M1CF+beIEFj6SxlqVKF59Ec=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "303a062fdd8e89f233db05868468975d17855d80",
+        "rev": "c39a78eba6ed2a022cc3218db90d485077101496",
         "type": "github"
       },
       "original": {
@@ -41,11 +41,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1737569578,
-        "narHash": "sha256-6qY0pk2QmUtBT9Mywdvif0i/CLVgpCjMUn6g9vB+f3M=",
+        "lastModified": 1745487689,
+        "narHash": "sha256-FQoi3R0NjQeBAsEOo49b5tbDPcJSMWc3QhhaIi9eddw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "47addd76727f42d351590c905d9d1905ca895b82",
+        "rev": "5630cf13cceac06cefe9fc607e8dfa8fb342dde3",
         "type": "github"
       },
       "original": {
@@ -65,11 +65,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1737581772,
-        "narHash": "sha256-t1P2Pe3FAX9TlJsCZbmJ3wn+C4qr6aSMypAOu8WNsN0=",
+        "lastModified": 1745694049,
+        "narHash": "sha256-fxvRYH/tS7hGQeg9zCVh5RBcSWT+JGJet7RA8Ss+rC0=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "582af7ee9c8d84f5d534272fc7de9f292bd849be",
+        "rev": "d8887c0758bbd2d5f752d5bd405d4491e90e7ed6",
         "type": "github"
       },
       "original": {

flake.nix

@@ -21,7 +21,7 @@
         lib = nixpkgs.lib;
         rustToolchain = fenix.packages.${system}.fromToolchainName {
           name = (lib.importTOML ./rust-toolchain.toml).toolchain.channel;
-          sha256 = "sha256-f/CVA1EC61EWbh0SjaRNhLL0Ypx2ObupbzigZp8NmL4=";
+          sha256 = "sha256-arzEYlWLGGYeOhECHpBxQd2joZ4rPKV3qLNnZ+eql6A=";
         };
       in
       {

grafana/README.md

@@ -2,30 +2,63 @@
 
 ## Overview
 
-This repository maintains the Grafana dashboards for GreptimeDB. It has two types of dashboards:
+This repository contains Grafana dashboards for visualizing metrics and logs of GreptimeDB instances running in either cluster or standalone mode. **The Grafana version should be greater than 9.0**.
 
-- `cluster/dashboard.json`: The Grafana dashboard for the GreptimeDB cluster. Read the [dashboard.md](./dashboards/cluster/dashboard.md) for more details.
-- `standalone/dashboard.json`: The Grafana dashboard for the standalone GreptimeDB instance. **It's generated from the `cluster/dashboard.json` by removing the instance filter through the `make dashboards` command**. Read the [dashboard.md](./dashboards/standalone/dashboard.md) for more details.
+We highly recommend using the self-monitoring feature provided by [GreptimeDB Operator](https://github.com/GrepTimeTeam/greptimedb-operator) to automatically collect metrics and logs from your GreptimeDB instances and store them in a dedicated GreptimeDB instance.
 
-As the rapid development of GreptimeDB, the metrics may be changed, and please feel free to submit your feedback and/or contribution to this dashboard 🤗
+- **Metrics Dashboards**
 
-**NOTE**: 
+  - `dashboards/metrics/cluster/dashboard.json`: The Grafana dashboard for the GreptimeDB cluster. Read the [dashboard.md](./dashboards/metrics/cluster/dashboard.md) for more details.
+  
+  - `dashboards/metrics/standalone/dashboard.json`: The Grafana dashboard for the standalone GreptimeDB instance. **It's generated from the `cluster/dashboard.json` by removing the instance filter through the `make dashboards` command**. Read the [dashboard.md](./dashboards/metrics/standalone/dashboard.md) for more details.
 
-- The Grafana version should be greater than 9.0.
+- **Logs Dashboard**
 
-- If you want to modify the dashboards, you only need to modify the `cluster/dashboard.json` and run the `make dashboards` command to generate the `standalone/dashboard.json` and other related files.
+  The `dashboards/logs/dashboard.json` provides a comprehensive Grafana dashboard for visualizing GreptimeDB logs. To utilize this dashboard effectively, you need to collect logs in JSON format from your GreptimeDB instances and store them in a dedicated GreptimeDB instance.
 
-To maintain the dashboards easily, we use the [`dac`](https://github.com/zyy17/dac) tool to generate the intermediate dashboards and markdown documents:
+  For proper integration, the logs table must adhere to the following schema design with the table name `_gt_logs`:
 
-- `cluster/dashboard.yaml`: The intermediate dashboard for the GreptimeDB cluster.
-- `standalone/dashboard.yaml`: The intermediate dashboard for the standalone GreptimeDB instance.
+  ```sql
+  CREATE TABLE IF NOT EXISTS `_gt_logs` (
+    `pod_ip` STRING NULL,
+    `namespace` STRING NULL,
+    `cluster` STRING NULL,
+    `file` STRING NULL,
+    `module_path` STRING NULL,
+    `level` STRING NULL,
+    `target` STRING NULL,
+    `role` STRING NULL,
+    `pod` STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),
+    `message` STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false'),
+    `err` STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false'),
+    `timestamp` TIMESTAMP(9) NOT NULL,
+    TIME INDEX (`timestamp`),
+    PRIMARY KEY (`level`, `target`, `role`)
+  )
+    ENGINE=mito
+  WITH (
+    append_mode = 'true'
+  )
+  ```
+
+## Development
+
+As GreptimeDB evolves rapidly, metrics may change over time. We welcome your feedback and contributions to improve these dashboards 🤗
+
+To modify the metrics dashboards, simply edit the `dashboards/metrics/cluster/dashboard.json` file and run the `make dashboards` command. This will automatically generate the updated `dashboards/metrics/standalone/dashboard.json` and other related files.
+
+For easier dashboard maintenance, we utilize the [`dac`](https://github.com/zyy17/dac) tool to generate human-readable intermediate dashboards and documentation:
+
+- `dashboards/metrics/cluster/dashboard.yaml`: The intermediate dashboard file for the GreptimeDB cluster.
+- `dashboards/metrics/standalone/dashboard.yaml`: The intermediate dashboard file for standalone GreptimeDB instances.
 
 ## Data Sources
 
-There are two data sources for the dashboards to fetch the metrics:
+The following data sources are used to fetch metrics and logs:
 
-- **Prometheus**: Expose the metrics of GreptimeDB.
-- **Information Schema**: It is the MySQL port of the current monitored instance. The `overview` dashboard will use this datasource to show the information schema of the current instance.
+- **`${metrics}`**: Prometheus data source for providing the GreptimeDB metrics.
+- **`${logs}`**: MySQL data source for providing the GreptimeDB logs.
+- **`${information_schema}`**: MySQL data source for providing the information schema of the current instance and used for the `overview` panel. It is the MySQL port of the current monitored instance.
 
 ## Instance Filters
 
@@ -43,9 +76,9 @@ And the legend will be like: `[{{instance}}]-[{{ pod }}]`.
 
 ## Deployment
 
-### Helm
+### (Recommended) Helm Chart
 
-If you use the Helm [chart](https://github.com/GreptimeTeam/helm-charts) to deploy a GreptimeDB cluster, you can enable self-monitoring by setting the following values in your Helm chart:
+If you use the [Helm Chart](https://github.com/GreptimeTeam/helm-charts) to deploy a GreptimeDB cluster, you can enable self-monitoring by setting the following values in your Helm chart:
 
 - `monitoring.enabled=true`: Deploys a standalone GreptimeDB instance dedicated to monitoring the cluster;
 - `grafana.enabled=true`: Deploys Grafana and automatically imports the monitoring dashboard;
@@ -85,5 +118,5 @@ The standalone GreptimeDB instance will collect metrics from your cluster, and t
 
 3. **Import the dashboards based on your deployment scenario**
 
-   - **Cluster**: Import the `cluster/dashboard.json` dashboard.
-   - **Standalone**: Import the `standalone/dashboard.json` dashboard.
+   - **Cluster**: Import the `dashboards/metrics/cluster/dashboard.json` dashboard.
+   - **Standalone**: Import the `dashboards/metrics/standalone/dashboard.json` dashboard.

grafana/dashboards/logs/dashboard.json

@@ -0,0 +1,292 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 12,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "default": false,
+        "type": "mysql",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 20,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": true,
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "dataset": "greptime_private",
+          "datasource": {
+            "type": "mysql",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "format": "table",
+          "rawQuery": true,
+          "rawSql": "SELECT `timestamp`, CONCAT('[', `level`, ']', ' ', '<', `target`, '>', ' ', `message`),\n  `role`,\n  `pod`,\n  `pod_ip`,\n  `namespace`,\n  `cluster`,\n  `err`,\n  `file`,\n  `module_path`\nFROM\n  `_gt_logs`\nWHERE\n  (\n    \"$level\" = \"'all'\"\n    OR `level` IN ($level)\n  ) \n  AND (\n    \"$role\" = \"'all'\"\n    OR `role` IN ($role)\n  )\n  AND (\n    \"$pod\" = \"\"\n    OR `pod` = '$pod'\n  )\n  AND (\n    \"$target\" = \"\"\n    OR `target` = '$target'\n  )\n  AND (\n    \"$search\" = \"\"\n    OR matches_term(`message`, '$search')\n  )\n  AND (\n    \"$exclude\" = \"\"\n    OR NOT matches_term(`message`, '$exclude')\n  )\n  AND $__timeFilter(`timestamp`)\nORDER BY `timestamp` DESC\nLIMIT $limit;\n",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          }
+        }
+      ],
+      "title": "Logs",
+      "type": "logs"
+    }
+  ],
+  "preload": false,
+  "refresh": "",
+  "schemaVersion": 41,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": "logs",
+          "value": "P98F38F12DB221A8C"
+        },
+        "includeAll": false,
+        "name": "datasource",
+        "options": [],
+        "query": "mysql",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      },
+      {
+        "allValue": "'all'",
+        "current": {
+          "text": [
+            "$__all"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "includeAll": true,
+        "label": "level",
+        "multi": true,
+        "name": "level",
+        "options": [
+          {
+            "selected": false,
+            "text": "INFO",
+            "value": "INFO"
+          },
+          {
+            "selected": false,
+            "text": "ERROR",
+            "value": "ERROR"
+          },
+          {
+            "selected": false,
+            "text": "WARN",
+            "value": "WARN"
+          },
+          {
+            "selected": false,
+            "text": "DEBUG",
+            "value": "DEBUG"
+          },
+          {
+            "selected": false,
+            "text": "TRACE",
+            "value": "TRACE"
+          }
+        ],
+        "query": "INFO,ERROR,WARN,DEBUG,TRACE",
+        "type": "custom"
+      },
+      {
+        "allValue": "'all'",
+        "current": {
+          "text": [
+            "$__all"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "includeAll": true,
+        "label": "role",
+        "multi": true,
+        "name": "role",
+        "options": [
+          {
+            "selected": false,
+            "text": "datanode",
+            "value": "datanode"
+          },
+          {
+            "selected": false,
+            "text": "frontend",
+            "value": "frontend"
+          },
+          {
+            "selected": false,
+            "text": "meta",
+            "value": "meta"
+          }
+        ],
+        "query": "datanode,frontend,meta",
+        "type": "custom"
+      },
+      {
+        "current": {
+          "text": "",
+          "value": ""
+        },
+        "label": "pod",
+        "name": "pod",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "type": "textbox"
+      },
+      {
+        "current": {
+          "text": "",
+          "value": ""
+        },
+        "label": "target",
+        "name": "target",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "type": "textbox"
+      },
+      {
+        "current": {
+          "text": "",
+          "value": ""
+        },
+        "label": "search",
+        "name": "search",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "type": "textbox"
+      },
+      {
+        "current": {
+          "text": "",
+          "value": ""
+        },
+        "label": "exclude",
+        "name": "exclude",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "type": "textbox"
+      },
+      {
+        "current": {
+          "text": "2000",
+          "value": "2000"
+        },
+        "includeAll": false,
+        "label": "limit",
+        "name": "limit",
+        "options": [
+          {
+            "selected": true,
+            "text": "2000",
+            "value": "2000"
+          },
+          {
+            "selected": false,
+            "text": "5000",
+            "value": "5000"
+          },
+          {
+            "selected": false,
+            "text": "8000",
+            "value": "8000"
+          }
+        ],
+        "query": "2000,5000,8000",
+        "type": "custom"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "GreptimeDB Logs",
+  "uid": "edx5veo4rd3wge2",
+  "version": 1
+}

grafana/scripts/check.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-DASHBOARD_DIR=${1:-grafana/dashboards}
+DASHBOARD_DIR=${1:-grafana/dashboards/metrics}
 
 check_dashboard_description() {
   for dashboard in $(find $DASHBOARD_DIR -name "*.json"); do
@@ -25,7 +25,7 @@ check_dashboard_description() {
 check_dashboards_generation() {
   ./grafana/scripts/gen-dashboards.sh
 
-  if [[ -n "$(git diff --name-only grafana/dashboards)" ]]; then
+  if [[ -n "$(git diff --name-only grafana/dashboards/metrics)" ]]; then
     echo "Error: The dashboards are not generated correctly. You should execute the `make dashboards` command."
     exit 1
   fi

grafana/scripts/gen-dashboards.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
-CLUSTER_DASHBOARD_DIR=${1:-grafana/dashboards/cluster}
-STANDALONE_DASHBOARD_DIR=${2:-grafana/dashboards/standalone}
+CLUSTER_DASHBOARD_DIR=${1:-grafana/dashboards/metrics/cluster}
+STANDALONE_DASHBOARD_DIR=${2:-grafana/dashboards/metrics/standalone}
 DAC_IMAGE=ghcr.io/zyy17/dac:20250423-522bd35
 
 remove_instance_filters() {

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2024-12-25"
+channel = "nightly-2025-04-15"

src/api/src/helper.rs

@@ -1050,7 +1050,7 @@ pub fn value_to_grpc_value(value: Value) -> GrpcValue {
             Value::Int64(v) => Some(ValueData::I64Value(v)),
             Value::Float32(v) => Some(ValueData::F32Value(*v)),
             Value::Float64(v) => Some(ValueData::F64Value(*v)),
-            Value::String(v) => Some(ValueData::StringValue(v.as_utf8().to_string())),
+            Value::String(v) => Some(ValueData::StringValue(v.into_string())),
             Value::Binary(v) => Some(ValueData::BinaryValue(v.to_vec())),
             Value::Date(v) => Some(ValueData::DateValue(v.val())),
             Value::Timestamp(v) => Some(match v.unit() {

src/auth/src/common.rs

@@ -36,7 +36,7 @@ pub fn userinfo_by_name(username: Option<String>) -> UserInfoRef {
 }
 
 pub fn user_provider_from_option(opt: &String) -> Result<UserProviderRef> {
-    let (name, content) = opt.split_once(':').context(InvalidConfigSnafu {
+    let (name, content) = opt.split_once(':').with_context(|| InvalidConfigSnafu {
         value: opt.to_string(),
         msg: "UserProviderOption must be in format `<option>:<value>`",
     })?;
@@ -57,6 +57,24 @@ pub fn user_provider_from_option(opt: &String) -> Result<UserProviderRef> {
     }
 }
 
+pub fn static_user_provider_from_option(opt: &String) -> Result<StaticUserProvider> {
+    let (name, content) = opt.split_once(':').with_context(|| InvalidConfigSnafu {
+        value: opt.to_string(),
+        msg: "UserProviderOption must be in format `<option>:<value>`",
+    })?;
+    match name {
+        STATIC_USER_PROVIDER => {
+            let provider = StaticUserProvider::new(content)?;
+            Ok(provider)
+        }
+        _ => InvalidConfigSnafu {
+            value: name.to_string(),
+            msg: format!("Invalid UserProviderOption, expect only {STATIC_USER_PROVIDER}"),
+        }
+        .fail(),
+    }
+}
+
 type Username<'a> = &'a str;
 type HostOrIp<'a> = &'a str;
 

src/auth/src/error.rs

@@ -38,6 +38,14 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to convert to utf8"))]
+    FromUtf8 {
+        #[snafu(source)]
+        error: std::string::FromUtf8Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Authentication source failure"))]
     AuthBackend {
         #[snafu(implicit)]
@@ -85,7 +93,7 @@ impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
             Error::InvalidConfig { .. } => StatusCode::InvalidArguments,
-            Error::IllegalParam { .. } => StatusCode::InvalidArguments,
+            Error::IllegalParam { .. } | Error::FromUtf8 { .. } => StatusCode::InvalidArguments,
             Error::FileWatch { .. } => StatusCode::InvalidArguments,
             Error::InternalState { .. } => StatusCode::Unexpected,
             Error::Io { .. } => StatusCode::StorageUnavailable,

src/auth/src/lib.rs

@@ -22,10 +22,12 @@ mod user_provider;
 pub mod tests;
 
 pub use common::{
-    auth_mysql, user_provider_from_option, userinfo_by_name, HashedPassword, Identity, Password,
+    auth_mysql, static_user_provider_from_option, user_provider_from_option, userinfo_by_name,
+    HashedPassword, Identity, Password,
 };
 pub use permission::{PermissionChecker, PermissionReq, PermissionResp};
 pub use user_info::UserInfo;
+pub use user_provider::static_user_provider::StaticUserProvider;
 pub use user_provider::UserProvider;
 
 /// pub type alias

src/auth/src/user_provider/static_user_provider.rs

@@ -15,15 +15,15 @@
 use std::collections::HashMap;
 
 use async_trait::async_trait;
-use snafu::OptionExt;
+use snafu::{OptionExt, ResultExt};
 
-use crate::error::{InvalidConfigSnafu, Result};
+use crate::error::{FromUtf8Snafu, InvalidConfigSnafu, Result};
 use crate::user_provider::{authenticate_with_credential, load_credential_from_file};
 use crate::{Identity, Password, UserInfoRef, UserProvider};
 
 pub(crate) const STATIC_USER_PROVIDER: &str = "static_user_provider";
 
-pub(crate) struct StaticUserProvider {
+pub struct StaticUserProvider {
     users: HashMap<String, Vec<u8>>,
 }
 
@@ -60,6 +60,18 @@ impl StaticUserProvider {
                 .fail(),
         }
     }
+
+    /// Return a random username/password pair
+    /// This is useful for invoking from other components in the cluster
+    pub fn get_one_user_pwd(&self) -> Result<(String, String)> {
+        let kv = self.users.iter().next().context(InvalidConfigSnafu {
+            value: "",
+            msg: "Expect at least one pair of username and password",
+        })?;
+        let username = kv.0;
+        let pwd = String::from_utf8(kv.1.clone()).context(FromUtf8Snafu)?;
+        Ok((username.clone(), pwd))
+    }
 }
 
 #[async_trait]

src/catalog/src/system_schema/pg_catalog/pg_namespace/oid_map.rs

@@ -84,12 +84,6 @@ mod tests {
         let key1 = "3178510";
         let key2 = "4215648";
 
-        // have collision
-        assert_eq!(
-            oid_map.hasher.hash_one(key1) as u32,
-            oid_map.hasher.hash_one(key2) as u32
-        );
-
         // insert them into oid_map
         let oid1 = oid_map.get_oid(key1);
         let oid2 = oid_map.get_oid(key2);

src/cli/Cargo.toml

@@ -51,7 +51,6 @@ opendal = { version = "0.51.1", features = [
 query.workspace = true
 rand.workspace = true
 reqwest.workspace = true
-rustyline = "10.1"
 serde.workspace = true
 serde_json.workspace = true
 servers.workspace = true

src/cli/src/cmd.rs

@@ -1,154 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::error::{Error, InvalidReplCommandSnafu, Result};
-
-/// Represents the parsed command from the user (which may be over many lines)
-#[derive(Debug, PartialEq)]
-pub(crate) enum ReplCommand {
-    Help,
-    UseDatabase { db_name: String },
-    Sql { sql: String },
-    Exit,
-}
-
-impl TryFrom<&str> for ReplCommand {
-    type Error = Error;
-
-    fn try_from(input: &str) -> Result<Self> {
-        let input = input.trim();
-        if input.is_empty() {
-            return InvalidReplCommandSnafu {
-                reason: "No command specified".to_string(),
-            }
-            .fail();
-        }
-
-        // If line ends with ';', it must be treated as a complete input.
-        // However, the opposite is not true.
-        let input_is_completed = input.ends_with(';');
-
-        let input = input.strip_suffix(';').map(|x| x.trim()).unwrap_or(input);
-        let lowercase = input.to_lowercase();
-        match lowercase.as_str() {
-            "help" => Ok(Self::Help),
-            "exit" | "quit" => Ok(Self::Exit),
-            _ => match input.split_once(' ') {
-                Some((maybe_use, database)) if maybe_use.to_lowercase() == "use" => {
-                    Ok(Self::UseDatabase {
-                        db_name: database.trim().to_string(),
-                    })
-                }
-                // Any valid SQL must contains at least one whitespace.
-                Some(_) if input_is_completed => Ok(Self::Sql {
-                    sql: input.to_string(),
-                }),
-                _ => InvalidReplCommandSnafu {
-                    reason: format!("unknown command '{input}', maybe input is not completed"),
-                }
-                .fail(),
-            },
-        }
-    }
-}
-
-impl ReplCommand {
-    pub fn help() -> &'static str {
-        r#"
-Available commands (case insensitive):
-- 'help': print this help
-- 'exit' or 'quit': exit the REPL
-- 'use <your database name>': switch to another database/schema context
-- Other typed in text will be treated as SQL.
-  You can enter new line while typing, just remember to end it with ';'.
-"#
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::error::Error::InvalidReplCommand;
-
-    #[test]
-    fn test_from_str() {
-        fn test_ok(s: &str, expected: ReplCommand) {
-            let actual: ReplCommand = s.try_into().unwrap();
-            assert_eq!(expected, actual, "'{}'", s);
-        }
-
-        fn test_err(s: &str) {
-            let result: Result<ReplCommand> = s.try_into();
-            assert!(matches!(result, Err(InvalidReplCommand { .. })))
-        }
-
-        test_err("");
-        test_err("  ");
-        test_err("\t");
-
-        test_ok("help", ReplCommand::Help);
-        test_ok("help", ReplCommand::Help);
-        test_ok("  help", ReplCommand::Help);
-        test_ok("  help  ", ReplCommand::Help);
-        test_ok("  HELP  ", ReplCommand::Help);
-        test_ok("  Help;  ", ReplCommand::Help);
-        test_ok("  help  ; ", ReplCommand::Help);
-
-        test_ok("exit", ReplCommand::Exit);
-        test_ok("exit;", ReplCommand::Exit);
-        test_ok("exit ;", ReplCommand::Exit);
-        test_ok("EXIT", ReplCommand::Exit);
-
-        test_ok("quit", ReplCommand::Exit);
-        test_ok("quit;", ReplCommand::Exit);
-        test_ok("quit ;", ReplCommand::Exit);
-        test_ok("QUIT", ReplCommand::Exit);
-
-        test_ok(
-            "use Foo",
-            ReplCommand::UseDatabase {
-                db_name: "Foo".to_string(),
-            },
-        );
-        test_ok(
-            "  use Foo ;  ",
-            ReplCommand::UseDatabase {
-                db_name: "Foo".to_string(),
-            },
-        );
-        // ensure that database name is case sensitive
-        test_ok(
-            "  use FOO ;  ",
-            ReplCommand::UseDatabase {
-                db_name: "FOO".to_string(),
-            },
-        );
-
-        // ensure that we aren't messing with capitalization
-        test_ok(
-            "SELECT * from foo;",
-            ReplCommand::Sql {
-                sql: "SELECT * from foo".to_string(),
-            },
-        );
-        // Input line (that don't belong to any other cases above) must ends with ';' to make it a valid SQL.
-        test_err("insert blah");
-        test_ok(
-            "insert blah;",
-            ReplCommand::Sql {
-                sql: "insert blah".to_string(),
-            },
-        );
-    }
-}

src/cli/src/error.rs

@@ -101,9 +101,6 @@ pub enum Error {
         error: reqwest::Error,
     },
 
-    #[snafu(display("Invalid REPL command: {reason}"))]
-    InvalidReplCommand { reason: String },
-
     #[snafu(display("Failed to parse SQL: {}", sql))]
     ParseSql {
         sql: String,
@@ -254,7 +251,6 @@ impl ErrorExt for Error {
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
-            | Error::InvalidReplCommand { .. }
             | Error::InitTimezone { .. }
             | Error::ConnectEtcd { .. }
             | Error::CreateDir { .. }

src/cli/src/helper.rs

@@ -1,112 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::borrow::Cow;
-
-use rustyline::completion::Completer;
-use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
-use rustyline::hint::{Hinter, HistoryHinter};
-use rustyline::validate::{ValidationContext, ValidationResult, Validator};
-
-use crate::cmd::ReplCommand;
-
-pub(crate) struct RustylineHelper {
-    hinter: HistoryHinter,
-    highlighter: MatchingBracketHighlighter,
-}
-
-impl Default for RustylineHelper {
-    fn default() -> Self {
-        Self {
-            hinter: HistoryHinter {},
-            highlighter: MatchingBracketHighlighter::default(),
-        }
-    }
-}
-
-impl rustyline::Helper for RustylineHelper {}
-
-impl Validator for RustylineHelper {
-    fn validate(&self, ctx: &mut ValidationContext<'_>) -> rustyline::Result<ValidationResult> {
-        let input = ctx.input();
-        match ReplCommand::try_from(input) {
-            Ok(_) => Ok(ValidationResult::Valid(None)),
-            Err(e) => {
-                if input.trim_end().ends_with(';') {
-                    // If line ends with ';', it HAS to be a valid command.
-                    Ok(ValidationResult::Invalid(Some(e.to_string())))
-                } else {
-                    Ok(ValidationResult::Incomplete)
-                }
-            }
-        }
-    }
-}
-
-impl Hinter for RustylineHelper {
-    type Hint = String;
-
-    fn hint(&self, line: &str, pos: usize, ctx: &rustyline::Context<'_>) -> Option<Self::Hint> {
-        self.hinter.hint(line, pos, ctx)
-    }
-}
-
-impl Highlighter for RustylineHelper {
-    fn highlight<'l>(&self, line: &'l str, pos: usize) -> Cow<'l, str> {
-        self.highlighter.highlight(line, pos)
-    }
-
-    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(
-        &'s self,
-        prompt: &'p str,
-        default: bool,
-    ) -> Cow<'b, str> {
-        self.highlighter.highlight_prompt(prompt, default)
-    }
-
-    fn highlight_hint<'h>(&self, hint: &'h str) -> Cow<'h, str> {
-        use nu_ansi_term::Style;
-        Cow::Owned(Style::new().dimmed().paint(hint).to_string())
-    }
-
-    fn highlight_candidate<'c>(
-        &self,
-        candidate: &'c str,
-        completion: rustyline::CompletionType,
-    ) -> Cow<'c, str> {
-        self.highlighter.highlight_candidate(candidate, completion)
-    }
-
-    fn highlight_char(&self, line: &str, pos: usize) -> bool {
-        self.highlighter.highlight_char(line, pos)
-    }
-}
-
-impl Completer for RustylineHelper {
-    type Candidate = String;
-
-    fn complete(
-        &self,
-        line: &str,
-        pos: usize,
-        ctx: &rustyline::Context<'_>,
-    ) -> rustyline::Result<(usize, Vec<Self::Candidate>)> {
-        // If there is a hint, use that as the auto-complete when user hits `tab`
-        if let Some(hint) = self.hinter.hint(line, pos, ctx) {
-            Ok((pos, vec![hint]))
-        } else {
-            Ok((0, vec![]))
-        }
-    }
-}

src/cli/src/lib.rs

@@ -13,15 +13,9 @@
 // limitations under the License.
 
 mod bench;
-pub mod error;
-// Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373
-#[allow(unused)]
-mod cmd;
-mod export;
-mod helper;
-
-// Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373
 mod database;
+pub mod error;
+mod export;
 mod import;
 
 use async_trait::async_trait;

src/client/src/database.rs

@@ -42,11 +42,11 @@ use futures::future;
 use futures_util::{Stream, StreamExt, TryStreamExt};
 use prost::Message;
 use snafu::{ensure, ResultExt};
-use tonic::metadata::{AsciiMetadataKey, MetadataValue};
+use tonic::metadata::{AsciiMetadataKey, AsciiMetadataValue, MetadataMap, MetadataValue};
 use tonic::transport::Channel;
 
 use crate::error::{
-    ConvertFlightDataSnafu, Error, FlightGetSnafu, IllegalFlightMessagesSnafu, InvalidAsciiSnafu,
+    ConvertFlightDataSnafu, Error, FlightGetSnafu, IllegalFlightMessagesSnafu,
     InvalidTonicMetadataValueSnafu, ServerSnafu,
 };
 use crate::{from_grpc_response, Client, Result};
@@ -165,26 +165,27 @@ impl Database {
 
         let mut request = tonic::Request::new(request);
         let metadata = request.metadata_mut();
-        for (key, value) in hints {
-            let key = AsciiMetadataKey::from_bytes(format!("x-greptime-hint-{}", key).as_bytes())
-                .map_err(|_| {
-                InvalidAsciiSnafu {
-                    value: key.to_string(),
-                }
-                .build()
-            })?;
-            let value = value.parse().map_err(|_| {
-                InvalidAsciiSnafu {
-                    value: value.to_string(),
-                }
-                .build()
-            })?;
-            metadata.insert(key, value);
-        }
+        Self::put_hints(metadata, hints)?;
+
         let response = client.handle(request).await?.into_inner();
         from_grpc_response(response)
     }
 
+    fn put_hints(metadata: &mut MetadataMap, hints: &[(&str, &str)]) -> Result<()> {
+        let Some(value) = hints
+            .iter()
+            .map(|(k, v)| format!("{}={}", k, v))
+            .reduce(|a, b| format!("{},{}", a, b))
+        else {
+            return Ok(());
+        };
+
+        let key = AsciiMetadataKey::from_static("x-greptime-hints");
+        let value = AsciiMetadataValue::from_str(&value).context(InvalidTonicMetadataValueSnafu)?;
+        metadata.insert(key, value);
+        Ok(())
+    }
+
     pub async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = make_database_client(&self.client)?.inner;
         let request = self.to_rpc_request(request);
@@ -242,39 +243,49 @@ impl Database {
     where
         S: AsRef<str>,
     {
-        self.do_get(Request::Query(QueryRequest {
+        self.sql_with_hint(sql, &[]).await
+    }
+
+    pub async fn sql_with_hint<S>(&self, sql: S, hints: &[(&str, &str)]) -> Result<Output>
+    where
+        S: AsRef<str>,
+    {
+        let request = Request::Query(QueryRequest {
             query: Some(Query::Sql(sql.as_ref().to_string())),
-        }))
-        .await
+        });
+        self.do_get(request, hints).await
     }
 
     pub async fn logical_plan(&self, logical_plan: Vec<u8>) -> Result<Output> {
-        self.do_get(Request::Query(QueryRequest {
+        let request = Request::Query(QueryRequest {
             query: Some(Query::LogicalPlan(logical_plan)),
-        }))
-        .await
+        });
+        self.do_get(request, &[]).await
     }
 
     pub async fn create(&self, expr: CreateTableExpr) -> Result<Output> {
-        self.do_get(Request::Ddl(DdlRequest {
+        let request = Request::Ddl(DdlRequest {
             expr: Some(DdlExpr::CreateTable(expr)),
-        }))
-        .await
+        });
+        self.do_get(request, &[]).await
     }
 
     pub async fn alter(&self, expr: AlterTableExpr) -> Result<Output> {
-        self.do_get(Request::Ddl(DdlRequest {
+        let request = Request::Ddl(DdlRequest {
             expr: Some(DdlExpr::AlterTable(expr)),
-        }))
-        .await
+        });
+        self.do_get(request, &[]).await
     }
 
-    async fn do_get(&self, request: Request) -> Result<Output> {
+    async fn do_get(&self, request: Request, hints: &[(&str, &str)]) -> Result<Output> {
         let request = self.to_rpc_request(request);
         let request = Ticket {
             ticket: request.encode_to_vec().into(),
         };
 
+        let mut request = tonic::Request::new(request);
+        Self::put_hints(request.metadata_mut(), hints)?;
+
         let mut client = self.client.make_flight_client()?;
 
         let response = client.mut_inner().do_get(request).await.or_else(|e| {

src/client/src/error.rs

@@ -110,13 +110,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to parse ascii string: {}", value))]
-    InvalidAscii {
-        value: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Invalid Tonic metadata value"))]
     InvalidTonicMetadataValue {
         #[snafu(source)]
@@ -143,10 +136,7 @@ impl ErrorExt for Error {
             | Error::ConvertFlightData { source, .. }
             | Error::CreateTlsChannel { source, .. } => source.status_code(),
             Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected,
-
-            Error::InvalidAscii { .. } | Error::InvalidTonicMetadataValue { .. } => {
-                StatusCode::InvalidArguments
-            }
+            Error::InvalidTonicMetadataValue { .. } => StatusCode::InvalidArguments,
         }
     }
 

src/cmd/src/bin/greptime.rs

@@ -15,9 +15,11 @@
 #![doc = include_str!("../../../../README.md")]
 
 use clap::{Parser, Subcommand};
+use cmd::datanode::builder::InstanceBuilder;
 use cmd::error::{InitTlsProviderSnafu, Result};
 use cmd::options::GlobalOptions;
 use cmd::{cli, datanode, flownode, frontend, metasrv, standalone, App};
+use common_base::Plugins;
 use common_version::version;
 use servers::install_ring_crypto_provider;
 
@@ -102,10 +104,10 @@ async fn main_body() -> Result<()> {
 async fn start(cli: Command) -> Result<()> {
     match cli.subcmd {
         SubCommand::Datanode(cmd) => {
-            cmd.build(cmd.load_options(&cli.global_options)?)
-                .await?
-                .run()
-                .await
+            let opts = cmd.load_options(&cli.global_options)?;
+            let plugins = Plugins::new();
+            let builder = InstanceBuilder::try_new_with_init(opts, plugins).await?;
+            cmd.build_with(builder).await?.run().await
         }
         SubCommand::Flownode(cmd) => {
             cmd.build(cmd.load_options(&cli.global_options)?)

src/cmd/src/cli.rs

@@ -58,7 +58,7 @@ impl App for Instance {
         false
     }
 
-    async fn stop(&self) -> Result<()> {
+    async fn stop(&mut self) -> Result<()> {
         Ok(())
     }
 }

src/cmd/src/datanode.rs

@@ -12,33 +12,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+pub mod builder;
+
 use std::time::Duration;
 
 use async_trait::async_trait;
-use cache::build_datanode_cache_registry;
-use catalog::kvbackend::MetaKvBackend;
 use clap::Parser;
-use common_base::Plugins;
 use common_config::Configurable;
-use common_meta::cache::LayeredCacheRegistryBuilder;
 use common_telemetry::logging::TracingOptions;
 use common_telemetry::{info, warn};
-use common_version::{short_version, version};
 use common_wal::config::DatanodeWalConfig;
-use datanode::datanode::{Datanode, DatanodeBuilder};
-use datanode::service::DatanodeServiceBuilder;
-use meta_client::{MetaClientOptions, MetaClientType};
-use servers::Mode;
-use snafu::{ensure, OptionExt, ResultExt};
+use datanode::datanode::Datanode;
+use meta_client::MetaClientOptions;
+use snafu::{ensure, ResultExt};
 use tracing_appender::non_blocking::WorkerGuard;
 
+use crate::datanode::builder::InstanceBuilder;
 use crate::error::{
-    LoadLayeredConfigSnafu, MetaClientInitSnafu, MissingConfigSnafu, Result, ShutdownDatanodeSnafu,
-    StartDatanodeSnafu,
+    LoadLayeredConfigSnafu, MissingConfigSnafu, Result, ShutdownDatanodeSnafu, StartDatanodeSnafu,
 };
 use crate::options::{GlobalOptions, GreptimeOptions};
-use crate::{log_versions, App};
+use crate::App;
 
 pub const APP_NAME: &str = "greptime-datanode";
 
@@ -83,7 +77,7 @@ impl App for Instance {
         self.datanode.start().await.context(StartDatanodeSnafu)
     }
 
-    async fn stop(&self) -> Result<()> {
+    async fn stop(&mut self) -> Result<()> {
         self.datanode
             .shutdown()
             .await
@@ -98,8 +92,8 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(&self, opts: DatanodeOptions) -> Result<Instance> {
-        self.subcmd.build(opts).await
+    pub async fn build_with(&self, builder: InstanceBuilder) -> Result<Instance> {
+        self.subcmd.build_with(builder).await
     }
 
     pub fn load_options(&self, global_options: &GlobalOptions) -> Result<DatanodeOptions> {
@@ -115,9 +109,12 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(&self, opts: DatanodeOptions) -> Result<Instance> {
+    async fn build_with(&self, builder: InstanceBuilder) -> Result<Instance> {
         match self {
-            SubCommand::Start(cmd) => cmd.build(opts).await,
+            SubCommand::Start(cmd) => {
+                info!("Building datanode with {:#?}", cmd);
+                builder.build().await
+            }
         }
     }
 }
@@ -263,74 +260,6 @@ impl StartCommand {
 
         Ok(())
     }
-
-    async fn build(&self, opts: DatanodeOptions) -> Result<Instance> {
-        common_runtime::init_global_runtimes(&opts.runtime);
-
-        let guard = common_telemetry::init_global_logging(
-            APP_NAME,
-            &opts.component.logging,
-            &opts.component.tracing,
-            opts.component.node_id.map(|x| x.to_string()),
-        );
-        log_versions(version(), short_version(), APP_NAME);
-
-        info!("Datanode start command: {:#?}", self);
-        info!("Datanode options: {:#?}", opts);
-
-        let plugin_opts = opts.plugins;
-        let mut opts = opts.component;
-        opts.grpc.detect_server_addr();
-        let mut plugins = Plugins::new();
-        plugins::setup_datanode_plugins(&mut plugins, &plugin_opts, &opts)
-            .await
-            .context(StartDatanodeSnafu)?;
-
-        let member_id = opts
-            .node_id
-            .context(MissingConfigSnafu { msg: "'node_id'" })?;
-
-        let meta_config = opts.meta_client.as_ref().context(MissingConfigSnafu {
-            msg: "'meta_client_options'",
-        })?;
-
-        let meta_client = meta_client::create_meta_client(
-            MetaClientType::Datanode { member_id },
-            meta_config,
-            None,
-        )
-        .await
-        .context(MetaClientInitSnafu)?;
-
-        let meta_backend = Arc::new(MetaKvBackend {
-            client: meta_client.clone(),
-        });
-
-        // Builds cache registry for datanode.
-        let layered_cache_registry = Arc::new(
-            LayeredCacheRegistryBuilder::default()
-                .add_cache_registry(build_datanode_cache_registry(meta_backend.clone()))
-                .build(),
-        );
-
-        let mut datanode = DatanodeBuilder::new(opts.clone(), plugins, Mode::Distributed)
-            .with_meta_client(meta_client)
-            .with_kv_backend(meta_backend)
-            .with_cache_registry(layered_cache_registry)
-            .build()
-            .await
-            .context(StartDatanodeSnafu)?;
-
-        let services = DatanodeServiceBuilder::new(&opts)
-            .with_default_grpc_server(&datanode.region_server())
-            .enable_http_service()
-            .build()
-            .await
-            .context(StartDatanodeSnafu)?;
-        datanode.setup_services(services);
-
-        Ok(Instance::new(datanode, guard))
-    }
 }
 
 #[cfg(test)]
@@ -352,7 +281,6 @@ mod tests {
         common_telemetry::init_default_ut_logging();
         let mut file = create_named_temp_file();
         let toml_str = r#"
-            mode = "distributed"
             enable_memory_catalog = false
             node_id = 42
 
@@ -379,7 +307,6 @@ mod tests {
     fn test_read_from_config_file() {
         let mut file = create_named_temp_file();
         let toml_str = r#"
-            mode = "distributed"
             enable_memory_catalog = false
             node_id = 42
 
@@ -545,7 +472,6 @@ mod tests {
     fn test_config_precedence_order() {
         let mut file = create_named_temp_file();
         let toml_str = r#"
-            mode = "distributed"
             enable_memory_catalog = false
             node_id = 42
             rpc_addr = "127.0.0.1:3001"

src/cmd/src/datanode/builder.rs

@@ -0,0 +1,137 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use cache::build_datanode_cache_registry;
+use catalog::kvbackend::MetaKvBackend;
+use common_base::Plugins;
+use common_meta::cache::LayeredCacheRegistryBuilder;
+use common_telemetry::info;
+use common_version::{short_version, version};
+use datanode::datanode::DatanodeBuilder;
+use datanode::service::DatanodeServiceBuilder;
+use meta_client::MetaClientType;
+use snafu::{OptionExt, ResultExt};
+use tracing_appender::non_blocking::WorkerGuard;
+
+use crate::datanode::{DatanodeOptions, Instance, APP_NAME};
+use crate::error::{MetaClientInitSnafu, MissingConfigSnafu, Result, StartDatanodeSnafu};
+use crate::log_versions;
+
+/// Builder for Datanode instance.
+pub struct InstanceBuilder {
+    guard: Vec<WorkerGuard>,
+    opts: DatanodeOptions,
+    datanode_builder: DatanodeBuilder,
+}
+
+impl InstanceBuilder {
+    /// Try to create a new [InstanceBuilder], and do some initialization work like allocating
+    /// runtime resources, setting up global logging and plugins, etc.
+    pub async fn try_new_with_init(
+        mut opts: DatanodeOptions,
+        mut plugins: Plugins,
+    ) -> Result<Self> {
+        let guard = Self::init(&mut opts, &mut plugins).await?;
+
+        let datanode_builder = Self::datanode_builder(&opts, plugins).await?;
+
+        Ok(Self {
+            guard,
+            opts,
+            datanode_builder,
+        })
+    }
+
+    async fn init(opts: &mut DatanodeOptions, plugins: &mut Plugins) -> Result<Vec<WorkerGuard>> {
+        common_runtime::init_global_runtimes(&opts.runtime);
+
+        let dn_opts = &mut opts.component;
+        let guard = common_telemetry::init_global_logging(
+            APP_NAME,
+            &dn_opts.logging,
+            &dn_opts.tracing,
+            dn_opts.node_id.map(|x| x.to_string()),
+        );
+
+        log_versions(version(), short_version(), APP_NAME);
+
+        plugins::setup_datanode_plugins(plugins, &opts.plugins, dn_opts)
+            .await
+            .context(StartDatanodeSnafu)?;
+
+        dn_opts.grpc.detect_server_addr();
+
+        info!("Initialized Datanode instance with {:#?}", opts);
+        Ok(guard)
+    }
+
+    async fn datanode_builder(opts: &DatanodeOptions, plugins: Plugins) -> Result<DatanodeBuilder> {
+        let dn_opts = &opts.component;
+
+        let member_id = dn_opts
+            .node_id
+            .context(MissingConfigSnafu { msg: "'node_id'" })?;
+        let meta_client_options = dn_opts.meta_client.as_ref().context(MissingConfigSnafu {
+            msg: "meta client options",
+        })?;
+        let client = meta_client::create_meta_client(
+            MetaClientType::Datanode { member_id },
+            meta_client_options,
+            Some(&plugins),
+        )
+        .await
+        .context(MetaClientInitSnafu)?;
+
+        let backend = Arc::new(MetaKvBackend {
+            client: client.clone(),
+        });
+        let mut builder = DatanodeBuilder::new(dn_opts.clone(), plugins.clone(), backend.clone());
+
+        let registry = Arc::new(
+            LayeredCacheRegistryBuilder::default()
+                .add_cache_registry(build_datanode_cache_registry(backend))
+                .build(),
+        );
+        builder
+            .with_cache_registry(registry)
+            .with_meta_client(client.clone());
+        Ok(builder)
+    }
+
+    /// Get the mutable builder for Datanode, in case you want to change some fields before the
+    /// final construction.
+    pub fn mut_datanode_builder(&mut self) -> &mut DatanodeBuilder {
+        &mut self.datanode_builder
+    }
+
+    /// Try to build the Datanode instance.
+    pub async fn build(self) -> Result<Instance> {
+        let mut datanode = self
+            .datanode_builder
+            .build()
+            .await
+            .context(StartDatanodeSnafu)?;
+
+        let services = DatanodeServiceBuilder::new(&self.opts.component)
+            .with_default_grpc_server(&datanode.region_server())
+            .enable_http_service()
+            .build()
+            .context(StartDatanodeSnafu)?;
+        datanode.setup_services(services);
+
+        Ok(Instance::new(datanode, self.guard))
+    }
+}

src/cmd/src/error.rs

@@ -177,9 +177,6 @@ pub enum Error {
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Invalid REPL command: {reason}"))]
-    InvalidReplCommand { reason: String },
-
     #[snafu(display("Failed to parse SQL: {}", sql))]
     ParseSql {
         sql: String,
@@ -331,7 +328,6 @@ impl ErrorExt for Error {
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
-            | Error::InvalidReplCommand { .. }
             | Error::InitTimezone { .. }
             | Error::ConnectEtcd { .. }
             | Error::CreateDir { .. }

src/cmd/src/flownode.rs

@@ -33,7 +33,8 @@ use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
 use common_version::{short_version, version};
 use flow::{
-    FlownodeBuilder, FlownodeInstance, FlownodeServiceBuilder, FrontendClient, FrontendInvoker,
+    get_flow_auth_options, FlownodeBuilder, FlownodeInstance, FlownodeServiceBuilder,
+    FrontendClient, FrontendInvoker,
 };
 use meta_client::{MetaClientOptions, MetaClientType};
 use snafu::{ensure, OptionExt, ResultExt};
@@ -82,10 +83,14 @@ impl App for Instance {
     }
 
     async fn start(&mut self) -> Result<()> {
+        plugins::start_flownode_plugins(self.flownode.flow_engine().plugins().clone())
+            .await
+            .context(StartFlownodeSnafu)?;
+
         self.flownode.start().await.context(StartFlownodeSnafu)
     }
 
-    async fn stop(&self) -> Result<()> {
+    async fn stop(&mut self) -> Result<()> {
         self.flownode
             .shutdown()
             .await
@@ -151,6 +156,9 @@ struct StartCommand {
     /// HTTP request timeout in seconds.
     #[clap(long)]
     http_timeout: Option<u64>,
+    /// User Provider cfg, for auth, currently only support static user provider
+    #[clap(long)]
+    user_provider: Option<String>,
 }
 
 impl StartCommand {
@@ -214,6 +222,10 @@ impl StartCommand {
             opts.http.timeout = Duration::from_secs(http_timeout);
         }
 
+        if let Some(user_provider) = &self.user_provider {
+            opts.user_provider = Some(user_provider.clone());
+        }
+
         ensure!(
             opts.node_id.is_some(),
             MissingConfigSnafu {
@@ -238,9 +250,15 @@ impl StartCommand {
         info!("Flownode start command: {:#?}", self);
         info!("Flownode options: {:#?}", opts);
 
+        let plugin_opts = opts.plugins;
         let mut opts = opts.component;
         opts.grpc.detect_server_addr();
 
+        let mut plugins = Plugins::new();
+        plugins::setup_flownode_plugins(&mut plugins, &plugin_opts, &opts)
+            .await
+            .context(StartFlownodeSnafu)?;
+
         let member_id = opts
             .node_id
             .context(MissingConfigSnafu { msg: "'node_id'" })?;
@@ -315,10 +333,12 @@ impl StartCommand {
         );
 
         let flow_metadata_manager = Arc::new(FlowMetadataManager::new(cached_meta_backend.clone()));
-        let frontend_client = FrontendClient::from_meta_client(meta_client.clone());
+        let flow_auth_header = get_flow_auth_options(&opts).context(StartFlownodeSnafu)?;
+        let frontend_client =
+            FrontendClient::from_meta_client(meta_client.clone(), flow_auth_header);
         let flownode_builder = FlownodeBuilder::new(
             opts.clone(),
-            Plugins::new(),
+            plugins,
             table_metadata_manager,
             catalog_manager.clone(),
             flow_metadata_manager,
@@ -331,7 +351,6 @@ impl StartCommand {
             .with_grpc_server(flownode.flownode_server().clone())
             .enable_http_service()
             .build()
-            .await
             .context(StartFlownodeSnafu)?;
         flownode.setup_services(services);
         let flownode = flownode;

src/cmd/src/frontend.rs

@@ -89,7 +89,7 @@ impl App for Instance {
             .context(error::StartFrontendSnafu)
     }
 
-    async fn stop(&self) -> Result<()> {
+    async fn stop(&mut self) -> Result<()> {
         self.frontend
             .shutdown()
             .await
@@ -382,7 +382,6 @@ impl StartCommand {
 
         let servers = Services::new(opts, instance.clone(), plugins)
             .build()
-            .await
             .context(error::StartFrontendSnafu)?;
 
         let frontend = Frontend {
@@ -448,8 +447,6 @@ mod tests {
     fn test_read_from_config_file() {
         let mut file = create_named_temp_file();
         let toml_str = r#"
-            mode = "distributed"
-
             [http]
             addr = "127.0.0.1:4000"
             timeout = "0s"
@@ -538,8 +535,6 @@ mod tests {
     fn test_config_precedence_order() {
         let mut file = create_named_temp_file();
         let toml_str = r#"
-            mode = "distributed"
-
             [http]
             addr = "127.0.0.1:4000"
 

src/cmd/src/lib.rs

@@ -74,7 +74,7 @@ pub trait App: Send {
         true
     }
 
-    async fn stop(&self) -> Result<()>;
+    async fn stop(&mut self) -> Result<()>;
 
     async fn run(&mut self) -> Result<()> {
         info!("Starting app: {}", self.name());

src/cmd/src/metasrv.rs

@@ -69,7 +69,7 @@ impl App for Instance {
         self.instance.start().await.context(StartMetaServerSnafu)
     }
 
-    async fn stop(&self) -> Result<()> {
+    async fn stop(&mut self) -> Result<()> {
         self.instance
             .shutdown()
             .await

src/cmd/src/standalone.rs

@@ -75,7 +75,6 @@ use servers::export_metrics::{ExportMetricsOption, ExportMetricsTask};
 use servers::grpc::GrpcOptions;
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
-use servers::Mode;
 use snafu::ResultExt;
 use tokio::sync::RwLock;
 use tracing_appender::non_blocking::WorkerGuard;
@@ -256,8 +255,8 @@ pub struct Instance {
 
 impl Instance {
     /// Find the socket addr of a server by its `name`.
-    pub async fn server_addr(&self, name: &str) -> Option<SocketAddr> {
-        self.frontend.server_handlers().addr(name).await
+    pub fn server_addr(&self, name: &str) -> Option<SocketAddr> {
+        self.frontend.server_handlers().addr(name)
     }
 }
 
@@ -294,7 +293,7 @@ impl App for Instance {
         Ok(())
     }
 
-    async fn stop(&self) -> Result<()> {
+    async fn stop(&mut self) -> Result<()> {
         self.frontend
             .shutdown()
             .await
@@ -497,12 +496,9 @@ impl StartCommand {
             .build(),
         );
 
-        let datanode = DatanodeBuilder::new(dn_opts, plugins.clone(), Mode::Standalone)
-            .with_kv_backend(kv_backend.clone())
-            .with_cache_registry(layered_cache_registry.clone())
-            .build()
-            .await
-            .context(error::StartDatanodeSnafu)?;
+        let mut builder = DatanodeBuilder::new(dn_opts, plugins.clone(), kv_backend.clone());
+        builder.with_cache_registry(layered_cache_registry.clone());
+        let datanode = builder.build().await.context(error::StartDatanodeSnafu)?;
 
         let information_extension = Arc::new(StandaloneInformationExtension::new(
             datanode.region_server(),
@@ -634,7 +630,6 @@ impl StartCommand {
 
         let servers = Services::new(opts, fe_instance.clone(), plugins)
             .build()
-            .await
             .context(error::StartFrontendSnafu)?;
 
         let frontend = Frontend {
@@ -858,8 +853,6 @@ mod tests {
     fn test_read_from_config_file() {
         let mut file = create_named_temp_file();
         let toml_str = r#"
-            mode = "distributed"
-
             enable_memory_catalog = true
 
             [wal]
@@ -990,8 +983,6 @@ mod tests {
     fn test_config_precedence_order() {
         let mut file = create_named_temp_file();
         let toml_str = r#"
-            mode = "standalone"
-
             [http]
             addr = "127.0.0.1:4000"
 

src/common/config/src/config.rs

@@ -111,11 +111,9 @@ mod tests {
     use serde::{Deserialize, Serialize};
 
     use super::*;
-    use crate::Mode;
 
-    #[derive(Debug, Serialize, Deserialize)]
+    #[derive(Debug, Serialize, Deserialize, Default)]
     struct TestDatanodeConfig {
-        mode: Mode,
         node_id: Option<u64>,
         logging: LoggingOptions,
         meta_client: Option<MetaClientOptions>,
@@ -123,19 +121,6 @@ mod tests {
         storage: StorageConfig,
     }
 
-    impl Default for TestDatanodeConfig {
-        fn default() -> Self {
-            Self {
-                mode: Mode::Distributed,
-                node_id: None,
-                logging: LoggingOptions::default(),
-                meta_client: None,
-                wal: DatanodeWalConfig::default(),
-                storage: StorageConfig::default(),
-            }
-        }
-    }
-
     impl Configurable for TestDatanodeConfig {
         fn env_list_keys() -> Option<&'static [&'static str]> {
             Some(&["meta_client.metasrv_addrs"])
@@ -146,7 +131,6 @@ mod tests {
     fn test_load_layered_options() {
         let mut file = create_named_temp_file();
         let toml_str = r#"
-            mode = "distributed"
             enable_memory_catalog = false
             rpc_addr = "127.0.0.1:3001"
             rpc_hostname = "127.0.0.1"

src/common/config/src/lib.rs

@@ -26,16 +26,6 @@ pub fn metadata_store_dir(store_dir: &str) -> String {
     format!("{store_dir}/metadata")
 }
 
-/// The Server running mode
-#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Copy)]
-#[serde(rename_all = "lowercase")]
-pub enum Mode {
-    // The single process mode.
-    Standalone,
-    // The distributed cluster mode.
-    Distributed,
-}
-
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct KvBackendConfig {

src/common/function/Cargo.toml

@@ -13,7 +13,7 @@ default = ["geo"]
 geo = ["geohash", "h3o", "s2", "wkt", "geo-types", "dep:geo"]
 
 [dependencies]
-ahash = "0.8"
+ahash.workspace = true
 api.workspace = true
 arc-swap = "1.0"
 async-trait.workspace = true

src/common/function/src/aggr.rs

@@ -19,4 +19,4 @@ mod uddsketch_state;
 pub use geo_path::{GeoPathAccumulator, GEO_PATH_NAME};
 pub(crate) use hll::HllStateType;
 pub use hll::{HllState, HLL_MERGE_NAME, HLL_NAME};
-pub use uddsketch_state::{UddSketchState, UDDSKETCH_STATE_NAME};
+pub use uddsketch_state::{UddSketchState, UDDSKETCH_MERGE_NAME, UDDSKETCH_STATE_NAME};

src/common/function/src/aggr/uddsketch_state.rs

@@ -31,23 +31,28 @@ use datafusion::physical_plan::expressions::Literal;
 use datafusion::prelude::create_udaf;
 use datatypes::arrow::array::ArrayRef;
 use datatypes::arrow::datatypes::{DataType, Float64Type};
+use serde::{Deserialize, Serialize};
 use uddsketch::{SketchHashKey, UDDSketch};
 
 pub const UDDSKETCH_STATE_NAME: &str = "uddsketch_state";
 
-#[derive(Debug)]
+pub const UDDSKETCH_MERGE_NAME: &str = "uddsketch_merge";
+
+#[derive(Debug, Serialize, Deserialize)]
 pub struct UddSketchState {
     uddsketch: UDDSketch,
+    error_rate: f64,
 }
 
 impl UddSketchState {
     pub fn new(bucket_size: u64, error_rate: f64) -> Self {
         Self {
             uddsketch: UDDSketch::new(bucket_size, error_rate),
+            error_rate,
         }
     }
 
-    pub fn udf_impl() -> AggregateUDF {
+    pub fn state_udf_impl() -> AggregateUDF {
         create_udaf(
             UDDSKETCH_STATE_NAME,
             vec![DataType::Int64, DataType::Float64, DataType::Float64],
@@ -61,18 +66,55 @@ impl UddSketchState {
         )
     }
 
+    /// Create a UDF for the `uddsketch_merge` function.
+    ///
+    /// `uddsketch_merge` accepts bucket size, error rate, and a binary column of states generated by `uddsketch_state`
+    /// and merges them into a single state.
+    ///
+    /// The bucket size and error rate must be the same as the original state.
+    pub fn merge_udf_impl() -> AggregateUDF {
+        create_udaf(
+            UDDSKETCH_MERGE_NAME,
+            vec![DataType::Int64, DataType::Float64, DataType::Binary],
+            Arc::new(DataType::Binary),
+            Volatility::Immutable,
+            Arc::new(|args| {
+                let (bucket_size, error_rate) = downcast_accumulator_args(args)?;
+                Ok(Box::new(UddSketchState::new(bucket_size, error_rate)))
+            }),
+            Arc::new(vec![DataType::Binary]),
+        )
+    }
+
     fn update(&mut self, value: f64) {
         self.uddsketch.add_value(value);
     }
 
-    fn merge(&mut self, raw: &[u8]) {
-        if let Ok(uddsketch) = bincode::deserialize::<UDDSketch>(raw) {
-            if uddsketch.count() != 0 {
-                self.uddsketch.merge_sketch(&uddsketch);
+    fn merge(&mut self, raw: &[u8]) -> DfResult<()> {
+        if let Ok(uddsketch) = bincode::deserialize::<Self>(raw) {
+            if uddsketch.uddsketch.count() != 0 {
+                if self.uddsketch.max_allowed_buckets() != uddsketch.uddsketch.max_allowed_buckets()
+                    || (self.error_rate - uddsketch.error_rate).abs() >= 1e-9
+                {
+                    return Err(DataFusionError::Plan(format!(
+                        "Merging UDDSketch with different parameters: arguments={:?} vs actual input={:?}",
+                        (
+                            self.uddsketch.max_allowed_buckets(),
+                            self.error_rate
+                        ),
+                        (uddsketch.uddsketch.max_allowed_buckets(), uddsketch.error_rate)
+                    )));
+                }
+                self.uddsketch.merge_sketch(&uddsketch.uddsketch);
             }
         } else {
             trace!("Warning: Failed to deserialize UDDSketch from {:?}", raw);
+            return Err(DataFusionError::Plan(
+                "Failed to deserialize UDDSketch from binary".to_string(),
+            ));
         }
+
+        Ok(())
     }
 }
 
@@ -113,9 +155,21 @@ fn downcast_accumulator_args(args: AccumulatorArgs) -> DfResult<(u64, f64)> {
 impl DfAccumulator for UddSketchState {
     fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> {
         let array = &values[2]; // the third column is data value
-        let f64_array = as_primitive_array::<Float64Type>(array)?;
-        for v in f64_array.iter().flatten() {
-            self.update(v);
+        match array.data_type() {
+            DataType::Float64 => {
+                let f64_array = as_primitive_array::<Float64Type>(array)?;
+                for v in f64_array.iter().flatten() {
+                    self.update(v);
+                }
+            }
+            // meaning instantiate as `uddsketch_merge`
+            DataType::Binary => self.merge_batch(&[array.clone()])?,
+            _ => {
+                return not_impl_err!(
+                    "UDDSketch functions do not support data type: {}",
+                    array.data_type()
+                )
+            }
         }
 
         Ok(())
@@ -123,7 +177,7 @@ impl DfAccumulator for UddSketchState {
 
     fn evaluate(&mut self) -> DfResult<ScalarValue> {
         Ok(ScalarValue::Binary(Some(
-            bincode::serialize(&self.uddsketch).map_err(|e| {
+            bincode::serialize(&self).map_err(|e| {
                 DataFusionError::Internal(format!("Failed to serialize UDDSketch: {}", e))
             })?,
         )))
@@ -150,7 +204,7 @@ impl DfAccumulator for UddSketchState {
 
     fn state(&mut self) -> DfResult<Vec<ScalarValue>> {
         Ok(vec![ScalarValue::Binary(Some(
-            bincode::serialize(&self.uddsketch).map_err(|e| {
+            bincode::serialize(&self).map_err(|e| {
                 DataFusionError::Internal(format!("Failed to serialize UDDSketch: {}", e))
             })?,
         ))])
@@ -160,7 +214,7 @@ impl DfAccumulator for UddSketchState {
         let array = &states[0];
         let binary_array = as_binary_array(array)?;
         for v in binary_array.iter().flatten() {
-            self.merge(v);
+            self.merge(v)?;
         }
 
         Ok(())
@@ -182,8 +236,8 @@ mod tests {
 
         let result = state.evaluate().unwrap();
         if let ScalarValue::Binary(Some(bytes)) = result {
-            let deserialized: UDDSketch = bincode::deserialize(&bytes).unwrap();
-            assert_eq!(deserialized.count(), 3);
+            let deserialized: UddSketchState = bincode::deserialize(&bytes).unwrap();
+            assert_eq!(deserialized.uddsketch.count(), 3);
         } else {
             panic!("Expected binary scalar value");
         }
@@ -201,13 +255,15 @@ mod tests {
         // Create new state and merge the serialized data
         let mut new_state = UddSketchState::new(10, 0.01);
         if let ScalarValue::Binary(Some(bytes)) = &serialized {
-            new_state.merge(bytes);
+            new_state.merge(bytes).unwrap();
 
             // Verify the merged state matches original by comparing deserialized values
-            let original_sketch: UDDSketch = bincode::deserialize(bytes).unwrap();
+            let original_sketch: UddSketchState = bincode::deserialize(bytes).unwrap();
+            let original_sketch = original_sketch.uddsketch;
             let new_result = new_state.evaluate().unwrap();
             if let ScalarValue::Binary(Some(new_bytes)) = new_result {
-                let new_sketch: UDDSketch = bincode::deserialize(&new_bytes).unwrap();
+                let new_sketch: UddSketchState = bincode::deserialize(&new_bytes).unwrap();
+                let new_sketch = new_sketch.uddsketch;
                 assert_eq!(original_sketch.count(), new_sketch.count());
                 assert_eq!(original_sketch.sum(), new_sketch.sum());
                 assert_eq!(original_sketch.mean(), new_sketch.mean());
@@ -244,7 +300,8 @@ mod tests {
 
         let result = state.evaluate().unwrap();
         if let ScalarValue::Binary(Some(bytes)) = result {
-            let deserialized: UDDSketch = bincode::deserialize(&bytes).unwrap();
+            let deserialized: UddSketchState = bincode::deserialize(&bytes).unwrap();
+            let deserialized = deserialized.uddsketch;
             assert_eq!(deserialized.count(), 3);
         } else {
             panic!("Expected binary scalar value");
@@ -273,7 +330,8 @@ mod tests {
 
             let result = merged_state.evaluate().unwrap();
             if let ScalarValue::Binary(Some(bytes)) = result {
-                let deserialized: UDDSketch = bincode::deserialize(&bytes).unwrap();
+                let deserialized: UddSketchState = bincode::deserialize(&bytes).unwrap();
+                let deserialized = deserialized.uddsketch;
                 assert_eq!(deserialized.count(), 2);
             } else {
                 panic!("Expected binary scalar value");

src/common/function/src/scalars/matches_term.rs

@@ -12,8 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt;
+use std::iter::repeat_n;
 use std::sync::Arc;
-use std::{fmt, iter};
 
 use common_query::error::{InvalidFuncArgsSnafu, Result};
 use common_query::prelude::Volatility;
@@ -126,9 +127,10 @@ impl Function for MatchesTermFunction {
             let term = term_column.get_ref(0).as_string().unwrap();
             match term {
                 None => {
-                    return Ok(Arc::new(BooleanVector::from_iter(
-                        iter::repeat(None).take(text_column.len()),
-                    )));
+                    return Ok(Arc::new(BooleanVector::from_iter(repeat_n(
+                        None,
+                        text_column.len(),
+                    ))));
                 }
                 Some(term) => Some(MatchesTermFinder::new(term)),
             }
@@ -217,7 +219,7 @@ impl MatchesTermFinder {
         }
 
         let mut pos = 0;
-        while let Some(found_pos) = self.finder.find(text[pos..].as_bytes()) {
+        while let Some(found_pos) = self.finder.find(&text.as_bytes()[pos..]) {
             let actual_pos = pos + found_pos;
 
             let prev_ok = self.starts_with_non_alnum

src/common/grpc/src/flight.rs

@@ -192,6 +192,10 @@ impl FlightDecoder {
             }
         }
     }
+
+    pub fn schema(&self) -> Option<&SchemaRef> {
+        self.schema.as_ref()
+    }
 }
 
 pub fn flight_messages_to_recordbatches(messages: Vec<FlightMessage>) -> Result<RecordBatches> {

src/common/meta/src/cache/flow/table_flownode.rs

@@ -24,39 +24,21 @@ use crate::cache::{CacheContainer, Initializer};
 use crate::error::Result;
 use crate::instruction::{CacheIdent, CreateFlow, DropFlow};
 use crate::key::flow::{TableFlowManager, TableFlowManagerRef};
-use crate::key::{FlowId, FlowPartitionId};
 use crate::kv_backend::KvBackendRef;
 use crate::peer::Peer;
+use crate::FlownodeId;
 
-/// Flow id&flow partition key
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct FlowIdent {
-    pub flow_id: FlowId,
-    pub partition_id: FlowPartitionId,
-}
-
-impl FlowIdent {
-    pub fn new(flow_id: FlowId, partition_id: FlowPartitionId) -> Self {
-        Self {
-            flow_id,
-            partition_id,
-        }
-    }
-}
-
-/// cache for TableFlowManager, the table_id part is in the outer cache
-/// include flownode_id, flow_id, partition_id mapping to Peer
-type FlownodeFlowSet = Arc<HashMap<FlowIdent, Peer>>;
+type FlownodeSet = Arc<HashMap<FlownodeId, Peer>>;
 
 pub type TableFlownodeSetCacheRef = Arc<TableFlownodeSetCache>;
 
 /// [TableFlownodeSetCache] caches the [TableId] to [FlownodeSet] mapping.
-pub type TableFlownodeSetCache = CacheContainer<TableId, FlownodeFlowSet, CacheIdent>;
+pub type TableFlownodeSetCache = CacheContainer<TableId, FlownodeSet, CacheIdent>;
 
 /// Constructs a [TableFlownodeSetCache].
 pub fn new_table_flownode_set_cache(
     name: String,
-    cache: Cache<TableId, FlownodeFlowSet>,
+    cache: Cache<TableId, FlownodeSet>,
     kv_backend: KvBackendRef,
 ) -> TableFlownodeSetCache {
     let table_flow_manager = Arc::new(TableFlowManager::new(kv_backend));
@@ -65,7 +47,7 @@ pub fn new_table_flownode_set_cache(
     CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
 }
 
-fn init_factory(table_flow_manager: TableFlowManagerRef) -> Initializer<TableId, FlownodeFlowSet> {
+fn init_factory(table_flow_manager: TableFlowManagerRef) -> Initializer<TableId, FlownodeSet> {
     Arc::new(move |&table_id| {
         let table_flow_manager = table_flow_manager.clone();
         Box::pin(async move {
@@ -75,12 +57,7 @@ fn init_factory(table_flow_manager: TableFlowManagerRef) -> Initializer<TableId,
                 .map(|flows| {
                     flows
                         .into_iter()
-                        .map(|(key, value)| {
-                            (
-                                FlowIdent::new(key.flow_id(), key.partition_id()),
-                                value.peer,
-                            )
-                        })
+                        .map(|(key, value)| (key.flownode_id(), value.peer))
                         .collect::<HashMap<_, _>>()
                 })
                 // We must cache the `HashSet` even if it's empty,
@@ -94,33 +71,26 @@ fn init_factory(table_flow_manager: TableFlowManagerRef) -> Initializer<TableId,
 }
 
 async fn handle_create_flow(
-    cache: &Cache<TableId, FlownodeFlowSet>,
+    cache: &Cache<TableId, FlownodeSet>,
     CreateFlow {
-        flow_id,
         source_table_ids,
-        partition_to_peer_mapping: flow_part2nodes,
+        flownodes: flownode_peers,
     }: &CreateFlow,
 ) {
     for table_id in source_table_ids {
         let entry = cache.entry(*table_id);
         entry
             .and_compute_with(
-                async |entry: Option<moka::Entry<u32, FlownodeFlowSet>>| match entry {
+                async |entry: Option<moka::Entry<u32, Arc<HashMap<u64, _>>>>| match entry {
                     Some(entry) => {
                         let mut map = entry.into_value().as_ref().clone();
-                        map.extend(
-                            flow_part2nodes.iter().map(|(part, peer)| {
-                                (FlowIdent::new(*flow_id, *part), peer.clone())
-                            }),
-                        );
+                        map.extend(flownode_peers.iter().map(|peer| (peer.id, peer.clone())));
 
                         Op::Put(Arc::new(map))
                     }
-                    None => {
-                        Op::Put(Arc::new(HashMap::from_iter(flow_part2nodes.iter().map(
-                            |(part, peer)| (FlowIdent::new(*flow_id, *part), peer.clone()),
-                        ))))
-                    }
+                    None => Op::Put(Arc::new(HashMap::from_iter(
+                        flownode_peers.iter().map(|peer| (peer.id, peer.clone())),
+                    ))),
                 },
             )
             .await;
@@ -128,23 +98,21 @@ async fn handle_create_flow(
 }
 
 async fn handle_drop_flow(
-    cache: &Cache<TableId, FlownodeFlowSet>,
+    cache: &Cache<TableId, FlownodeSet>,
     DropFlow {
-        flow_id,
         source_table_ids,
-        flow_part2node_id,
+        flownode_ids,
     }: &DropFlow,
 ) {
     for table_id in source_table_ids {
         let entry = cache.entry(*table_id);
         entry
             .and_compute_with(
-                async |entry: Option<moka::Entry<u32, FlownodeFlowSet>>| match entry {
+                async |entry: Option<moka::Entry<u32, Arc<HashMap<u64, _>>>>| match entry {
                     Some(entry) => {
                         let mut set = entry.into_value().as_ref().clone();
-                        for (part, _node) in flow_part2node_id {
-                            let key = FlowIdent::new(*flow_id, *part);
-                            set.remove(&key);
+                        for flownode_id in flownode_ids {
+                            set.remove(flownode_id);
                         }
 
                         Op::Put(Arc::new(set))
@@ -160,7 +128,7 @@ async fn handle_drop_flow(
 }
 
 fn invalidator<'a>(
-    cache: &'a Cache<TableId, FlownodeFlowSet>,
+    cache: &'a Cache<TableId, FlownodeSet>,
     ident: &'a CacheIdent,
 ) -> BoxFuture<'a, Result<()>> {
     Box::pin(async move {
@@ -186,7 +154,7 @@ mod tests {
     use moka::future::CacheBuilder;
     use table::table_name::TableName;
 
-    use crate::cache::flow::table_flownode::{new_table_flownode_set_cache, FlowIdent};
+    use crate::cache::flow::table_flownode::new_table_flownode_set_cache;
     use crate::instruction::{CacheIdent, CreateFlow, DropFlow};
     use crate::key::flow::flow_info::FlowInfoValue;
     use crate::key::flow::flow_route::FlowRouteValue;
@@ -246,16 +214,12 @@ mod tests {
         let set = cache.get(1024).await.unwrap().unwrap();
         assert_eq!(
             set.as_ref().clone(),
-            HashMap::from_iter(
-                (1..=3).map(|i| { (FlowIdent::new(1024, (i - 1) as u32), Peer::empty(i),) })
-            )
+            HashMap::from_iter((1..=3).map(|i| { (i, Peer::empty(i),) }))
         );
         let set = cache.get(1025).await.unwrap().unwrap();
         assert_eq!(
             set.as_ref().clone(),
-            HashMap::from_iter(
-                (1..=3).map(|i| { (FlowIdent::new(1024, (i - 1) as u32), Peer::empty(i),) })
-            )
+            HashMap::from_iter((1..=3).map(|i| { (i, Peer::empty(i),) }))
         );
         let result = cache.get(1026).await.unwrap().unwrap();
         assert_eq!(result.len(), 0);
@@ -267,9 +231,8 @@ mod tests {
         let cache = CacheBuilder::new(128).build();
         let cache = new_table_flownode_set_cache("test".to_string(), cache, mem_kv);
         let ident = vec![CacheIdent::CreateFlow(CreateFlow {
-            flow_id: 2001,
             source_table_ids: vec![1024, 1025],
-            partition_to_peer_mapping: (1..=5).map(|i| (i as u32, Peer::empty(i + 1))).collect(),
+            flownodes: (1..=5).map(Peer::empty).collect(),
         })];
         cache.invalidate(&ident).await.unwrap();
         let set = cache.get(1024).await.unwrap().unwrap();
@@ -278,54 +241,6 @@ mod tests {
         assert_eq!(set.len(), 5);
     }
 
-    #[tokio::test]
-    async fn test_replace_flow() {
-        let mem_kv = Arc::new(MemoryKvBackend::default());
-        let cache = CacheBuilder::new(128).build();
-        let cache = new_table_flownode_set_cache("test".to_string(), cache, mem_kv);
-        let ident = vec![CacheIdent::CreateFlow(CreateFlow {
-            flow_id: 2001,
-            source_table_ids: vec![1024, 1025],
-            partition_to_peer_mapping: (1..=5).map(|i| (i as u32, Peer::empty(i + 1))).collect(),
-        })];
-        cache.invalidate(&ident).await.unwrap();
-        let set = cache.get(1024).await.unwrap().unwrap();
-        assert_eq!(set.len(), 5);
-        let set = cache.get(1025).await.unwrap().unwrap();
-        assert_eq!(set.len(), 5);
-
-        let drop_then_create_flow = vec![
-            CacheIdent::DropFlow(DropFlow {
-                flow_id: 2001,
-                source_table_ids: vec![1024, 1025],
-                flow_part2node_id: (1..=5).map(|i| (i as u32, i + 1)).collect(),
-            }),
-            CacheIdent::CreateFlow(CreateFlow {
-                flow_id: 2001,
-                source_table_ids: vec![1026, 1027],
-                partition_to_peer_mapping: (11..=15)
-                    .map(|i| (i as u32, Peer::empty(i + 1)))
-                    .collect(),
-            }),
-            CacheIdent::FlowId(2001),
-        ];
-        cache.invalidate(&drop_then_create_flow).await.unwrap();
-
-        let set = cache.get(1024).await.unwrap().unwrap();
-        assert!(set.is_empty());
-
-        let expected = HashMap::from_iter(
-            (11..=15).map(|i| (FlowIdent::new(2001, i as u32), Peer::empty(i + 1))),
-        );
-        let set = cache.get(1026).await.unwrap().unwrap();
-
-        assert_eq!(set.as_ref().clone(), expected);
-
-        let set = cache.get(1027).await.unwrap().unwrap();
-
-        assert_eq!(set.as_ref().clone(), expected);
-    }
-
     #[tokio::test]
     async fn test_drop_flow() {
         let mem_kv = Arc::new(MemoryKvBackend::default());
@@ -333,57 +248,34 @@ mod tests {
         let cache = new_table_flownode_set_cache("test".to_string(), cache, mem_kv);
         let ident = vec![
             CacheIdent::CreateFlow(CreateFlow {
-                flow_id: 2001,
                 source_table_ids: vec![1024, 1025],
-                partition_to_peer_mapping: (1..=5)
-                    .map(|i| (i as u32, Peer::empty(i + 1)))
-                    .collect(),
+                flownodes: (1..=5).map(Peer::empty).collect(),
             }),
             CacheIdent::CreateFlow(CreateFlow {
-                flow_id: 2002,
                 source_table_ids: vec![1024, 1025],
-                partition_to_peer_mapping: (11..=12)
-                    .map(|i| (i as u32, Peer::empty(i + 1)))
-                    .collect(),
-            }),
-            // same flownode that hold multiple flows
-            CacheIdent::CreateFlow(CreateFlow {
-                flow_id: 2003,
-                source_table_ids: vec![1024, 1025],
-                partition_to_peer_mapping: (1..=5)
-                    .map(|i| (i as u32, Peer::empty(i + 1)))
-                    .collect(),
+                flownodes: (11..=12).map(Peer::empty).collect(),
             }),
         ];
         cache.invalidate(&ident).await.unwrap();
         let set = cache.get(1024).await.unwrap().unwrap();
-        assert_eq!(set.len(), 12);
+        assert_eq!(set.len(), 7);
         let set = cache.get(1025).await.unwrap().unwrap();
-        assert_eq!(set.len(), 12);
+        assert_eq!(set.len(), 7);
 
         let ident = vec![CacheIdent::DropFlow(DropFlow {
-            flow_id: 2001,
             source_table_ids: vec![1024, 1025],
-            flow_part2node_id: (1..=5).map(|i| (i as u32, i + 1)).collect(),
+            flownode_ids: vec![1, 2, 3, 4, 5],
         })];
         cache.invalidate(&ident).await.unwrap();
         let set = cache.get(1024).await.unwrap().unwrap();
         assert_eq!(
             set.as_ref().clone(),
-            HashMap::from_iter(
-                (11..=12)
-                    .map(|i| (FlowIdent::new(2002, i as u32), Peer::empty(i + 1)))
-                    .chain((1..=5).map(|i| (FlowIdent::new(2003, i as u32), Peer::empty(i + 1))))
-            )
+            HashMap::from_iter((11..=12).map(|i| { (i, Peer::empty(i),) }))
         );
         let set = cache.get(1025).await.unwrap().unwrap();
         assert_eq!(
             set.as_ref().clone(),
-            HashMap::from_iter(
-                (11..=12)
-                    .map(|i| (FlowIdent::new(2002, i as u32), Peer::empty(i + 1)))
-                    .chain((1..=5).map(|i| (FlowIdent::new(2003, i as u32), Peer::empty(i + 1))))
-            )
+            HashMap::from_iter((11..=12).map(|i| { (i, Peer::empty(i),) }))
         );
     }
 }

src/common/meta/src/cache_invalidator.rs

@@ -16,12 +16,9 @@ use std::sync::Arc;
 
 use crate::error::Result;
 use crate::flow_name::FlowName;
-use crate::instruction::{CacheIdent, DropFlow};
+use crate::instruction::CacheIdent;
 use crate::key::flow::flow_info::FlowInfoKey;
 use crate::key::flow::flow_name::FlowNameKey;
-use crate::key::flow::flow_route::FlowRouteKey;
-use crate::key::flow::flownode_flow::FlownodeFlowKey;
-use crate::key::flow::table_flow::TableFlowKey;
 use crate::key::schema_name::SchemaNameKey;
 use crate::key::table_info::TableInfoKey;
 use crate::key::table_name::TableNameKey;
@@ -92,40 +89,9 @@ where
                     let key: SchemaNameKey = schema_name.into();
                     self.invalidate_key(&key.to_bytes()).await;
                 }
-                CacheIdent::CreateFlow(_) => {
+                CacheIdent::CreateFlow(_) | CacheIdent::DropFlow(_) => {
                     // Do nothing
                 }
-                CacheIdent::DropFlow(DropFlow {
-                    flow_id,
-                    source_table_ids,
-                    flow_part2node_id,
-                }) => {
-                    // invalidate flow route/flownode flow/table flow
-                    let mut keys = Vec::with_capacity(
-                        source_table_ids.len() * flow_part2node_id.len()
-                            + flow_part2node_id.len() * 2,
-                    );
-                    for table_id in source_table_ids {
-                        for (partition_id, node_id) in flow_part2node_id {
-                            let key =
-                                TableFlowKey::new(*table_id, *node_id, *flow_id, *partition_id)
-                                    .to_bytes();
-                            keys.push(key);
-                        }
-                    }
-
-                    for (partition_id, node_id) in flow_part2node_id {
-                        let key =
-                            FlownodeFlowKey::new(*node_id, *flow_id, *partition_id).to_bytes();
-                        keys.push(key);
-                        let key = FlowRouteKey::new(*flow_id, *partition_id).to_bytes();
-                        keys.push(key);
-                    }
-
-                    for key in keys {
-                        self.invalidate_key(&key).await;
-                    }
-                }
                 CacheIdent::FlowName(FlowName {
                     catalog_name,
                     flow_name,

src/common/meta/src/ddl/create_flow.rs

@@ -39,7 +39,7 @@ use crate::cache_invalidator::Context;
 use crate::ddl::utils::{add_peer_context_if_needed, handle_retry_error};
 use crate::ddl::DdlContext;
 use crate::error::{self, Result, UnexpectedSnafu};
-use crate::instruction::{CacheIdent, CreateFlow, DropFlow};
+use crate::instruction::{CacheIdent, CreateFlow};
 use crate::key::flow::flow_info::FlowInfoValue;
 use crate::key::flow::flow_route::FlowRouteValue;
 use crate::key::table_name::TableNameKey;
@@ -70,7 +70,6 @@ impl CreateFlowProcedure {
                 query_context,
                 state: CreateFlowState::Prepare,
                 prev_flow_info_value: None,
-                did_replace: false,
                 flow_type: None,
             },
         }
@@ -225,7 +224,6 @@ impl CreateFlowProcedure {
                 .update_flow_metadata(flow_id, prev_flow_value, &flow_info, flow_routes)
                 .await?;
             info!("Replaced flow metadata for flow {flow_id}");
-            self.data.did_replace = true;
         } else {
             self.context
                 .flow_metadata_manager
@@ -242,43 +240,22 @@ impl CreateFlowProcedure {
         debug_assert!(self.data.state == CreateFlowState::InvalidateFlowCache);
         // Safety: The flow id must be allocated.
         let flow_id = self.data.flow_id.unwrap();
-        let did_replace = self.data.did_replace;
         let ctx = Context {
             subject: Some("Invalidate flow cache by creating flow".to_string()),
         };
 
-        let mut caches = vec![];
-
-        // if did replaced, invalidate the flow cache with drop the old flow
-        if did_replace {
-            let old_flow_info = self.data.prev_flow_info_value.as_ref().unwrap();
-
-            // only drop flow is needed, since flow name haven't changed, and flow id already invalidated below
-            caches.extend([CacheIdent::DropFlow(DropFlow {
-                flow_id,
-                source_table_ids: old_flow_info.source_table_ids.clone(),
-                flow_part2node_id: old_flow_info.flownode_ids().clone().into_iter().collect(),
-            })]);
-        }
-
-        let (_flow_info, flow_routes) = (&self.data).into();
-        let flow_part2peers = flow_routes
-            .into_iter()
-            .map(|(part_id, route)| (part_id, route.peer))
-            .collect();
-
-        caches.extend([
-            CacheIdent::CreateFlow(CreateFlow {
-                flow_id,
-                source_table_ids: self.data.source_table_ids.clone(),
-                partition_to_peer_mapping: flow_part2peers,
-            }),
-            CacheIdent::FlowId(flow_id),
-        ]);
-
         self.context
             .cache_invalidator
-            .invalidate(&ctx, &caches)
+            .invalidate(
+                &ctx,
+                &[
+                    CacheIdent::CreateFlow(CreateFlow {
+                        source_table_ids: self.data.source_table_ids.clone(),
+                        flownodes: self.data.peers.clone(),
+                    }),
+                    CacheIdent::FlowId(flow_id),
+                ],
+            )
             .await?;
 
         Ok(Status::done_with_output(flow_id))
@@ -400,10 +377,6 @@ pub struct CreateFlowData {
     /// For verify if prev value is consistent when need to update flow metadata.
     /// only set when `or_replace` is true.
     pub(crate) prev_flow_info_value: Option<DeserializedValueWithBytes<FlowInfoValue>>,
-    /// Only set to true when replace actually happened.
-    /// This is used to determine whether to invalidate the cache.
-    #[serde(default)]
-    pub(crate) did_replace: bool,
     pub(crate) flow_type: Option<FlowType>,
 }
 

src/common/meta/src/ddl/drop_flow.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 mod metadata;
-
 use api::v1::flow::{flow_request, DropRequest, FlowRequest};
 use async_trait::async_trait;
 use common_catalog::format_full_flow_name;
@@ -154,12 +153,6 @@ impl DropFlowProcedure {
         };
         let flow_info_value = self.data.flow_info_value.as_ref().unwrap();
 
-        let flow_part2nodes = flow_info_value
-            .flownode_ids()
-            .clone()
-            .into_iter()
-            .collect::<Vec<_>>();
-
         self.context
             .cache_invalidator
             .invalidate(
@@ -171,9 +164,8 @@ impl DropFlowProcedure {
                         flow_name: flow_info_value.flow_name.to_string(),
                     }),
                     CacheIdent::DropFlow(DropFlow {
-                        flow_id,
                         source_table_ids: flow_info_value.source_table_ids.clone(),
-                        flow_part2node_id: flow_part2nodes,
+                        flownode_ids: flow_info_value.flownode_ids.values().cloned().collect(),
                     }),
                 ],
             )

src/common/meta/src/error.rs

@@ -514,25 +514,11 @@ pub enum Error {
     },
 
     #[snafu(display(
-        "Failed to get a Kafka partition client, topic: {}, partition: {}",
+        "Failed to build a Kafka partition client, topic: {}, partition: {}",
         topic,
         partition
     ))]
-    KafkaPartitionClient {
-        topic: String,
-        partition: i32,
-        #[snafu(implicit)]
-        location: Location,
-        #[snafu(source)]
-        error: rskafka::client::error::Error,
-    },
-
-    #[snafu(display(
-        "Failed to get offset from Kafka, topic: {}, partition: {}",
-        topic,
-        partition
-    ))]
-    KafkaGetOffset {
+    BuildKafkaPartitionClient {
         topic: String,
         partition: i32,
         #[snafu(implicit)]
@@ -857,7 +843,7 @@ impl ErrorExt for Error {
             | EncodeWalOptions { .. }
             | BuildKafkaClient { .. }
             | BuildKafkaCtrlClient { .. }
-            | KafkaPartitionClient { .. }
+            | BuildKafkaPartitionClient { .. }
             | ResolveKafkaEndpoint { .. }
             | ProduceRecord { .. }
             | CreateKafkaWalTopic { .. }
@@ -866,8 +852,7 @@ impl ErrorExt for Error {
             | ProcedureOutput { .. }
             | FromUtf8 { .. }
             | MetadataCorruption { .. }
-            | ParseWalOptions { .. }
-            | KafkaGetOffset { .. } => StatusCode::Unexpected,
+            | ParseWalOptions { .. } => StatusCode::Unexpected,
 
             SendMessage { .. } | GetKvCache { .. } | CacheNotGet { .. } => StatusCode::Internal,
 

src/common/meta/src/instruction.rs

@@ -24,7 +24,7 @@ use table::table_name::TableName;
 
 use crate::flow_name::FlowName;
 use crate::key::schema_name::SchemaName;
-use crate::key::{FlowId, FlowPartitionId};
+use crate::key::FlowId;
 use crate::peer::Peer;
 use crate::{DatanodeId, FlownodeId};
 
@@ -184,19 +184,14 @@ pub enum CacheIdent {
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct CreateFlow {
-    /// The unique identifier for the flow.
-    pub flow_id: FlowId,
     pub source_table_ids: Vec<TableId>,
-    /// Mapping of flow partition to peer information
-    pub partition_to_peer_mapping: Vec<(FlowPartitionId, Peer)>,
+    pub flownodes: Vec<Peer>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DropFlow {
-    pub flow_id: FlowId,
     pub source_table_ids: Vec<TableId>,
-    /// Mapping of flow partition to flownode id
-    pub flow_part2node_id: Vec<(FlowPartitionId, FlownodeId)>,
+    pub flownode_ids: Vec<FlownodeId>,
 }
 
 /// Flushes a batch of regions.
@@ -222,7 +217,9 @@ pub enum Instruction {
     /// Invalidates batch cache.
     InvalidateCaches(Vec<CacheIdent>),
     /// Flushes regions.
-    FlushRegion(FlushRegions),
+    FlushRegions(FlushRegions),
+    /// Flushes a single region.
+    FlushRegion(RegionId),
 }
 
 /// The reply of [UpgradeRegion].
@@ -253,6 +250,7 @@ pub enum InstructionReply {
     CloseRegion(SimpleReply),
     UpgradeRegion(UpgradeRegionReply),
     DowngradeRegion(DowngradeRegionReply),
+    FlushRegion(SimpleReply),
 }
 
 impl Display for InstructionReply {
@@ -264,6 +262,7 @@ impl Display for InstructionReply {
             Self::DowngradeRegion(reply) => {
                 write!(f, "InstructionReply::DowngradeRegion({})", reply)
             }
+            Self::FlushRegion(reply) => write!(f, "InstructionReply::FlushRegion({})", reply),
         }
     }
 }

src/common/meta/src/key.rs

@@ -112,7 +112,7 @@ pub mod test_utils;
 mod tombstone;
 pub mod topic_name;
 pub mod topic_region;
-pub(crate) mod txn_helper;
+pub mod txn_helper;
 pub mod view_info;
 
 use std::collections::{BTreeMap, HashMap, HashSet};

src/common/meta/src/key/datanode_table.rs

@@ -256,11 +256,6 @@ impl DatanodeTableManager {
             })?
             .and_then(|r| DatanodeTableValue::try_from_raw_value(&r.value))?
             .region_info;
-
-        // If the region options are the same, we don't need to update it.
-        if region_info.region_options == new_region_options {
-            return Ok(Txn::new());
-        }
         // substitute region options only.
         region_info.region_options = new_region_options;
 

src/common/meta/src/key/flow.rs

@@ -45,7 +45,7 @@ use crate::kv_backend::KvBackendRef;
 use crate::rpc::store::BatchDeleteRequest;
 
 /// The key of `__flow/` scope.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, PartialEq)]
 pub struct FlowScoped<T> {
     inner: T,
 }
@@ -246,32 +246,27 @@ impl FlowMetadataManager {
         new_flow_info: &FlowInfoValue,
         flow_routes: Vec<(FlowPartitionId, FlowRouteValue)>,
     ) -> Result<()> {
-        let (update_flow_flow_name_txn, on_create_flow_flow_name_failure) =
+        let (create_flow_flow_name_txn, on_create_flow_flow_name_failure) =
             self.flow_name_manager.build_update_txn(
                 &new_flow_info.catalog_name,
                 &new_flow_info.flow_name,
                 flow_id,
             )?;
 
-        let (update_flow_txn, on_create_flow_failure) =
+        let (create_flow_txn, on_create_flow_failure) =
             self.flow_info_manager
                 .build_update_txn(flow_id, current_flow_info, new_flow_info)?;
 
-        let update_flow_routes_txn = self.flow_route_manager.build_update_txn(
-            flow_id,
-            current_flow_info,
-            flow_routes.clone(),
-        )?;
+        let create_flow_routes_txn = self
+            .flow_route_manager
+            .build_create_txn(flow_id, flow_routes.clone())?;
 
-        let update_flownode_flow_txn = self.flownode_flow_manager.build_update_txn(
-            flow_id,
-            current_flow_info,
-            new_flow_info.flownode_ids().clone(),
-        );
+        let create_flownode_flow_txn = self
+            .flownode_flow_manager
+            .build_create_txn(flow_id, new_flow_info.flownode_ids().clone());
 
-        let update_table_flow_txn = self.table_flow_manager.build_update_txn(
+        let create_table_flow_txn = self.table_flow_manager.build_create_txn(
             flow_id,
-            current_flow_info,
             flow_routes
                 .into_iter()
                 .map(|(partition_id, route)| (partition_id, TableFlowValue { peer: route.peer }))
@@ -280,11 +275,11 @@ impl FlowMetadataManager {
         )?;
 
         let txn = Txn::merge_all(vec![
-            update_flow_flow_name_txn,
-            update_flow_txn,
-            update_flow_routes_txn,
-            update_flownode_flow_txn,
-            update_table_flow_txn,
+            create_flow_flow_name_txn,
+            create_flow_txn,
+            create_flow_routes_txn,
+            create_flownode_flow_txn,
+            create_table_flow_txn,
         ]);
         info!(
             "Creating flow {}.{}({}), with {} txn operations",
@@ -788,141 +783,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn test_update_flow_metadata_diff_flownode() {
-        let mem_kv = Arc::new(MemoryKvBackend::default());
-        let flow_metadata_manager = FlowMetadataManager::new(mem_kv.clone());
-        let flow_id = 10;
-        let flow_value = test_flow_info_value(
-            "flow",
-            [(0u32, 1u64), (1u32, 2u64)].into(),
-            vec![1024, 1025, 1026],
-        );
-        let flow_routes = vec![
-            (
-                0u32,
-                FlowRouteValue {
-                    peer: Peer::empty(1),
-                },
-            ),
-            (
-                1,
-                FlowRouteValue {
-                    peer: Peer::empty(2),
-                },
-            ),
-        ];
-        flow_metadata_manager
-            .create_flow_metadata(flow_id, flow_value.clone(), flow_routes.clone())
-            .await
-            .unwrap();
-
-        let new_flow_value = {
-            let mut tmp = flow_value.clone();
-            tmp.raw_sql = "new".to_string();
-            // move to different flownodes
-            tmp.flownode_ids = [(0, 3u64), (1, 4u64)].into();
-            tmp
-        };
-        let new_flow_routes = vec![
-            (
-                0u32,
-                FlowRouteValue {
-                    peer: Peer::empty(3),
-                },
-            ),
-            (
-                1,
-                FlowRouteValue {
-                    peer: Peer::empty(4),
-                },
-            ),
-        ];
-
-        // Update flow instead
-        flow_metadata_manager
-            .update_flow_metadata(
-                flow_id,
-                &DeserializedValueWithBytes::from_inner(flow_value.clone()),
-                &new_flow_value,
-                new_flow_routes.clone(),
-            )
-            .await
-            .unwrap();
-
-        let got = flow_metadata_manager
-            .flow_info_manager()
-            .get(flow_id)
-            .await
-            .unwrap()
-            .unwrap();
-        let routes = flow_metadata_manager
-            .flow_route_manager()
-            .routes(flow_id)
-            .await
-            .unwrap();
-        assert_eq!(
-            routes,
-            vec![
-                (
-                    FlowRouteKey::new(flow_id, 0),
-                    FlowRouteValue {
-                        peer: Peer::empty(3),
-                    },
-                ),
-                (
-                    FlowRouteKey::new(flow_id, 1),
-                    FlowRouteValue {
-                        peer: Peer::empty(4),
-                    },
-                ),
-            ]
-        );
-        assert_eq!(got, new_flow_value);
-
-        let flows = flow_metadata_manager
-            .flownode_flow_manager()
-            .flows(1)
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        // should moved to different flownode
-        assert_eq!(flows, vec![]);
-
-        let flows = flow_metadata_manager
-            .flownode_flow_manager()
-            .flows(3)
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        assert_eq!(flows, vec![(flow_id, 0)]);
-
-        for table_id in [1024, 1025, 1026] {
-            let nodes = flow_metadata_manager
-                .table_flow_manager()
-                .flows(table_id)
-                .await
-                .unwrap();
-            assert_eq!(
-                nodes,
-                vec![
-                    (
-                        TableFlowKey::new(table_id, 3, flow_id, 0),
-                        TableFlowValue {
-                            peer: Peer::empty(3)
-                        }
-                    ),
-                    (
-                        TableFlowKey::new(table_id, 4, flow_id, 1),
-                        TableFlowValue {
-                            peer: Peer::empty(4)
-                        }
-                    )
-                ]
-            );
-        }
-    }
-
     #[tokio::test]
     async fn test_update_flow_metadata_flow_replace_diff_id_err() {
         let mem_kv = Arc::new(MemoryKvBackend::default());

src/common/meta/src/key/flow/flow_info.rs

@@ -153,15 +153,6 @@ impl FlowInfoValue {
         &self.flownode_ids
     }
 
-    /// Insert a new flownode id for a partition.
-    pub fn insert_flownode_id(
-        &mut self,
-        partition: FlowPartitionId,
-        node: FlownodeId,
-    ) -> Option<FlownodeId> {
-        self.flownode_ids.insert(partition, node)
-    }
-
     /// Returns the `source_table`.
     pub fn source_table_ids(&self) -> &[TableId] {
         &self.source_table_ids
@@ -281,11 +272,10 @@ impl FlowInfoManager {
         let raw_value = new_flow_value.try_as_raw_value()?;
         let prev_value = current_flow_value.get_raw_bytes();
         let txn = Txn::new()
-            .when(vec![Compare::new(
-                key.clone(),
-                CompareOp::Equal,
-                Some(prev_value),
-            )])
+            .when(vec![
+                Compare::new(key.clone(), CompareOp::NotEqual, None),
+                Compare::new(key.clone(), CompareOp::Equal, Some(prev_value)),
+            ])
             .and_then(vec![TxnOp::Put(key.clone(), raw_value)])
             .or_else(vec![TxnOp::Get(key.clone())]);
 

src/common/meta/src/key/flow/flow_route.rs

@@ -19,12 +19,9 @@ use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 
 use crate::error::{self, Result};
-use crate::key::flow::flow_info::FlowInfoValue;
 use crate::key::flow::{flownode_addr_helper, FlowScoped};
 use crate::key::node_address::NodeAddressKey;
-use crate::key::{
-    BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue,
-};
+use crate::key::{BytesAdapter, FlowId, FlowPartitionId, MetadataKey, MetadataValue};
 use crate::kv_backend::txn::{Txn, TxnOp};
 use crate::kv_backend::KvBackendRef;
 use crate::peer::Peer;
@@ -42,7 +39,7 @@ lazy_static! {
 /// The key stores the route info of the flow.
 ///
 /// The layout: `__flow/route/{flow_id}/{partition_id}`.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, PartialEq)]
 pub struct FlowRouteKey(FlowScoped<FlowRouteKeyInner>);
 
 impl FlowRouteKey {
@@ -145,12 +142,6 @@ pub struct FlowRouteValue {
     pub(crate) peer: Peer,
 }
 
-impl From<Peer> for FlowRouteValue {
-    fn from(peer: Peer) -> Self {
-        Self { peer }
-    }
-}
-
 impl FlowRouteValue {
     /// Returns the `peer`.
     pub fn peer(&self) -> &Peer {
@@ -213,33 +204,6 @@ impl FlowRouteManager {
         Ok(Txn::new().and_then(txns))
     }
 
-    /// Builds a update flow routes transaction.
-    ///
-    /// Puts `__flow/route/{flow_id}/{partition_id}` keys.
-    /// Also removes `__flow/route/{flow_id}/{old_partition_id}` keys.
-    pub(crate) fn build_update_txn<I: IntoIterator<Item = (FlowPartitionId, FlowRouteValue)>>(
-        &self,
-        flow_id: FlowId,
-        current_flow_info: &DeserializedValueWithBytes<FlowInfoValue>,
-        flow_routes: I,
-    ) -> Result<Txn> {
-        let del_txns = current_flow_info
-            .flownode_ids()
-            .iter()
-            .map(|(partition_id, _)| {
-                let key = FlowRouteKey::new(flow_id, *partition_id).to_bytes();
-                Ok(TxnOp::Delete(key))
-            });
-
-        let put_txns = flow_routes.into_iter().map(|(partition_id, route)| {
-            let key = FlowRouteKey::new(flow_id, partition_id).to_bytes();
-
-            Ok(TxnOp::Put(key, route.try_as_raw_value()?))
-        });
-        let txns = del_txns.chain(put_txns).collect::<Result<Vec<_>>>()?;
-        Ok(Txn::new().and_then(txns))
-    }
-
     async fn remap_flow_route_addresses(
         &self,
         flow_routes: &mut [(FlowRouteKey, FlowRouteValue)],

src/common/meta/src/key/flow/flownode_flow.rs

@@ -19,9 +19,8 @@ use regex::Regex;
 use snafu::OptionExt;
 
 use crate::error::{self, Result};
-use crate::key::flow::flow_info::FlowInfoValue;
 use crate::key::flow::FlowScoped;
-use crate::key::{BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey};
+use crate::key::{BytesAdapter, FlowId, FlowPartitionId, MetadataKey};
 use crate::kv_backend::txn::{Txn, TxnOp};
 use crate::kv_backend::KvBackendRef;
 use crate::range_stream::{PaginationStream, DEFAULT_PAGE_SIZE};
@@ -166,17 +165,6 @@ impl FlownodeFlowManager {
         Self { kv_backend }
     }
 
-    /// Whether given flow exist on this flownode.
-    pub async fn exists(
-        &self,
-        flownode_id: FlownodeId,
-        flow_id: FlowId,
-        partition_id: FlowPartitionId,
-    ) -> Result<bool> {
-        let key = FlownodeFlowKey::new(flownode_id, flow_id, partition_id).to_bytes();
-        Ok(self.kv_backend.get(&key).await?.is_some())
-    }
-
     /// Retrieves all [FlowId] and [FlowPartitionId]s of the specified `flownode_id`.
     pub fn flows(
         &self,
@@ -214,33 +202,6 @@ impl FlownodeFlowManager {
 
         Txn::new().and_then(txns)
     }
-
-    /// Builds a update flownode flow transaction.
-    ///
-    /// Puts `__flownode_flow/{flownode_id}/{flow_id}/{partition_id}` keys.
-    /// Remove the old `__flownode_flow/{old_flownode_id}/{flow_id}/{old_partition_id}` keys.
-    pub(crate) fn build_update_txn<I: IntoIterator<Item = (FlowPartitionId, FlownodeId)>>(
-        &self,
-        flow_id: FlowId,
-        current_flow_info: &DeserializedValueWithBytes<FlowInfoValue>,
-        flownode_ids: I,
-    ) -> Txn {
-        let del_txns =
-            current_flow_info
-                .flownode_ids()
-                .iter()
-                .map(|(partition_id, flownode_id)| {
-                    let key = FlownodeFlowKey::new(*flownode_id, flow_id, *partition_id).to_bytes();
-                    TxnOp::Delete(key)
-                });
-        let put_txns = flownode_ids.into_iter().map(|(partition_id, flownode_id)| {
-            let key = FlownodeFlowKey::new(flownode_id, flow_id, partition_id).to_bytes();
-            TxnOp::Put(key, vec![])
-        });
-        let txns = del_txns.chain(put_txns).collect::<Vec<_>>();
-
-        Txn::new().and_then(txns)
-    }
 }
 
 #[cfg(test)]

src/common/meta/src/key/flow/table_flow.rs

@@ -22,12 +22,9 @@ use snafu::OptionExt;
 use table::metadata::TableId;
 
 use crate::error::{self, Result};
-use crate::key::flow::flow_info::FlowInfoValue;
 use crate::key::flow::{flownode_addr_helper, FlowScoped};
 use crate::key::node_address::NodeAddressKey;
-use crate::key::{
-    BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue,
-};
+use crate::key::{BytesAdapter, FlowId, FlowPartitionId, MetadataKey, MetadataValue};
 use crate::kv_backend::txn::{Txn, TxnOp};
 use crate::kv_backend::KvBackendRef;
 use crate::peer::Peer;
@@ -218,7 +215,7 @@ impl TableFlowManager {
 
     /// Builds a create table flow transaction.
     ///
-    /// Puts `__flow/source_table/{table_id}/{node_id}/{flow_id}/{partition_id}` keys.
+    /// Puts `__flow/source_table/{table_id}/{node_id}/{partition_id}` keys.
     pub fn build_create_txn(
         &self,
         flow_id: FlowId,
@@ -242,44 +239,6 @@ impl TableFlowManager {
         Ok(Txn::new().and_then(txns))
     }
 
-    /// Builds a update table flow transaction.
-    ///
-    /// Puts `__flow/source_table/{table_id}/{node_id}/{flow_id}/{partition_id}` keys,
-    /// Also remove previous
-    /// `__flow/source_table/{table_id}/{old_node_id}/{flow_id}/{partition_id}` keys.
-    pub fn build_update_txn(
-        &self,
-        flow_id: FlowId,
-        current_flow_info: &DeserializedValueWithBytes<FlowInfoValue>,
-        table_flow_values: Vec<(FlowPartitionId, TableFlowValue)>,
-        source_table_ids: &[TableId],
-    ) -> Result<Txn> {
-        let mut txns = Vec::with_capacity(2 * source_table_ids.len() * table_flow_values.len());
-
-        // first remove the old keys
-        for (part_id, node_id) in current_flow_info.flownode_ids() {
-            for source_table_id in current_flow_info.source_table_ids() {
-                txns.push(TxnOp::Delete(
-                    TableFlowKey::new(*source_table_id, *node_id, flow_id, *part_id).to_bytes(),
-                ));
-            }
-        }
-
-        for (partition_id, table_flow_value) in table_flow_values {
-            let flownode_id = table_flow_value.peer.id;
-            let value = table_flow_value.try_as_raw_value()?;
-            for source_table_id in source_table_ids {
-                txns.push(TxnOp::Put(
-                    TableFlowKey::new(*source_table_id, flownode_id, flow_id, partition_id)
-                        .to_bytes(),
-                    value.clone(),
-                ));
-            }
-        }
-
-        Ok(Txn::new().and_then(txns))
-    }
-
     async fn remap_table_flow_addresses(
         &self,
         table_flows: &mut [(TableFlowKey, TableFlowValue)],

src/common/meta/src/key/table_route.rs

@@ -478,10 +478,11 @@ impl TableRouteStorage {
         ))
     }
 
+    // TODO(LFC): restore its original visibility after some test utility codes are refined
     /// Builds a update table route transaction,
     /// it expected the remote value equals the `current_table_route_value`.
     /// It retrieves the latest value if the comparing failed.
-    pub(crate) fn build_update_txn(
+    pub fn build_update_txn(
         &self,
         table_id: TableId,
         current_table_route_value: &DeserializedValueWithBytes<TableRouteValue>,

src/common/meta/src/key/txn_helper.rs

@@ -25,7 +25,7 @@ pub struct TxnOpGetResponseSet(Vec<KeyValue>);
 
 impl TxnOpGetResponseSet {
     /// Returns a filter to consume a [KeyValue] where the key equals `key`.
-    pub(crate) fn filter(key: Vec<u8>) -> impl FnMut(&mut TxnOpGetResponseSet) -> Option<Vec<u8>> {
+    pub fn filter(key: Vec<u8>) -> impl FnMut(&mut TxnOpGetResponseSet) -> Option<Vec<u8>> {
         move |set| {
             let pos = set.0.iter().position(|kv| kv.key == key);
             match pos {
@@ -36,7 +36,7 @@ impl TxnOpGetResponseSet {
     }
 
     /// Returns a decoder to decode bytes to `DeserializedValueWithBytes<T>`.
-    pub(crate) fn decode_with<F, T>(
+    pub fn decode_with<F, T>(
         mut f: F,
     ) -> impl FnMut(&mut TxnOpGetResponseSet) -> Result<Option<DeserializedValueWithBytes<T>>>
     where

src/common/meta/src/lib.rs

@@ -15,8 +15,6 @@
 #![feature(assert_matches)]
 #![feature(btree_extract_if)]
 #![feature(let_chains)]
-#![feature(extract_if)]
-#![feature(hash_extract_if)]
 
 pub mod cache;
 pub mod cache_invalidator;

src/common/meta/src/lock_key.rs

@@ -121,8 +121,8 @@ pub enum FlowNameLock {
 }
 
 impl FlowNameLock {
-    pub fn new(catalog: &str, table: &str) -> Self {
-        Self::Write(format!("{catalog}.{table}"))
+    pub fn new(catalog: &str, flow_name: &str) -> Self {
+        Self::Write(format!("{catalog}.{flow_name}"))
     }
 }
 

src/common/meta/src/rpc/router.rs

@@ -176,15 +176,12 @@ impl TableRoute {
                 })?
                 .into();
 
-            let leader_peer = peers
-                .get(region_route.leader_peer_index as usize)
-                .cloned()
-                .map(Into::into);
+            let leader_peer = peers.get(region_route.leader_peer_index as usize).cloned();
 
             let follower_peers = region_route
                 .follower_peer_indexes
                 .into_iter()
-                .filter_map(|x| peers.get(x as usize).cloned().map(Into::into))
+                .filter_map(|x| peers.get(x as usize).cloned())
                 .collect::<Vec<_>>();
 
             region_routes.push(RegionRoute {

src/common/meta/src/test_util.rs

@@ -20,8 +20,6 @@ use api::v1::region::{InsertRequests, RegionRequest};
 pub use common_base::AffectedRows;
 use common_query::request::QueryRequest;
 use common_recordbatch::SendableRecordBatchStream;
-use common_wal::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig};
-use common_wal::config::kafka::MetasrvKafkaConfig;
 
 use crate::cache_invalidator::DummyCacheInvalidator;
 use crate::ddl::flow_meta::FlowMetadataAllocator;
@@ -39,8 +37,7 @@ use crate::peer::{Peer, PeerLookupService};
 use crate::region_keeper::MemoryRegionKeeper;
 use crate::region_registry::LeaderRegionRegistry;
 use crate::sequence::SequenceBuilder;
-use crate::wal_options_allocator::topic_pool::KafkaTopicPool;
-use crate::wal_options_allocator::{build_kafka_topic_creator, WalOptionsAllocator};
+use crate::wal_options_allocator::WalOptionsAllocator;
 use crate::{DatanodeId, FlownodeId};
 
 #[async_trait::async_trait]
@@ -202,34 +199,3 @@ impl PeerLookupService for NoopPeerLookupService {
         Ok(Some(Peer::empty(id)))
     }
 }
-
-/// Create a kafka topic pool for testing.
-pub async fn test_kafka_topic_pool(
-    broker_endpoints: Vec<String>,
-    num_topics: usize,
-    auto_create_topics: bool,
-    topic_name_prefix: Option<&str>,
-) -> KafkaTopicPool {
-    let mut config = MetasrvKafkaConfig {
-        connection: KafkaConnectionConfig {
-            broker_endpoints,
-            ..Default::default()
-        },
-        kafka_topic: KafkaTopicConfig {
-            num_topics,
-
-            ..Default::default()
-        },
-        auto_create_topics,
-        ..Default::default()
-    };
-    if let Some(prefix) = topic_name_prefix {
-        config.kafka_topic.topic_name_prefix = prefix.to_string();
-    }
-    let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
-    let topic_creator = build_kafka_topic_creator(&config.connection, &config.kafka_topic)
-        .await
-        .unwrap();
-
-    KafkaTopicPool::new(&config, kv_backend, topic_creator)
-}

src/common/meta/src/wal_options_allocator.rs

@@ -112,9 +112,7 @@ pub async fn build_wal_options_allocator(
                 NAME_PATTERN_REGEX.is_match(prefix),
                 InvalidTopicNamePrefixSnafu { prefix }
             );
-            let topic_creator =
-                build_kafka_topic_creator(&kafka_config.connection, &kafka_config.kafka_topic)
-                    .await?;
+            let topic_creator = build_kafka_topic_creator(kafka_config).await?;
             let topic_pool = KafkaTopicPool::new(kafka_config, kv_backend, topic_creator);
             Ok(WalOptionsAllocator::Kafka(topic_pool))
         }
@@ -153,16 +151,13 @@ pub fn prepare_wal_options(
 mod tests {
     use std::assert_matches::assert_matches;
 
-    use common_wal::config::kafka::common::KafkaTopicConfig;
+    use common_wal::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig};
     use common_wal::config::kafka::MetasrvKafkaConfig;
-    use common_wal::maybe_skip_kafka_integration_test;
-    use common_wal::test_util::get_kafka_endpoints;
+    use common_wal::test_util::run_test_with_kafka_wal;
 
     use super::*;
     use crate::error::Error;
     use crate::kv_backend::memory::MemoryKvBackend;
-    use crate::test_util::test_kafka_topic_pool;
-    use crate::wal_options_allocator::selector::RoundRobinTopicSelector;
 
     // Tests that the wal options allocator could successfully allocate raft-engine wal options.
     #[tokio::test]
@@ -202,42 +197,55 @@ mod tests {
         assert_matches!(got, Error::InvalidTopicNamePrefix { .. });
     }
 
+    // Tests that the wal options allocator could successfully allocate Kafka wal options.
     #[tokio::test]
-    async fn test_allocator_with_kafka_allocate_wal_options() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let num_topics = 5;
-        let mut topic_pool = test_kafka_topic_pool(
-            get_kafka_endpoints(),
-            num_topics,
-            true,
-            Some("test_allocator_with_kafka"),
-        )
-        .await;
-        topic_pool.selector = Arc::new(RoundRobinTopicSelector::default());
-        let topics = topic_pool.topics.clone();
-        // clean up the topics before test
-        let topic_creator = topic_pool.topic_creator();
-        topic_creator.delete_topics(&topics).await.unwrap();
+    async fn test_allocator_with_kafka() {
+        run_test_with_kafka_wal(|broker_endpoints| {
+            Box::pin(async {
+                let topics = (0..256)
+                    .map(|i| format!("test_allocator_with_kafka_{}_{}", i, uuid::Uuid::new_v4()))
+                    .collect::<Vec<_>>();
 
-        // Creates an options allocator.
-        let allocator = WalOptionsAllocator::Kafka(topic_pool);
-        allocator.start().await.unwrap();
+                // Creates a topic manager.
+                let kafka_topic = KafkaTopicConfig {
+                    replication_factor: broker_endpoints.len() as i16,
+                    ..Default::default()
+                };
+                let config = MetasrvKafkaConfig {
+                    connection: KafkaConnectionConfig {
+                        broker_endpoints,
+                        ..Default::default()
+                    },
+                    kafka_topic,
+                    ..Default::default()
+                };
+                let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+                let topic_creator = build_kafka_topic_creator(&config).await.unwrap();
+                let mut topic_pool = KafkaTopicPool::new(&config, kv_backend, topic_creator);
+                topic_pool.topics.clone_from(&topics);
+                topic_pool.selector = Arc::new(selector::RoundRobinTopicSelector::default());
 
-        let num_regions = 3;
-        let regions = (0..num_regions).collect::<Vec<_>>();
-        let got = allocate_region_wal_options(regions.clone(), &allocator, false).unwrap();
+                // Creates an options allocator.
+                let allocator = WalOptionsAllocator::Kafka(topic_pool);
+                allocator.start().await.unwrap();
 
-        // Check the allocated wal options contain the expected topics.
-        let expected = (0..num_regions)
-            .map(|i| {
-                let options = WalOptions::Kafka(KafkaWalOptions {
-                    topic: topics[i as usize].clone(),
-                });
-                (i, serde_json::to_string(&options).unwrap())
+                let num_regions = 32;
+                let regions = (0..num_regions).collect::<Vec<_>>();
+                let got = allocate_region_wal_options(regions.clone(), &allocator, false).unwrap();
+
+                // Check the allocated wal options contain the expected topics.
+                let expected = (0..num_regions)
+                    .map(|i| {
+                        let options = WalOptions::Kafka(KafkaWalOptions {
+                            topic: topics[i as usize].clone(),
+                        });
+                        (i, serde_json::to_string(&options).unwrap())
+                    })
+                    .collect::<HashMap<_, _>>();
+                assert_eq!(got, expected);
             })
-            .collect::<HashMap<_, _>>();
-        assert_eq!(got, expected);
+        })
+        .await;
     }
 
     #[tokio::test]

src/common/meta/src/wal_options_allocator/topic_creator.rs

@@ -12,21 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_telemetry::{debug, error, info};
-use common_wal::config::kafka::common::{
-    KafkaConnectionConfig, KafkaTopicConfig, DEFAULT_BACKOFF_CONFIG,
-};
+use common_telemetry::{error, info};
+use common_wal::config::kafka::common::DEFAULT_BACKOFF_CONFIG;
+use common_wal::config::kafka::MetasrvKafkaConfig;
 use rskafka::client::error::Error as RsKafkaError;
 use rskafka::client::error::ProtocolError::TopicAlreadyExists;
-use rskafka::client::partition::{Compression, OffsetAt, PartitionClient, UnknownTopicHandling};
+use rskafka::client::partition::{Compression, UnknownTopicHandling};
 use rskafka::client::{Client, ClientBuilder};
 use rskafka::record::Record;
 use snafu::ResultExt;
 
 use crate::error::{
-    BuildKafkaClientSnafu, BuildKafkaCtrlClientSnafu, CreateKafkaWalTopicSnafu,
-    KafkaGetOffsetSnafu, KafkaPartitionClientSnafu, ProduceRecordSnafu, ResolveKafkaEndpointSnafu,
-    Result, TlsConfigSnafu,
+    BuildKafkaClientSnafu, BuildKafkaCtrlClientSnafu, BuildKafkaPartitionClientSnafu,
+    CreateKafkaWalTopicSnafu, ProduceRecordSnafu, ResolveKafkaEndpointSnafu, Result,
+    TlsConfigSnafu,
 };
 
 // Each topic only has one partition for now.
@@ -71,47 +70,21 @@ impl KafkaTopicCreator {
                     info!("The topic {} already exists", topic);
                     Ok(())
                 } else {
-                    error!(e; "Failed to create a topic {}", topic);
+                    error!("Failed to create a topic {}, error {:?}", topic, e);
                     Err(e).context(CreateKafkaWalTopicSnafu)
                 }
             }
         }
     }
 
-    async fn prepare_topic(&self, topic: &String) -> Result<()> {
-        let partition_client = self.partition_client(topic).await?;
-        self.append_noop_record(topic, &partition_client).await?;
-        Ok(())
-    }
-
-    /// Creates a [PartitionClient] for the given topic.
-    async fn partition_client(&self, topic: &str) -> Result<PartitionClient> {
-        self.client
+    async fn append_noop_record(&self, topic: &String, client: &Client) -> Result<()> {
+        let partition_client = client
             .partition_client(topic, DEFAULT_PARTITION, UnknownTopicHandling::Retry)
             .await
-            .context(KafkaPartitionClientSnafu {
+            .context(BuildKafkaPartitionClientSnafu {
                 topic,
                 partition: DEFAULT_PARTITION,
-            })
-    }
-
-    /// Appends a noop record to the topic.
-    /// It only appends a noop record if the topic is empty.
-    async fn append_noop_record(
-        &self,
-        topic: &String,
-        partition_client: &PartitionClient,
-    ) -> Result<()> {
-        let end_offset = partition_client
-            .get_offset(OffsetAt::Latest)
-            .await
-            .context(KafkaGetOffsetSnafu {
-                topic: topic.to_string(),
-                partition: DEFAULT_PARTITION,
             })?;
-        if end_offset > 0 {
-            return Ok(());
-        }
 
         partition_client
             .produce(
@@ -125,28 +98,22 @@ impl KafkaTopicCreator {
             )
             .await
             .context(ProduceRecordSnafu { topic })?;
-        debug!("Appended a noop record to topic {}", topic);
 
         Ok(())
     }
 
-    /// Creates topics in Kafka.
-    pub async fn create_topics(&self, topics: &[String]) -> Result<()> {
-        let tasks = topics
-            .iter()
-            .map(|topic| async { self.create_topic(topic, &self.client).await })
-            .collect::<Vec<_>>();
-        futures::future::try_join_all(tasks).await.map(|_| ())
-    }
-
     /// Prepares topics in Kafka.
-    ///
-    /// It appends a noop record to each topic if the topic is empty.
-    pub async fn prepare_topics(&self, topics: &[String]) -> Result<()> {
+    /// 1. Creates missing topics.
+    /// 2. Appends a noop record to each topic.
+    pub async fn prepare_topics(&self, topics: &[&String]) -> Result<()> {
         // Try to create missing topics.
         let tasks = topics
             .iter()
-            .map(|topic| async { self.prepare_topic(topic).await })
+            .map(|topic| async {
+                self.create_topic(topic, &self.client).await?;
+                self.append_noop_record(topic, &self.client).await?;
+                Ok(())
+            })
             .collect::<Vec<_>>();
         futures::future::try_join_all(tasks).await.map(|_| ())
     }
@@ -162,244 +129,34 @@ impl KafkaTopicCreator {
     }
 }
 
-#[cfg(test)]
-impl KafkaTopicCreator {
-    pub async fn delete_topics(&self, topics: &[String]) -> Result<()> {
-        let tasks = topics
-            .iter()
-            .map(|topic| async { self.delete_topic(topic, &self.client).await })
-            .collect::<Vec<_>>();
-        futures::future::try_join_all(tasks).await.map(|_| ())
-    }
-
-    async fn delete_topic(&self, topic: &String, client: &Client) -> Result<()> {
-        let controller = client
-            .controller_client()
-            .context(BuildKafkaCtrlClientSnafu)?;
-        match controller.delete_topic(topic, 10).await {
-            Ok(_) => {
-                info!("Successfully deleted topic {}", topic);
-                Ok(())
-            }
-            Err(e) => {
-                if Self::is_unknown_topic_err(&e) {
-                    info!("The topic {} does not exist", topic);
-                    Ok(())
-                } else {
-                    panic!("Failed to delete a topic {}, error: {}", topic, e);
-                }
-            }
-        }
-    }
-
-    fn is_unknown_topic_err(e: &RsKafkaError) -> bool {
-        matches!(
-            e,
-            &RsKafkaError::ServerError {
-                protocol_error: rskafka::client::error::ProtocolError::UnknownTopicOrPartition,
-                ..
-            }
-        )
-    }
-
-    pub async fn get_partition_client(&self, topic: &str) -> PartitionClient {
-        self.partition_client(topic).await.unwrap()
-    }
-}
 /// Builds a kafka [Client](rskafka::client::Client).
-pub async fn build_kafka_client(connection: &KafkaConnectionConfig) -> Result<Client> {
+pub async fn build_kafka_client(config: &MetasrvKafkaConfig) -> Result<Client> {
     // Builds an kafka controller client for creating topics.
-    let broker_endpoints = common_wal::resolve_to_ipv4(&connection.broker_endpoints)
+    let broker_endpoints = common_wal::resolve_to_ipv4(&config.connection.broker_endpoints)
         .await
         .context(ResolveKafkaEndpointSnafu)?;
     let mut builder = ClientBuilder::new(broker_endpoints).backoff_config(DEFAULT_BACKOFF_CONFIG);
-    if let Some(sasl) = &connection.sasl {
+    if let Some(sasl) = &config.connection.sasl {
         builder = builder.sasl_config(sasl.config.clone().into_sasl_config());
     };
-    if let Some(tls) = &connection.tls {
+    if let Some(tls) = &config.connection.tls {
         builder = builder.tls_config(tls.to_tls_config().await.context(TlsConfigSnafu)?)
     };
     builder
         .build()
         .await
         .with_context(|_| BuildKafkaClientSnafu {
-            broker_endpoints: connection.broker_endpoints.clone(),
+            broker_endpoints: config.connection.broker_endpoints.clone(),
         })
 }
 
 /// Builds a [KafkaTopicCreator].
-pub async fn build_kafka_topic_creator(
-    connection: &KafkaConnectionConfig,
-    kafka_topic: &KafkaTopicConfig,
-) -> Result<KafkaTopicCreator> {
-    let client = build_kafka_client(connection).await?;
+pub async fn build_kafka_topic_creator(config: &MetasrvKafkaConfig) -> Result<KafkaTopicCreator> {
+    let client = build_kafka_client(config).await?;
     Ok(KafkaTopicCreator {
         client,
-        num_partitions: kafka_topic.num_partitions,
-        replication_factor: kafka_topic.replication_factor,
-        create_topic_timeout: kafka_topic.create_topic_timeout.as_millis() as i32,
+        num_partitions: config.kafka_topic.num_partitions,
+        replication_factor: config.kafka_topic.replication_factor,
+        create_topic_timeout: config.kafka_topic.create_topic_timeout.as_millis() as i32,
     })
 }
-
-#[cfg(test)]
-mod tests {
-    use common_wal::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig};
-    use common_wal::maybe_skip_kafka_integration_test;
-    use common_wal::test_util::get_kafka_endpoints;
-
-    use super::*;
-
-    async fn test_topic_creator(broker_endpoints: Vec<String>) -> KafkaTopicCreator {
-        let connection = KafkaConnectionConfig {
-            broker_endpoints,
-            ..Default::default()
-        };
-        let kafka_topic = KafkaTopicConfig::default();
-
-        build_kafka_topic_creator(&connection, &kafka_topic)
-            .await
-            .unwrap()
-    }
-
-    async fn append_records(partition_client: &PartitionClient, num_records: usize) -> Result<()> {
-        for i in 0..num_records {
-            partition_client
-                .produce(
-                    vec![Record {
-                        key: Some(b"test".to_vec()),
-                        value: Some(format!("test {}", i).as_bytes().to_vec()),
-                        timestamp: chrono::Utc::now(),
-                        headers: Default::default(),
-                    }],
-                    Compression::Lz4,
-                )
-                .await
-                .unwrap();
-        }
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_append_noop_record_to_empty_topic() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let prefix = "append_noop_record_to_empty_topic";
-        let creator = test_topic_creator(get_kafka_endpoints()).await;
-
-        let topic = format!("{}{}", prefix, "0");
-        // Clean up the topics before test
-        creator.delete_topics(&[topic.to_string()]).await.unwrap();
-        creator.create_topics(&[topic.to_string()]).await.unwrap();
-
-        let partition_client = creator.partition_client(&topic).await.unwrap();
-        let end_offset = partition_client.get_offset(OffsetAt::Latest).await.unwrap();
-        assert_eq!(end_offset, 0);
-
-        // The topic is not empty, so no noop record is appended.
-        creator
-            .append_noop_record(&topic, &partition_client)
-            .await
-            .unwrap();
-        let end_offset = partition_client.get_offset(OffsetAt::Latest).await.unwrap();
-        assert_eq!(end_offset, 1);
-    }
-
-    #[tokio::test]
-    async fn test_append_noop_record_to_non_empty_topic() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let prefix = "append_noop_record_to_non_empty_topic";
-        let creator = test_topic_creator(get_kafka_endpoints()).await;
-
-        let topic = format!("{}{}", prefix, "0");
-        // Clean up the topics before test
-        creator.delete_topics(&[topic.to_string()]).await.unwrap();
-
-        creator.create_topics(&[topic.to_string()]).await.unwrap();
-        let partition_client = creator.partition_client(&topic).await.unwrap();
-        append_records(&partition_client, 2).await.unwrap();
-
-        let end_offset = partition_client.get_offset(OffsetAt::Latest).await.unwrap();
-        assert_eq!(end_offset, 2);
-
-        // The topic is not empty, so no noop record is appended.
-        creator
-            .append_noop_record(&topic, &partition_client)
-            .await
-            .unwrap();
-        let end_offset = partition_client.get_offset(OffsetAt::Latest).await.unwrap();
-        assert_eq!(end_offset, 2);
-    }
-
-    #[tokio::test]
-    async fn test_create_topic() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let prefix = "create_topic";
-        let creator = test_topic_creator(get_kafka_endpoints()).await;
-
-        let topic = format!("{}{}", prefix, "0");
-        // Clean up the topics before test
-        creator.delete_topics(&[topic.to_string()]).await.unwrap();
-
-        creator.create_topics(&[topic.to_string()]).await.unwrap();
-        // Should be ok
-        creator.create_topics(&[topic.to_string()]).await.unwrap();
-
-        let partition_client = creator.partition_client(&topic).await.unwrap();
-        let end_offset = partition_client.get_offset(OffsetAt::Latest).await.unwrap();
-        assert_eq!(end_offset, 0);
-    }
-
-    #[tokio::test]
-    async fn test_prepare_topic() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let prefix = "prepare_topic";
-        let creator = test_topic_creator(get_kafka_endpoints()).await;
-
-        let topic = format!("{}{}", prefix, "0");
-        // Clean up the topics before test
-        creator.delete_topics(&[topic.to_string()]).await.unwrap();
-
-        creator.create_topics(&[topic.to_string()]).await.unwrap();
-        creator.prepare_topic(&topic).await.unwrap();
-
-        let partition_client = creator.partition_client(&topic).await.unwrap();
-        let start_offset = partition_client
-            .get_offset(OffsetAt::Earliest)
-            .await
-            .unwrap();
-        assert_eq!(start_offset, 0);
-
-        let end_offset = partition_client.get_offset(OffsetAt::Latest).await.unwrap();
-        assert_eq!(end_offset, 1);
-    }
-
-    #[tokio::test]
-    async fn test_prepare_topic_with_stale_records_without_pruning() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-
-        let prefix = "prepare_topic_with_stale_records_without_pruning";
-        let creator = test_topic_creator(get_kafka_endpoints()).await;
-
-        let topic = format!("{}{}", prefix, "0");
-        // Clean up the topics before test
-        creator.delete_topics(&[topic.to_string()]).await.unwrap();
-
-        creator.create_topics(&[topic.to_string()]).await.unwrap();
-        let partition_client = creator.partition_client(&topic).await.unwrap();
-        append_records(&partition_client, 10).await.unwrap();
-
-        creator.prepare_topic(&topic).await.unwrap();
-
-        let end_offset = partition_client.get_offset(OffsetAt::Latest).await.unwrap();
-        assert_eq!(end_offset, 10);
-        let start_offset = partition_client
-            .get_offset(OffsetAt::Earliest)
-            .await
-            .unwrap();
-        assert_eq!(start_offset, 0);
-    }
-}

src/common/meta/src/wal_options_allocator/topic_manager.rs

@@ -40,21 +40,24 @@ impl KafkaTopicManager {
         Ok(topics)
     }
 
-    /// Returns the topics that are not prepared.
-    pub async fn unprepare_topics(&self, all_topics: &[String]) -> Result<Vec<String>> {
+    /// Restores topics from the key-value backend. and returns the topics that are not stored in kvbackend.
+    pub async fn get_topics_to_create<'a>(
+        &self,
+        all_topics: &'a [String],
+    ) -> Result<Vec<&'a String>> {
         let existing_topics = self.restore_topics().await?;
         let existing_topic_set = existing_topics.iter().collect::<HashSet<_>>();
         let mut topics_to_create = Vec::with_capacity(all_topics.len());
         for topic in all_topics {
             if !existing_topic_set.contains(topic) {
-                topics_to_create.push(topic.to_string());
+                topics_to_create.push(topic);
             }
         }
         Ok(topics_to_create)
     }
 
-    /// Persists prepared topics into the key-value backend.
-    pub async fn persist_prepared_topics(&self, topics: &[String]) -> Result<()> {
+    /// Persists topics into the key-value backend.
+    pub async fn persist_topics(&self, topics: &[String]) -> Result<()> {
         self.topic_name_manager
             .batch_put(
                 topics
@@ -67,14 +70,6 @@ impl KafkaTopicManager {
     }
 }
 
-#[cfg(test)]
-impl KafkaTopicManager {
-    /// Lists all topics in the key-value backend.
-    pub async fn list_topics(&self) -> Result<Vec<String>> {
-        self.topic_name_manager.range().await
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
@@ -95,11 +90,11 @@ mod tests {
 
         // No legacy topics.
         let mut topics_to_be_created = topic_kvbackend_manager
-            .unprepare_topics(&all_topics)
+            .get_topics_to_create(&all_topics)
             .await
             .unwrap();
         topics_to_be_created.sort();
-        let mut expected = all_topics.clone();
+        let mut expected = all_topics.iter().collect::<Vec<_>>();
         expected.sort();
         assert_eq!(expected, topics_to_be_created);
 
@@ -114,7 +109,7 @@ mod tests {
         assert!(res.prev_kv.is_none());
 
         let topics_to_be_created = topic_kvbackend_manager
-            .unprepare_topics(&all_topics)
+            .get_topics_to_create(&all_topics)
             .await
             .unwrap();
         assert!(topics_to_be_created.is_empty());
@@ -149,21 +144,21 @@ mod tests {
         let topic_kvbackend_manager = KafkaTopicManager::new(kv_backend);
 
         let mut topics_to_be_created = topic_kvbackend_manager
-            .unprepare_topics(&all_topics)
+            .get_topics_to_create(&all_topics)
             .await
             .unwrap();
         topics_to_be_created.sort();
-        let mut expected = all_topics.clone();
+        let mut expected = all_topics.iter().collect::<Vec<_>>();
         expected.sort();
         assert_eq!(expected, topics_to_be_created);
 
         // Persists topics to kv backend.
         topic_kvbackend_manager
-            .persist_prepared_topics(&all_topics)
+            .persist_topics(&all_topics)
             .await
             .unwrap();
         let topics_to_be_created = topic_kvbackend_manager
-            .unprepare_topics(&all_topics)
+            .get_topics_to_create(&all_topics)
             .await
             .unwrap();
         assert!(topics_to_be_created.is_empty());

src/common/meta/src/wal_options_allocator/topic_pool.rs

@@ -15,7 +15,6 @@
 use std::fmt::{self, Formatter};
 use std::sync::Arc;
 
-use common_telemetry::info;
 use common_wal::config::kafka::MetasrvKafkaConfig;
 use common_wal::TopicSelectorType;
 use snafu::ensure;
@@ -78,35 +77,27 @@ impl KafkaTopicPool {
     }
 
     /// Tries to activate the topic manager when metasrv becomes the leader.
-    ///
     /// First tries to restore persisted topics from the kv backend.
-    /// If there are unprepared topics (topics that exist in the configuration but not in the kv backend),
-    /// it will create these topics in Kafka if `auto_create_topics` is enabled.
-    ///
-    /// Then it prepares all unprepared topics by appending a noop record if the topic is empty,
-    /// and persists them in the kv backend for future use.
+    /// If not enough topics retrieved, it will try to contact the Kafka cluster and request creating more topics.
     pub async fn activate(&self) -> Result<()> {
+        if !self.auto_create_topics {
+            return Ok(());
+        }
+
         let num_topics = self.topics.len();
         ensure!(num_topics > 0, InvalidNumTopicsSnafu { num_topics });
 
-        let unprepared_topics = self.topic_manager.unprepare_topics(&self.topics).await?;
+        let topics_to_be_created = self
+            .topic_manager
+            .get_topics_to_create(&self.topics)
+            .await?;
 
-        if !unprepared_topics.is_empty() {
-            if self.auto_create_topics {
-                info!("Creating {} topics", unprepared_topics.len());
-                self.topic_creator.create_topics(&unprepared_topics).await?;
-            } else {
-                info!("Auto create topics is disabled, skipping topic creation.");
-            }
+        if !topics_to_be_created.is_empty() {
             self.topic_creator
-                .prepare_topics(&unprepared_topics)
-                .await?;
-            self.topic_manager
-                .persist_prepared_topics(&unprepared_topics)
+                .prepare_topics(&topics_to_be_created)
                 .await?;
+            self.topic_manager.persist_topics(&self.topics).await?;
         }
-        info!("Activated topic pool with {} topics", self.topics.len());
-
         Ok(())
     }
 
@@ -123,147 +114,77 @@ impl KafkaTopicPool {
     }
 }
 
-#[cfg(test)]
-impl KafkaTopicPool {
-    pub(crate) fn topic_manager(&self) -> &KafkaTopicManager {
-        &self.topic_manager
-    }
-
-    pub(crate) fn topic_creator(&self) -> &KafkaTopicCreator {
-        &self.topic_creator
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use std::assert_matches::assert_matches;
-
-    use common_wal::maybe_skip_kafka_integration_test;
-    use common_wal::test_util::get_kafka_endpoints;
+    use common_wal::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig};
+    use common_wal::test_util::run_test_with_kafka_wal;
 
     use super::*;
-    use crate::error::Error;
-    use crate::test_util::test_kafka_topic_pool;
-    use crate::wal_options_allocator::selector::RoundRobinTopicSelector;
-
-    #[tokio::test]
-    async fn test_pool_invalid_number_topics_err() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let endpoints = get_kafka_endpoints();
-
-        let pool = test_kafka_topic_pool(endpoints.clone(), 0, false, None).await;
-        let err = pool.activate().await.unwrap_err();
-        assert_matches!(err, Error::InvalidNumTopics { .. });
-
-        let pool = test_kafka_topic_pool(endpoints, 0, true, None).await;
-        let err = pool.activate().await.unwrap_err();
-        assert_matches!(err, Error::InvalidNumTopics { .. });
-    }
-
-    #[tokio::test]
-    async fn test_pool_activate_unknown_topics_err() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let pool =
-            test_kafka_topic_pool(get_kafka_endpoints(), 1, false, Some("unknown_topic")).await;
-        let err = pool.activate().await.unwrap_err();
-        assert_matches!(err, Error::KafkaPartitionClient { .. });
-    }
-
-    #[tokio::test]
-    async fn test_pool_activate() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let pool =
-            test_kafka_topic_pool(get_kafka_endpoints(), 2, true, Some("pool_activate")).await;
-        // clean up the topics before test
-        let topic_creator = pool.topic_creator();
-        topic_creator.delete_topics(&pool.topics).await.unwrap();
-
-        let topic_manager = pool.topic_manager();
-        pool.activate().await.unwrap();
-        let topics = topic_manager.list_topics().await.unwrap();
-        assert_eq!(topics.len(), 2);
-    }
-
-    #[tokio::test]
-    async fn test_pool_activate_with_existing_topics() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let prefix = "pool_activate_with_existing_topics";
-        let pool = test_kafka_topic_pool(get_kafka_endpoints(), 2, true, Some(prefix)).await;
-        let topic_creator = pool.topic_creator();
-        topic_creator.delete_topics(&pool.topics).await.unwrap();
-
-        let topic_manager = pool.topic_manager();
-        // persists one topic info, then pool.activate() will create new topics that not persisted.
-        topic_manager
-            .persist_prepared_topics(&pool.topics[0..1])
-            .await
-            .unwrap();
-
-        pool.activate().await.unwrap();
-        let topics = topic_manager.list_topics().await.unwrap();
-        assert_eq!(topics.len(), 2);
-
-        let client = pool.topic_creator().client();
-        let topics = client
-            .list_topics()
-            .await
-            .unwrap()
-            .into_iter()
-            .filter(|t| t.name.starts_with(prefix))
-            .collect::<Vec<_>>();
-        assert_eq!(topics.len(), 1);
-    }
+    use crate::kv_backend::memory::MemoryKvBackend;
+    use crate::wal_options_allocator::topic_creator::build_kafka_topic_creator;
 
     /// Tests that the topic manager could allocate topics correctly.
     #[tokio::test]
     async fn test_alloc_topics() {
-        common_telemetry::init_default_ut_logging();
-        maybe_skip_kafka_integration_test!();
-        let num_topics = 5;
-        let mut topic_pool = test_kafka_topic_pool(
-            get_kafka_endpoints(),
-            num_topics,
-            true,
-            Some("test_allocator_with_kafka"),
-        )
+        run_test_with_kafka_wal(|broker_endpoints| {
+            Box::pin(async {
+                // Constructs topics that should be created.
+                let topics = (0..256)
+                    .map(|i| format!("test_alloc_topics_{}_{}", i, uuid::Uuid::new_v4()))
+                    .collect::<Vec<_>>();
+
+                // Creates a topic manager.
+                let kafka_topic = KafkaTopicConfig {
+                    replication_factor: broker_endpoints.len() as i16,
+                    ..Default::default()
+                };
+                let config = MetasrvKafkaConfig {
+                    connection: KafkaConnectionConfig {
+                        broker_endpoints,
+                        ..Default::default()
+                    },
+                    kafka_topic,
+                    ..Default::default()
+                };
+                let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+                let topic_creator = build_kafka_topic_creator(&config).await.unwrap();
+                let mut topic_pool = KafkaTopicPool::new(&config, kv_backend, topic_creator);
+                // Replaces the default topic pool with the constructed topics.
+                topic_pool.topics.clone_from(&topics);
+                // Replaces the default selector with a round-robin selector without shuffled.
+                topic_pool.selector = Arc::new(RoundRobinTopicSelector::default());
+                topic_pool.activate().await.unwrap();
+
+                // Selects exactly the number of `num_topics` topics one by one.
+                let got = (0..topics.len())
+                    .map(|_| topic_pool.select().unwrap())
+                    .cloned()
+                    .collect::<Vec<_>>();
+                assert_eq!(got, topics);
+
+                // Selects exactly the number of `num_topics` topics in a batching manner.
+                let got = topic_pool
+                    .select_batch(topics.len())
+                    .unwrap()
+                    .into_iter()
+                    .map(ToString::to_string)
+                    .collect::<Vec<_>>();
+                assert_eq!(got, topics);
+
+                // Selects more than the number of `num_topics` topics.
+                let got = topic_pool
+                    .select_batch(2 * topics.len())
+                    .unwrap()
+                    .into_iter()
+                    .map(ToString::to_string)
+                    .collect::<Vec<_>>();
+                let expected = vec![topics.clone(); 2]
+                    .into_iter()
+                    .flatten()
+                    .collect::<Vec<_>>();
+                assert_eq!(got, expected);
+            })
+        })
         .await;
-        topic_pool.selector = Arc::new(RoundRobinTopicSelector::default());
-        let topics = topic_pool.topics.clone();
-        // clean up the topics before test
-        let topic_creator = topic_pool.topic_creator();
-        topic_creator.delete_topics(&topics).await.unwrap();
-
-        // Selects exactly the number of `num_topics` topics one by one.
-        let got = (0..topics.len())
-            .map(|_| topic_pool.select().unwrap())
-            .cloned()
-            .collect::<Vec<_>>();
-        assert_eq!(got, topics);
-
-        // Selects exactly the number of `num_topics` topics in a batching manner.
-        let got = topic_pool
-            .select_batch(topics.len())
-            .unwrap()
-            .into_iter()
-            .map(ToString::to_string)
-            .collect::<Vec<_>>();
-        assert_eq!(got, topics);
-
-        // Selects more than the number of `num_topics` topics.
-        let got = topic_pool
-            .select_batch(2 * topics.len())
-            .unwrap()
-            .into_iter()
-            .map(ToString::to_string)
-            .collect::<Vec<_>>();
-        let expected = vec![topics.clone(); 2]
-            .into_iter()
-            .flatten()
-            .collect::<Vec<_>>();
-        assert_eq!(got, expected);
     }
 }

src/common/procedure-test/src/lib.rs

@@ -18,11 +18,13 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use async_trait::async_trait;
+use common_procedure::local::{acquire_dynamic_key_lock, DynamicKeyLockGuard};
+use common_procedure::rwlock::KeyRwLock;
 use common_procedure::store::poison_store::PoisonStore;
 use common_procedure::test_util::InMemoryPoisonStore;
 use common_procedure::{
     Context, ContextProvider, Output, PoisonKey, Procedure, ProcedureId, ProcedureState,
-    ProcedureWithId, Result, Status,
+    ProcedureWithId, Result, Status, StringKey,
 };
 
 /// A Mock [ContextProvider].
@@ -30,6 +32,7 @@ use common_procedure::{
 pub struct MockContextProvider {
     states: HashMap<ProcedureId, ProcedureState>,
     poison_manager: InMemoryPoisonStore,
+    dynamic_key_lock: Arc<KeyRwLock<String>>,
 }
 
 impl MockContextProvider {
@@ -38,6 +41,7 @@ impl MockContextProvider {
         MockContextProvider {
             states,
             poison_manager: InMemoryPoisonStore::default(),
+            dynamic_key_lock: Arc::new(KeyRwLock::new()),
         }
     }
 
@@ -58,6 +62,10 @@ impl ContextProvider for MockContextProvider {
             .try_put_poison(key.to_string(), procedure_id.to_string())
             .await
     }
+
+    async fn acquire_lock(&self, key: &StringKey) -> DynamicKeyLockGuard {
+        acquire_dynamic_key_lock(&self.dynamic_key_lock, key).await
+    }
 }
 
 /// Executes a procedure until it returns [Status::Done].

src/common/procedure/src/lib.rs

@@ -20,6 +20,7 @@ pub mod error;
 pub mod local;
 pub mod options;
 mod procedure;
+pub mod rwlock;
 pub mod store;
 pub mod watcher;
 
@@ -28,8 +29,8 @@ pub mod test_util;
 
 pub use crate::error::{Error, Result};
 pub use crate::procedure::{
-    BoxedProcedure, BoxedProcedureLoader, Context, ContextProvider, LockKey, Output, ParseIdError,
-    PoisonKey, PoisonKeys, Procedure, ProcedureId, ProcedureInfo, ProcedureManager,
-    ProcedureManagerRef, ProcedureState, ProcedureWithId, Status, StringKey,
+    BoxedProcedure, BoxedProcedureLoader, Context, ContextProvider, ContextProviderRef, LockKey,
+    Output, ParseIdError, PoisonKey, PoisonKeys, Procedure, ProcedureId, ProcedureInfo,
+    ProcedureManager, ProcedureManagerRef, ProcedureState, ProcedureWithId, Status, StringKey,
 };
 pub use crate::watcher::Watcher;

src/common/procedure/src/local.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 mod runner;
-mod rwlock;
 
 use std::collections::hash_map::Entry;
 use std::collections::{HashMap, HashSet, VecDeque};
@@ -30,7 +29,6 @@ use snafu::{ensure, OptionExt, ResultExt};
 use tokio::sync::watch::{self, Receiver, Sender};
 use tokio::sync::{Mutex as TokioMutex, Notify};
 
-use self::rwlock::KeyRwLock;
 use crate::error::{
     self, DuplicateProcedureSnafu, Error, LoaderConflictSnafu, ManagerNotStartSnafu,
     PoisonKeyNotDefinedSnafu, ProcedureNotFoundSnafu, Result, StartRemoveOutdatedMetaTaskSnafu,
@@ -38,11 +36,12 @@ use crate::error::{
 };
 use crate::local::runner::Runner;
 use crate::procedure::{BoxedProcedureLoader, InitProcedureState, PoisonKeys, ProcedureInfo};
+use crate::rwlock::{KeyRwLock, OwnedKeyRwLockGuard};
 use crate::store::poison_store::PoisonStoreRef;
 use crate::store::{ProcedureMessage, ProcedureMessages, ProcedureStore, StateStoreRef};
 use crate::{
     BoxedProcedure, ContextProvider, LockKey, PoisonKey, ProcedureId, ProcedureManager,
-    ProcedureState, ProcedureWithId, Watcher,
+    ProcedureState, ProcedureWithId, StringKey, Watcher,
 };
 
 /// The expired time of a procedure's metadata.
@@ -157,12 +156,80 @@ struct LoadedProcedure {
     step: u32,
 }
 
+/// The dynamic lock for procedure execution.
+///
+/// Unlike the procedure-level locks, these locks are acquired dynamically by the procedure
+/// during execution. They are only held when the procedure specifically needs these keys
+/// and are released as soon as the procedure no longer needs them.
+/// This allows for more fine-grained concurrency control during procedure execution.
+pub(crate) type DynamicKeyLock = Arc<KeyRwLock<String>>;
+
+/// Acquires a dynamic key lock for the given key.
+///
+/// This function takes a reference to the dynamic key lock and a pointer to the key.
+/// It then matches the key type and acquires the appropriate lock.
+pub async fn acquire_dynamic_key_lock(
+    lock: &DynamicKeyLock,
+    key: &StringKey,
+) -> DynamicKeyLockGuard {
+    match key {
+        StringKey::Share(key) => {
+            let guard = lock.read(key.to_string()).await;
+            DynamicKeyLockGuard {
+                guard: Some(OwnedKeyRwLockGuard::from(guard)),
+                key: key.to_string(),
+                lock: lock.clone(),
+            }
+        }
+        StringKey::Exclusive(key) => {
+            let guard = lock.write(key.to_string()).await;
+            DynamicKeyLockGuard {
+                guard: Some(OwnedKeyRwLockGuard::from(guard)),
+                key: key.to_string(),
+                lock: lock.clone(),
+            }
+        }
+    }
+}
+/// A guard for the dynamic key lock.
+///
+/// This guard is used to release the lock when the procedure no longer needs it.
+/// It also ensures that the lock is cleaned up when the guard is dropped.
+pub struct DynamicKeyLockGuard {
+    guard: Option<OwnedKeyRwLockGuard>,
+    key: String,
+    lock: DynamicKeyLock,
+}
+
+impl Drop for DynamicKeyLockGuard {
+    fn drop(&mut self) {
+        if let Some(guard) = self.guard.take() {
+            drop(guard);
+        }
+        self.lock.clean_keys(&[self.key.to_string()]);
+    }
+}
+
 /// Shared context of the manager.
 pub(crate) struct ManagerContext {
     /// Procedure loaders. The key is the type name of the procedure which the loader returns.
     loaders: Mutex<HashMap<String, BoxedProcedureLoader>>,
+    /// The key lock for the procedure.
+    ///
+    /// The lock keys are defined in `Procedure::lock_key()`.
+    /// These locks are acquired before the procedure starts and released after the procedure finishes.
+    /// They ensure exclusive access to resources throughout the entire procedure lifecycle.
     key_lock: KeyRwLock<String>,
+    /// The dynamic lock for procedure execution.
+    ///
+    /// Unlike the procedure-level locks, these locks are acquired dynamically by the procedure
+    /// during execution. They are only held when the procedure specifically needs these keys
+    /// and are released as soon as the procedure no longer needs them.
+    /// This allows for more fine-grained concurrency control during procedure execution.
+    dynamic_key_lock: DynamicKeyLock,
+    /// Procedures in the manager.
     procedures: RwLock<HashMap<ProcedureId, ProcedureMetaRef>>,
+    /// Running procedures.
     running_procedures: Mutex<HashSet<ProcedureId>>,
     /// Ids and finished time of finished procedures.
     finished_procedures: Mutex<VecDeque<(ProcedureId, Instant)>>,
@@ -199,6 +266,10 @@ impl ContextProvider for ManagerContext {
         let procedure_id = procedure_id.to_string();
         self.poison_manager.try_put_poison(key, procedure_id).await
     }
+
+    async fn acquire_lock(&self, key: &StringKey) -> DynamicKeyLockGuard {
+        acquire_dynamic_key_lock(&self.dynamic_key_lock, key).await
+    }
 }
 
 impl ManagerContext {
@@ -206,6 +277,7 @@ impl ManagerContext {
     fn new(poison_manager: PoisonStoreRef) -> ManagerContext {
         ManagerContext {
             key_lock: KeyRwLock::new(),
+            dynamic_key_lock: Arc::new(KeyRwLock::new()),
             loaders: Mutex::new(HashMap::new()),
             procedures: RwLock::new(HashMap::new()),
             running_procedures: Mutex::new(HashSet::new()),

src/common/procedure/src/local/runner.rs

@@ -23,9 +23,9 @@ use snafu::ResultExt;
 use tokio::time;
 
 use crate::error::{self, ProcedurePanicSnafu, Result, RollbackTimesExceededSnafu};
-use crate::local::rwlock::OwnedKeyRwLockGuard;
 use crate::local::{ManagerContext, ProcedureMeta, ProcedureMetaRef};
 use crate::procedure::{Output, StringKey};
+use crate::rwlock::OwnedKeyRwLockGuard;
 use crate::store::{ProcedureMessage, ProcedureStore};
 use crate::{
     BoxedProcedure, Context, Error, Procedure, ProcedureId, ProcedureState, ProcedureWithId, Status,
@@ -581,6 +581,7 @@ impl Runner {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::atomic::{AtomicU64, Ordering};
     use std::sync::Arc;
 
     use async_trait::async_trait;
@@ -588,13 +589,14 @@ mod tests {
     use common_error::mock::MockError;
     use common_error::status_code::StatusCode;
     use common_test_util::temp_dir::create_temp_dir;
+    use futures::future::join_all;
     use futures_util::future::BoxFuture;
     use futures_util::FutureExt;
     use object_store::{EntryMode, ObjectStore};
     use tokio::sync::mpsc;
 
     use super::*;
-    use crate::local::test_util;
+    use crate::local::{test_util, DynamicKeyLockGuard};
     use crate::procedure::PoisonKeys;
     use crate::store::proc_path;
     use crate::test_util::InMemoryPoisonStore;
@@ -666,6 +668,10 @@ mod tests {
             ) -> Result<()> {
                 unimplemented!()
             }
+
+            async fn acquire_lock(&self, _key: &StringKey) -> DynamicKeyLockGuard {
+                unimplemented!()
+            }
         }
 
         Context {
@@ -1674,4 +1680,66 @@ mod tests {
         // If the procedure is poisoned, the poison key shouldn't be deleted.
         assert_eq!(procedure_id, ROOT_ID);
     }
+
+    fn test_procedure_with_dynamic_lock(
+        shared_atomic_value: Arc<AtomicU64>,
+        id: u64,
+    ) -> (BoxedProcedure, Arc<ProcedureMeta>) {
+        let exec_fn = move |ctx: Context| {
+            let moved_shared_atomic_value = shared_atomic_value.clone();
+            let moved_ctx = ctx.clone();
+            async move {
+                debug!("Acquiring write lock, id: {}", id);
+                let key = StringKey::Exclusive("test_lock".to_string());
+                let guard = moved_ctx.provider.acquire_lock(&key).await;
+                debug!("Acquired write lock, id: {}", id);
+                let millis = rand::rng().random_range(10..=50);
+                tokio::time::sleep(Duration::from_millis(millis)).await;
+                let value = moved_shared_atomic_value.load(Ordering::Relaxed);
+                moved_shared_atomic_value.store(value + 1, Ordering::Relaxed);
+                debug!("Dropping write lock, id: {}", id);
+                drop(guard);
+
+                Ok(Status::done())
+            }
+            .boxed()
+        };
+
+        let adapter = ProcedureAdapter {
+            data: "dynamic_lock".to_string(),
+            lock_key: LockKey::new_exclusive([]),
+            poison_keys: PoisonKeys::new([]),
+            exec_fn,
+            rollback_fn: None,
+        };
+        let meta = adapter.new_meta(ROOT_ID);
+
+        (Box::new(adapter), meta)
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_execute_with_dynamic_lock() {
+        common_telemetry::init_default_ut_logging();
+        let shared_atomic_value = Arc::new(AtomicU64::new(0));
+        let (procedure1, meta1) = test_procedure_with_dynamic_lock(shared_atomic_value.clone(), 1);
+        let (procedure2, meta2) = test_procedure_with_dynamic_lock(shared_atomic_value.clone(), 2);
+
+        let dir = create_temp_dir("dynamic_lock");
+        let object_store = test_util::new_object_store(&dir);
+        let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
+        let mut runner1 = new_runner(meta1.clone(), procedure1, procedure_store.clone());
+        let mut runner2 = new_runner(meta2.clone(), procedure2, procedure_store.clone());
+        let ctx1 = context_with_provider(
+            meta1.id,
+            runner1.manager_ctx.clone() as Arc<dyn ContextProvider>,
+        );
+        let ctx2 = context_with_provider(
+            meta2.id,
+            // use same manager ctx as runner1
+            runner1.manager_ctx.clone() as Arc<dyn ContextProvider>,
+        );
+        let tasks = [runner1.execute_once(&ctx1), runner2.execute_once(&ctx2)];
+        join_all(tasks).await;
+        assert_eq!(shared_atomic_value.load(Ordering::Relaxed), 2);
+    }
 }

src/common/procedure/src/procedure.rs

@@ -25,6 +25,7 @@ use snafu::{ResultExt, Snafu};
 use uuid::Uuid;
 
 use crate::error::{self, Error, Result};
+use crate::local::DynamicKeyLockGuard;
 use crate::watcher::Watcher;
 
 pub type Output = Arc<dyn Any + Send + Sync>;
@@ -144,6 +145,9 @@ pub trait ContextProvider: Send + Sync {
     /// This method is used to mark a resource as being operated on by a procedure.
     /// If the poison key already exists with a different value, the operation will fail.
     async fn try_put_poison(&self, key: &PoisonKey, procedure_id: ProcedureId) -> Result<()>;
+
+    /// Acquires a key lock for the procedure.
+    async fn acquire_lock(&self, key: &StringKey) -> DynamicKeyLockGuard;
 }
 
 /// Reference-counted pointer to [ContextProvider].

src/common/procedure/src/rwlock.rs

@@ -18,8 +18,18 @@ use std::sync::{Arc, Mutex};
 
 use tokio::sync::{OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock};
 
+/// A guard that owns a read or write lock on a key.
+///
+/// This enum wraps either a read or write lock guard obtained from a `KeyRwLock`.
+/// The guard is automatically released when it is dropped.
 pub enum OwnedKeyRwLockGuard {
+    /// Represents a shared read lock on a key.
+    /// Multiple read locks can be held simultaneously for the same key.
     Read { _guard: OwnedRwLockReadGuard<()> },
+
+    /// Represents an exclusive write lock on a key.
+    /// Only one write lock can be held at a time for a given key,
+    /// and no read locks can be held simultaneously with a write lock.
     Write { _guard: OwnedRwLockWriteGuard<()> },
 }
 
@@ -36,7 +46,7 @@ impl From<OwnedRwLockWriteGuard<()>> for OwnedKeyRwLockGuard {
 }
 
 /// Locks based on a key, allowing other keys to lock independently.
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct KeyRwLock<K> {
     /// The inner map of locks for specific keys.
     inner: Mutex<HashMap<K, Arc<RwLock<()>>>>,

src/common/query/src/logical_plan/accumulator.rs

@@ -24,7 +24,7 @@ use datatypes::prelude::*;
 use datatypes::vectors::{Helper as VectorHelper, VectorRef};
 use snafu::ResultExt;
 
-use crate::error::{self, Error, FromScalarValueSnafu, IntoVectorSnafu, Result};
+use crate::error::{self, FromScalarValueSnafu, IntoVectorSnafu, Result};
 use crate::prelude::*;
 
 pub type AggregateFunctionCreatorRef = Arc<dyn AggregateFunctionCreator>;
@@ -166,8 +166,7 @@ impl DfAccumulator for DfAccumulatorAdaptor {
         let output_type = self.creator.output_type()?;
         let scalar_value = value
             .try_to_scalar_value(&output_type)
-            .context(error::ToScalarValueSnafu)
-            .map_err(Error::from)?;
+            .context(error::ToScalarValueSnafu)?;
         Ok(scalar_value)
     }
 

src/common/query/src/request.rs

@@ -17,6 +17,7 @@ use datafusion_expr::LogicalPlan;
 use store_api::storage::RegionId;
 
 /// The query request to be handled by the RegionServer (Datanode).
+#[derive(Clone, Debug)]
 pub struct QueryRequest {
     /// The header of this request. Often to store some context of the query. None means all to defaults.
     pub header: Option<RegionRequestHeader>,

src/common/wal/src/config/kafka/common.rs

@@ -23,16 +23,11 @@ use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 
 /// The default backoff config for kafka client.
-///
-/// If the operation fails, the client will retry 3 times.
-/// The backoff time is 100ms, 300ms, 900ms.
 pub const DEFAULT_BACKOFF_CONFIG: BackoffConfig = BackoffConfig {
     init_backoff: Duration::from_millis(100),
-    max_backoff: Duration::from_secs(1),
-    base: 3.0,
-    // The deadline shouldn't be too long,
-    // otherwise the client will block the worker loop for a long time.
-    deadline: Some(Duration::from_secs(3)),
+    max_backoff: Duration::from_secs(10),
+    base: 2.0,
+    deadline: Some(Duration::from_secs(120)),
 };
 
 /// Default interval for auto WAL pruning.

src/common/wal/src/test_util.rs

@@ -31,33 +31,3 @@ where
 
     test(endpoints).await
 }
-
-/// Get the kafka endpoints from the environment variable `GT_KAFKA_ENDPOINTS`.
-///
-/// The format of the environment variable is:
-/// ```
-/// GT_KAFKA_ENDPOINTS=localhost:9092,localhost:9093
-/// ```
-pub fn get_kafka_endpoints() -> Vec<String> {
-    let endpoints = std::env::var("GT_KAFKA_ENDPOINTS").unwrap();
-    endpoints
-        .split(',')
-        .map(|s| s.trim().to_string())
-        .collect::<Vec<_>>()
-}
-
-#[macro_export]
-/// Skip the test if the environment variable `GT_KAFKA_ENDPOINTS` is not set.
-///
-/// The format of the environment variable is:
-/// ```
-/// GT_KAFKA_ENDPOINTS=localhost:9092,localhost:9093
-/// ```
-macro_rules! maybe_skip_kafka_integration_test {
-    () => {
-        if std::env::var("GT_KAFKA_ENDPOINTS").is_err() {
-            common_telemetry::warn!("The endpoints is empty, skipping the test");
-            return;
-        }
-    };
-}

src/datanode/src/datanode.rs

@@ -43,10 +43,10 @@ use mito2::config::MitoConfig;
 use mito2::engine::MitoEngine;
 use object_store::manager::{ObjectStoreManager, ObjectStoreManagerRef};
 use object_store::util::normalize_dir;
+use query::dummy_catalog::TableProviderFactoryRef;
 use query::QueryEngineFactory;
 use servers::export_metrics::ExportMetricsTask;
 use servers::server::ServerHandlers;
-use servers::Mode;
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::path_utils::{region_dir, WAL_DIR};
 use store_api::region_engine::{RegionEngineRef, RegionRole};
@@ -58,8 +58,8 @@ use tokio::sync::Notify;
 use crate::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
 use crate::error::{
     self, BuildMetricEngineSnafu, BuildMitoEngineSnafu, CreateDirSnafu, GetMetadataSnafu,
-    MissingCacheSnafu, MissingKvBackendSnafu, MissingNodeIdSnafu, OpenLogStoreSnafu, Result,
-    ShutdownInstanceSnafu, ShutdownServerSnafu, StartServerSnafu,
+    MissingCacheSnafu, MissingNodeIdSnafu, OpenLogStoreSnafu, Result, ShutdownInstanceSnafu,
+    ShutdownServerSnafu, StartServerSnafu,
 };
 use crate::event_listener::{
     new_region_server_event_channel, NoopRegionServerEventListener, RegionServerEventListenerRef,
@@ -129,7 +129,7 @@ impl Datanode {
         self.services = services;
     }
 
-    pub async fn shutdown(&self) -> Result<()> {
+    pub async fn shutdown(&mut self) -> Result<()> {
         self.services
             .shutdown_all()
             .await
@@ -157,50 +157,49 @@ impl Datanode {
 
 pub struct DatanodeBuilder {
     opts: DatanodeOptions,
-    mode: Mode,
+    table_provider_factory: Option<TableProviderFactoryRef>,
     plugins: Plugins,
     meta_client: Option<MetaClientRef>,
-    kv_backend: Option<KvBackendRef>,
+    kv_backend: KvBackendRef,
     cache_registry: Option<Arc<LayeredCacheRegistry>>,
 }
 
 impl DatanodeBuilder {
-    /// `kv_backend` is optional. If absent, the builder will try to build one
-    /// by using the given `opts`
-    pub fn new(opts: DatanodeOptions, plugins: Plugins, mode: Mode) -> Self {
+    pub fn new(opts: DatanodeOptions, plugins: Plugins, kv_backend: KvBackendRef) -> Self {
         Self {
             opts,
-            mode,
+            table_provider_factory: None,
             plugins,
             meta_client: None,
-            kv_backend: None,
+            kv_backend,
             cache_registry: None,
         }
     }
 
-    pub fn with_meta_client(self, meta_client: MetaClientRef) -> Self {
-        Self {
-            meta_client: Some(meta_client),
-            ..self
-        }
+    pub fn options(&self) -> &DatanodeOptions {
+        &self.opts
     }
 
-    pub fn with_cache_registry(self, cache_registry: Arc<LayeredCacheRegistry>) -> Self {
-        Self {
-            cache_registry: Some(cache_registry),
-            ..self
-        }
+    pub fn with_meta_client(&mut self, client: MetaClientRef) -> &mut Self {
+        self.meta_client = Some(client);
+        self
     }
 
-    pub fn with_kv_backend(self, kv_backend: KvBackendRef) -> Self {
-        Self {
-            kv_backend: Some(kv_backend),
-            ..self
-        }
+    pub fn with_cache_registry(&mut self, registry: Arc<LayeredCacheRegistry>) -> &mut Self {
+        self.cache_registry = Some(registry);
+        self
+    }
+
+    pub fn kv_backend(&self) -> &KvBackendRef {
+        &self.kv_backend
+    }
+
+    pub fn with_table_provider_factory(&mut self, factory: TableProviderFactoryRef) -> &mut Self {
+        self.table_provider_factory = Some(factory);
+        self
     }
 
     pub async fn build(mut self) -> Result<Datanode> {
-        let mode = &self.mode;
         let node_id = self.opts.node_id.context(MissingNodeIdSnafu)?;
 
         let meta_client = self.meta_client.take();
@@ -210,8 +209,6 @@ impl DatanodeBuilder {
         // writable upon open.
         let controlled_by_metasrv = meta_client.is_some();
 
-        let kv_backend = self.kv_backend.take().context(MissingKvBackendSnafu)?;
-
         // build and initialize region server
         let (region_event_listener, region_event_receiver) = if controlled_by_metasrv {
             let (tx, rx) = new_region_server_event_channel();
@@ -233,7 +230,7 @@ impl DatanodeBuilder {
             .new_region_server(schema_metadata_manager, region_event_listener)
             .await?;
 
-        let datanode_table_manager = DatanodeTableManager::new(kv_backend.clone());
+        let datanode_table_manager = DatanodeTableManager::new(self.kv_backend.clone());
         let table_values = datanode_table_manager
             .tables(node_id)
             .try_collect::<Vec<_>>()
@@ -273,19 +270,18 @@ impl DatanodeBuilder {
             None
         };
 
+        let is_standalone = heartbeat_task.is_none();
         let greptimedb_telemetry_task = get_greptimedb_telemetry_task(
             Some(self.opts.storage.data_home.clone()),
-            mode,
-            self.opts.enable_telemetry,
+            is_standalone && self.opts.enable_telemetry,
         )
         .await;
 
-        let leases_notifier =
-            if self.opts.require_lease_before_startup && matches!(mode, Mode::Distributed) {
-                Some(Arc::new(Notify::new()))
-            } else {
-                None
-            };
+        let leases_notifier = if self.opts.require_lease_before_startup && !is_standalone {
+            Some(Arc::new(Notify::new()))
+        } else {
+            None
+        };
 
         let export_metrics_task =
             ExportMetricsTask::try_new(&self.opts.export_metrics, Some(&self.plugins))
@@ -363,7 +359,11 @@ impl DatanodeBuilder {
         );
         let query_engine = query_engine_factory.query_engine();
 
-        let table_provider_factory = Arc::new(DummyTableProviderFactory);
+        let table_provider_factory = self
+            .table_provider_factory
+            .clone()
+            .unwrap_or_else(|| Arc::new(DummyTableProviderFactory));
+
         let mut region_server = RegionServer::with_table_provider(
             query_engine,
             common_runtime::global_runtime(),
@@ -635,7 +635,6 @@ mod tests {
     use common_meta::kv_backend::memory::MemoryKvBackend;
     use common_meta::kv_backend::KvBackendRef;
     use mito2::engine::MITO_ENGINE_NAME;
-    use servers::Mode;
     use store_api::region_request::RegionRequest;
     use store_api::storage::RegionId;
 
@@ -671,19 +670,19 @@ mod tests {
         let kv_backend = Arc::new(MemoryKvBackend::new());
         let layered_cache_registry = Arc::new(
             LayeredCacheRegistryBuilder::default()
-                .add_cache_registry(build_datanode_cache_registry(kv_backend))
+                .add_cache_registry(build_datanode_cache_registry(kv_backend.clone()))
                 .build(),
         );
 
-        let builder = DatanodeBuilder::new(
+        let mut builder = DatanodeBuilder::new(
             DatanodeOptions {
                 node_id: Some(0),
                 ..Default::default()
             },
             Plugins::default(),
-            Mode::Standalone,
-        )
-        .with_cache_registry(layered_cache_registry);
+            kv_backend,
+        );
+        builder.with_cache_registry(layered_cache_registry);
 
         let kv = Arc::new(MemoryKvBackend::default()) as _;
         setup_table_datanode(&kv).await;

src/datanode/src/error.rs

@@ -150,12 +150,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Expect KvBackend but not found"))]
-    MissingKvBackend {
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Invalid SQL, error: {}", msg))]
     InvalidSql { msg: String },
 
@@ -426,7 +420,6 @@ impl ErrorExt for Error {
             | MissingRequiredField { .. }
             | RegionEngineNotFound { .. }
             | ParseAddr { .. }
-            | MissingKvBackend { .. }
             | TomlFormat { .. } => StatusCode::InvalidArguments,
 
             PayloadNotExist { .. }

src/datanode/src/greptimedb_telemetry.rs

@@ -20,7 +20,6 @@ use common_greptimedb_telemetry::{
     default_get_uuid, Collector, GreptimeDBTelemetry, GreptimeDBTelemetryTask,
     Mode as VersionReporterMode, TELEMETRY_INTERVAL,
 };
-use servers::Mode;
 
 struct StandaloneGreptimeDBTelemetryCollector {
     uuid: Option<String>,
@@ -55,7 +54,6 @@ impl Collector for StandaloneGreptimeDBTelemetryCollector {
 
 pub async fn get_greptimedb_telemetry_task(
     working_home: Option<String>,
-    mode: &Mode,
     enable: bool,
 ) -> Arc<GreptimeDBTelemetryTask> {
     if !enable || cfg!(test) || cfg!(debug_assertions) {
@@ -64,19 +62,14 @@ pub async fn get_greptimedb_telemetry_task(
     // Always enable.
     let should_report = Arc::new(AtomicBool::new(true));
 
-    match mode {
-        Mode::Standalone => Arc::new(GreptimeDBTelemetryTask::enable(
-            TELEMETRY_INTERVAL,
-            Box::new(GreptimeDBTelemetry::new(
-                working_home.clone(),
-                Box::new(StandaloneGreptimeDBTelemetryCollector {
-                    uuid: default_get_uuid(&working_home),
-                    retry: 0,
-                }),
-                should_report.clone(),
-            )),
-            should_report,
+    let uuid = default_get_uuid(&working_home);
+    Arc::new(GreptimeDBTelemetryTask::enable(
+        TELEMETRY_INTERVAL,
+        Box::new(GreptimeDBTelemetry::new(
+            working_home,
+            Box::new(StandaloneGreptimeDBTelemetryCollector { uuid, retry: 0 }),
+            should_report.clone(),
         )),
-        Mode::Distributed => Arc::new(GreptimeDBTelemetryTask::disable()),
-    }
+        should_report,
+    ))
 }

src/datanode/src/heartbeat/handler.rs

@@ -39,6 +39,7 @@ pub struct RegionHeartbeatResponseHandler {
     region_server: RegionServer,
     catchup_tasks: TaskTracker<()>,
     downgrade_tasks: TaskTracker<()>,
+    flush_tasks: TaskTracker<()>,
 }
 
 /// Handler of the instruction.
@@ -50,6 +51,7 @@ pub struct HandlerContext {
     region_server: RegionServer,
     catchup_tasks: TaskTracker<()>,
     downgrade_tasks: TaskTracker<()>,
+    flush_tasks: TaskTracker<()>,
 }
 
 impl HandlerContext {
@@ -63,6 +65,7 @@ impl HandlerContext {
             region_server,
             catchup_tasks: TaskTracker::new(),
             downgrade_tasks: TaskTracker::new(),
+            flush_tasks: TaskTracker::new(),
         }
     }
 }
@@ -74,6 +77,7 @@ impl RegionHeartbeatResponseHandler {
             region_server,
             catchup_tasks: TaskTracker::new(),
             downgrade_tasks: TaskTracker::new(),
+            flush_tasks: TaskTracker::new(),
         }
     }
 
@@ -95,8 +99,11 @@ impl RegionHeartbeatResponseHandler {
                 handler_context.handle_upgrade_region_instruction(upgrade_region)
             })),
             Instruction::InvalidateCaches(_) => InvalidHeartbeatResponseSnafu.fail(),
-            Instruction::FlushRegion(flush_regions) => Ok(Box::new(move |handler_context| {
-                handler_context.handle_flush_region_instruction(flush_regions)
+            Instruction::FlushRegions(flush_regions) => Ok(Box::new(move |handler_context| {
+                handler_context.handle_flush_regions_instruction(flush_regions)
+            })),
+            Instruction::FlushRegion(flush_region) => Ok(Box::new(move |handler_context| {
+                handler_context.handle_flush_region_instruction(flush_region)
             })),
         }
     }
@@ -111,6 +118,7 @@ impl HeartbeatResponseHandler for RegionHeartbeatResponseHandler {
                 | Some((_, Instruction::CloseRegion { .. }))
                 | Some((_, Instruction::DowngradeRegion { .. }))
                 | Some((_, Instruction::UpgradeRegion { .. }))
+                | Some((_, Instruction::FlushRegion { .. }))
         )
     }
 
@@ -124,12 +132,14 @@ impl HeartbeatResponseHandler for RegionHeartbeatResponseHandler {
         let region_server = self.region_server.clone();
         let catchup_tasks = self.catchup_tasks.clone();
         let downgrade_tasks = self.downgrade_tasks.clone();
+        let flush_tasks = self.flush_tasks.clone();
         let handler = Self::build_handler(instruction)?;
         let _handle = common_runtime::spawn_global(async move {
             let reply = handler(HandlerContext {
                 region_server,
                 catchup_tasks,
                 downgrade_tasks,
+                flush_tasks,
             })
             .await;
 

src/datanode/src/heartbeat/handler/flush_region.rs

@@ -12,16 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_meta::instruction::{FlushRegions, InstructionReply};
+use common_meta::instruction::{FlushRegions, InstructionReply, SimpleReply};
 use common_telemetry::warn;
 use futures_util::future::BoxFuture;
 use store_api::region_request::{RegionFlushRequest, RegionRequest};
+use store_api::storage::RegionId;
 
 use crate::error;
 use crate::heartbeat::handler::HandlerContext;
 
 impl HandlerContext {
-    pub(crate) fn handle_flush_region_instruction(
+    pub(crate) fn handle_flush_regions_instruction(
         self,
         flush_regions: FlushRegions,
     ) -> BoxFuture<'static, Option<InstructionReply>> {
@@ -49,6 +50,59 @@ impl HandlerContext {
             None
         })
     }
+
+    pub(crate) fn handle_flush_region_instruction(
+        self,
+        region_id: RegionId,
+    ) -> BoxFuture<'static, Option<InstructionReply>> {
+        Box::pin(async move {
+            let Some(writable) = self.region_server.is_region_leader(region_id) else {
+                return Some(InstructionReply::FlushRegion(SimpleReply {
+                    result: false,
+                    error: Some("Region is not leader".to_string()),
+                }));
+            };
+
+            if !writable {
+                return Some(InstructionReply::FlushRegion(SimpleReply {
+                    result: false,
+                    error: Some("Region is not writable".to_string()),
+                }));
+            }
+
+            let region_server_moved = self.region_server.clone();
+            let register_result = self
+                .flush_tasks
+                .try_register(
+                    region_id,
+                    Box::pin(async move {
+                        let request = RegionRequest::Flush(RegionFlushRequest {
+                            row_group_size: None,
+                        });
+                        region_server_moved
+                            .handle_request(region_id, request)
+                            .await?;
+                        Ok(())
+                    }),
+                )
+                .await;
+            if register_result.is_busy() {
+                warn!("Another flush task is running for the region: {region_id}");
+            }
+            let mut watcher = register_result.into_watcher();
+            let result = self.flush_tasks.wait_until_finish(&mut watcher).await;
+            match result {
+                Ok(()) => Some(InstructionReply::FlushRegion(SimpleReply {
+                    result: true,
+                    error: None,
+                })),
+                Err(err) => Some(InstructionReply::FlushRegion(SimpleReply {
+                    result: false,
+                    error: Some(format!("{err:?}")),
+                })),
+            }
+        })
+    }
 }
 
 #[cfg(test)]
@@ -84,7 +138,7 @@ mod tests {
 
         let reply = handler_context
             .clone()
-            .handle_flush_region_instruction(FlushRegions {
+            .handle_flush_regions_instruction(FlushRegions {
                 region_ids: region_ids.clone(),
             })
             .await;
@@ -94,7 +148,7 @@ mod tests {
         flushed_region_ids.write().unwrap().clear();
         let not_found_region_ids = (0..2).map(|i| RegionId::new(2048, i)).collect::<Vec<_>>();
         let reply = handler_context
-            .handle_flush_region_instruction(FlushRegions {
+            .handle_flush_regions_instruction(FlushRegions {
                 region_ids: not_found_region_ids.clone(),
             })
             .await;

src/datanode/src/heartbeat/task_tracker.rs

@@ -144,6 +144,11 @@ impl<T: Send + Sync + Clone + 'static> TaskTracker<T> {
         }
     }
 
+    /// Waits for a [RegisterResult] and returns a [WaitResult].
+    pub(crate) async fn wait_until_finish(&self, watcher: &mut TaskWatcher<T>) -> Result<T> {
+        wait(watcher).await
+    }
+
     /// Tries to register a new async task, returns [RegisterResult::Busy] if previous task is running.
     pub(crate) async fn try_register(
         &self,

src/datanode/src/service.rs

@@ -62,7 +62,7 @@ impl<'a> DatanodeServiceBuilder<'a> {
         }
     }
 
-    pub async fn build(mut self) -> Result<ServerHandlers> {
+    pub fn build(mut self) -> Result<ServerHandlers> {
         let handlers = ServerHandlers::default();
 
         if let Some(grpc_server) = self.grpc_server.take() {
@@ -70,7 +70,7 @@ impl<'a> DatanodeServiceBuilder<'a> {
                 addr: &self.opts.grpc.bind_addr,
             })?;
             let handler: ServerHandler = (Box::new(grpc_server), addr);
-            handlers.insert(handler).await;
+            handlers.insert(handler);
         }
 
         if self.enable_http_service {
@@ -82,7 +82,7 @@ impl<'a> DatanodeServiceBuilder<'a> {
                 addr: &self.opts.http.addr,
             })?;
             let handler: ServerHandler = (Box::new(http_server), addr);
-            handlers.insert(handler).await;
+            handlers.insert(handler);
         }
 
         Ok(handlers)

src/datanode/src/store.rs

@@ -25,7 +25,6 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use common_telemetry::{info, warn};
-use mito2::access_layer::{ATOMIC_WRITE_DIR, OLD_ATOMIC_WRITE_DIR};
 use object_store::layers::{LruCacheLayer, RetryInterceptor, RetryLayer};
 use object_store::services::Fs;
 use object_store::util::{join_dir, normalize_dir, with_instrument_layers};
@@ -169,13 +168,9 @@ async fn build_cache_layer(
     if let Some(path) = cache_path.as_ref()
         && !path.trim().is_empty()
     {
-        let atomic_temp_dir = join_dir(path, ATOMIC_WRITE_DIR);
+        let atomic_temp_dir = join_dir(path, ".tmp/");
         clean_temp_dir(&atomic_temp_dir)?;
 
-        // Compatible code. Remove this after a major release.
-        let old_atomic_temp_dir = join_dir(path, OLD_ATOMIC_WRITE_DIR);
-        clean_temp_dir(&old_atomic_temp_dir)?;
-
         let cache_store = Fs::default()
             .root(path)
             .atomic_write_dir(&atomic_temp_dir)

src/datanode/src/store/fs.rs

@@ -15,7 +15,6 @@
 use std::{fs, path};
 
 use common_telemetry::info;
-use mito2::access_layer::{ATOMIC_WRITE_DIR, OLD_ATOMIC_WRITE_DIR};
 use object_store::services::Fs;
 use object_store::util::join_dir;
 use object_store::ObjectStore;
@@ -34,13 +33,9 @@ pub async fn new_fs_object_store(
         .context(error::CreateDirSnafu { dir: data_home })?;
     info!("The file storage home is: {}", data_home);
 
-    let atomic_write_dir = join_dir(data_home, ATOMIC_WRITE_DIR);
+    let atomic_write_dir = join_dir(data_home, ".tmp/");
     store::clean_temp_dir(&atomic_write_dir)?;
 
-    // Compatible code. Remove this after a major release.
-    let old_atomic_temp_dir = join_dir(data_home, OLD_ATOMIC_WRITE_DIR);
-    store::clean_temp_dir(&old_atomic_temp_dir)?;
-
     let builder = Fs::default()
         .root(data_home)
         .atomic_write_dir(&atomic_write_dir);

src/datatypes/src/schema/constraint.rs

@@ -20,6 +20,7 @@ use snafu::{ensure, ResultExt};
 
 use crate::data_type::{ConcreteDataType, DataType};
 use crate::error::{self, Result};
+use crate::types::cast;
 use crate::value::Value;
 use crate::vectors::operations::VectorOp;
 use crate::vectors::{TimestampMillisecondVector, VectorRef};
@@ -178,6 +179,18 @@ impl ColumnDefaultConstraint {
         }
     }
 
+    /// Cast default value to given type
+    pub fn cast_to_datatype(&self, data_type: &ConcreteDataType) -> Result<Self> {
+        match self {
+            ColumnDefaultConstraint::Value(v) => Ok(Self::Value(cast(v.clone(), data_type)?)),
+            ColumnDefaultConstraint::Function(expr) => match &expr[..] {
+                // no need to cast, since function always require a data_type when need to create default value
+                CURRENT_TIMESTAMP | CURRENT_TIMESTAMP_FN | NOW_FN => Ok(self.clone()),
+                _ => error::UnsupportedDefaultExprSnafu { expr }.fail(),
+            },
+        }
+    }
+
     /// Only create default vector if it's impure, i.e., it's a function.
     ///
     /// This helps to delay creating constant default values to mito engine while also keeps impure default have consistent values
@@ -253,9 +266,10 @@ fn create_current_timestamp_vector(
     data_type: &ConcreteDataType,
     num_rows: usize,
 ) -> Result<VectorRef> {
-    let current_timestamp_vector = TimestampMillisecondVector::from_values(
-        std::iter::repeat(util::current_time_millis()).take(num_rows),
-    );
+    let current_timestamp_vector = TimestampMillisecondVector::from_values(std::iter::repeat_n(
+        util::current_time_millis(),
+        num_rows,
+    ));
     if data_type.is_timestamp() {
         current_timestamp_vector.cast(data_type)
     } else {

src/datatypes/src/vectors/constant.rs

@@ -198,8 +198,7 @@ impl fmt::Debug for ConstantVector {
 
 impl Serializable for ConstantVector {
     fn serialize_to_json(&self) -> Result<Vec<serde_json::Value>> {
-        std::iter::repeat(self.get(0))
-            .take(self.len())
+        std::iter::repeat_n(self.get(0), self.len())
             .map(serde_json::Value::try_from)
             .collect::<serde_json::Result<_>>()
             .context(SerializeSnafu)

src/datatypes/src/vectors/decimal.rs

@@ -412,7 +412,7 @@ pub(crate) fn replicate_decimal128(
                     // Safety: std::iter::Repeat and std::iter::Take implement TrustedLen.
                     builder
                         .mutable_array
-                        .append_trusted_len_iter(std::iter::repeat(data).take(repeat_times));
+                        .append_trusted_len_iter(std::iter::repeat_n(data, repeat_times));
                 }
             }
             None => {

src/datatypes/src/vectors/null.rs

@@ -120,9 +120,7 @@ impl fmt::Debug for NullVector {
 
 impl Serializable for NullVector {
     fn serialize_to_json(&self) -> Result<Vec<serde_json::Value>> {
-        Ok(std::iter::repeat(serde_json::Value::Null)
-            .take(self.len())
-            .collect())
+        Ok(std::iter::repeat_n(serde_json::Value::Null, self.len()).collect())
     }
 }