test!:Update config.yml

.cargo/config.toml

@@ -12,6 +12,3 @@ fetch = true
 checkout = true
 list_files = true
 internal_use_git2 = false
-
-[env]
-CARGO_WORKSPACE_DIR = { value = "", relative = true }

.github/ISSUE_TEMPLATE/config.yml

@@ -2,7 +2,7 @@ blank_issues_enabled: false
 contact_links:
   - name: Greptime Community Slack
     url: https://greptime.com/slack
-    about: Get free help from the Greptime community
+    about: Get free help from the Greptime community.
   - name: Greptime Community Discussion
     url: https://github.com/greptimeTeam/discussions
-    about: Get free help from the Greptime community
+    about: Get free help from the Greptime community.

.github/actions/build-linux-artifacts/action.yml

@@ -52,7 +52,7 @@ runs:
       uses: ./.github/actions/build-greptime-binary
       with:
         base-image: ubuntu
-        features: servers/dashboard
+        features: servers/dashboard,pg_kvbackend,mysql_kvbackend
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
         version: ${{ inputs.version }}
@@ -70,7 +70,7 @@ runs:
       if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Builds greptime for centos if the host machine is amd64.
       with:
         base-image: centos
-        features: servers/dashboard
+        features: servers/dashboard,pg_kvbackend,mysql_kvbackend
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
         version: ${{ inputs.version }}

.github/actions/release-cn-artifacts/action.yaml

@@ -64,11 +64,11 @@ inputs:
   upload-max-retry-times:
     description: Max retry times for uploading artifacts to S3
     required: false
-    default: "30"
+    default: "20"
   upload-retry-timeout:
     description: Timeout for uploading artifacts to S3
     required: false
-    default: "120" # minutes
+    default: "30" # minutes
 runs:
   using: composite
   steps:

.github/actions/setup-greptimedb-cluster/action.yml

@@ -10,13 +10,13 @@ inputs:
   meta-replicas:
     default: 2
     description: "Number of Metasrv replicas"
-  image-registry:
+  image-registry: 
     default: "docker.io"
     description: "Image registry"
-  image-repository:
+  image-repository: 
     default: "greptime/greptimedb"
     description: "Image repository"
-  image-tag:
+  image-tag: 
     default: "latest"
     description: 'Image tag'
   etcd-endpoints:
@@ -32,12 +32,12 @@ runs:
   steps:
   - name: Install GreptimeDB operator
     uses: nick-fields/retry@v3
-    with:
+    with: 
       timeout_minutes: 3
       max_attempts: 3
       shell: bash
       command: |
-        helm repo add greptime https://greptimeteam.github.io/helm-charts/
+        helm repo add greptime https://greptimeteam.github.io/helm-charts/ 
         helm repo update
         helm upgrade \
           --install \
@@ -48,10 +48,10 @@ runs:
           --wait-for-jobs
   - name: Install GreptimeDB cluster
     shell: bash
-    run: |
+    run: | 
       helm upgrade \
         --install my-greptimedb \
-        --set meta.backendStorage.etcd.endpoints=${{ inputs.etcd-endpoints }} \
+        --set meta.etcdEndpoints=${{ inputs.etcd-endpoints }} \
         --set meta.enableRegionFailover=${{ inputs.enable-region-failover }} \
         --set image.registry=${{ inputs.image-registry }} \
         --set image.repository=${{ inputs.image-repository }}  \
@@ -59,7 +59,7 @@ runs:
         --set base.podTemplate.main.resources.requests.cpu=50m \
         --set base.podTemplate.main.resources.requests.memory=256Mi \
         --set base.podTemplate.main.resources.limits.cpu=2000m \
-        --set base.podTemplate.main.resources.limits.memory=3Gi \
+        --set base.podTemplate.main.resources.limits.memory=2Gi \
         --set frontend.replicas=${{ inputs.frontend-replicas }} \
         --set datanode.replicas=${{ inputs.datanode-replicas }} \
         --set meta.replicas=${{ inputs.meta-replicas }} \
@@ -72,7 +72,7 @@ runs:
   - name: Wait for GreptimeDB
     shell: bash
     run: |
-      while true; do
+      while true; do 
         PHASE=$(kubectl -n my-greptimedb get gtc my-greptimedb -o jsonpath='{.status.clusterPhase}')
         if [ "$PHASE" == "Running" ]; then
           echo "Cluster is ready"
@@ -86,10 +86,10 @@ runs:
   - name: Print GreptimeDB info
     if: always()
     shell: bash
-    run: |
+    run: | 
       kubectl get all --show-labels -n my-greptimedb
   - name: Describe Nodes
     if: always()
     shell: bash
-    run: |
+    run: | 
       kubectl describe nodes

.github/labeler.yaml

@@ -1,15 +0,0 @@
-ci:
-  - changed-files:
-      - any-glob-to-any-file: .github/**
-
-docker:
-  - changed-files:
-      - any-glob-to-any-file: docker/**
-
-documentation:
-  - changed-files:
-      - any-glob-to-any-file: docs/**
-
-dashboard:
-  - changed-files:
-      - any-glob-to-any-file: grafana/**

.github/scripts/check-version.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Get current version
-CURRENT_VERSION=$1
-if [ -z "$CURRENT_VERSION" ]; then
-  echo "Error: Failed to get current version"
-  exit 1
-fi
-
-# Get the latest version from GitHub Releases
-API_RESPONSE=$(curl -s "https://api.github.com/repos/GreptimeTeam/greptimedb/releases/latest")
-
-if [ -z "$API_RESPONSE" ] || [ "$(echo "$API_RESPONSE" | jq -r '.message')" = "Not Found" ]; then
-  echo "Error: Failed to fetch latest version from GitHub"
-  exit 1
-fi
-
-# Get the latest version
-LATEST_VERSION=$(echo "$API_RESPONSE" | jq -r '.tag_name')
-
-if [ -z "$LATEST_VERSION" ] || [ "$LATEST_VERSION" = "null" ]; then
-  echo "Error: No valid version found in GitHub releases"
-  exit 1
-fi
-
-# Cleaned up version number format (removed possible 'v' prefix and -nightly suffix)
-CLEAN_CURRENT=$(echo "$CURRENT_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//')
-CLEAN_LATEST=$(echo "$LATEST_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//')
-
-echo "Current version: $CLEAN_CURRENT"
-echo "Latest release version: $CLEAN_LATEST"
-
-# Use sort -V to compare versions
-HIGHER_VERSION=$(printf "%s\n%s" "$CLEAN_CURRENT" "$CLEAN_LATEST" | sort -V | tail -n1)
-
-if [ "$HIGHER_VERSION" = "$CLEAN_CURRENT" ]; then
-  echo "Current version ($CLEAN_CURRENT) is NEWER than or EQUAL to latest ($CLEAN_LATEST)"
-  echo "should-push-latest-tag=true" >> $GITHUB_OUTPUT
-else
-  echo "Current version ($CLEAN_CURRENT) is OLDER than latest ($CLEAN_LATEST)"
-  echo "should-push-latest-tag=false" >> $GITHUB_OUTPUT
-fi

.github/scripts/create-version.sh

@@ -16,8 +16,7 @@ function create_version() {
 
   if [ -z "$NEXT_RELEASE_VERSION" ]; then
       echo "NEXT_RELEASE_VERSION is empty, use version from Cargo.toml" >&2
-      # NOTE: Need a `v` prefix for the version string.
-      export NEXT_RELEASE_VERSION=v$(grep '^version = ' Cargo.toml | cut -d '"' -f 2 | head -n 1)
+      export NEXT_RELEASE_VERSION=$(grep '^version = ' Cargo.toml | cut -d '"' -f 2 | head -n 1)
   fi
 
   if [ -z "$NIGHTLY_RELEASE_PREFIX" ]; then

.github/scripts/deploy-greptimedb.sh

@@ -68,7 +68,7 @@ function deploy_greptimedb_cluster() {
 
   helm install "$cluster_name" greptime/greptimedb-cluster \
     --set image.tag="$GREPTIMEDB_IMAGE_TAG" \
-    --set meta.backendStorage.etcd.endpoints="etcd.$install_namespace:2379" \
+    --set meta.etcdEndpoints="etcd.$install_namespace:2379" \
     -n "$install_namespace"
 
   # Wait for greptimedb cluster to be ready.
@@ -103,7 +103,7 @@ function deploy_greptimedb_cluster_with_s3_storage() {
 
   helm install "$cluster_name" greptime/greptimedb-cluster -n "$install_namespace" \
     --set image.tag="$GREPTIMEDB_IMAGE_TAG" \
-    --set meta.backendStorage.etcd.endpoints="etcd.$install_namespace:2379" \
+    --set meta.etcdEndpoints="etcd.$install_namespace:2379" \
     --set storage.s3.bucket="$AWS_CI_TEST_BUCKET" \
     --set storage.s3.region="$AWS_REGION" \
     --set storage.s3.root="$DATA_ROOT" \

.github/scripts/update-helm-charts-version.sh

@@ -30,7 +30,7 @@ update_helm_charts_version() {
 
   # Commit the changes.
   git add .
-  git commit -s -m "chore: Update GreptimeDB version to ${VERSION}"
+  git commit -m "chore: Update GreptimeDB version to ${VERSION}"
   git push origin $BRANCH_NAME
 
   # Create a Pull Request.

.github/scripts/update-homebrew-greptme-version.sh

@@ -26,7 +26,7 @@ update_homebrew_greptime_version() {
 
   # Commit the changes.
   git add .
-  git commit -s -m "chore: Update GreptimeDB version to ${VERSION}"
+  git commit -m "chore: Update GreptimeDB version to ${VERSION}"
   git push origin $BRANCH_NAME
 
   # Create a Pull Request.

.github/workflows/dev-build.yml

@@ -55,11 +55,6 @@ on:
         description: Build and push images to DockerHub and ACR
         required: false
         default: true
-      upload_artifacts_to_s3:
-        type: boolean
-        description: Whether upload artifacts to s3
-        required: false
-        default: false
       cargo_profile:
         type: choice
         description: The cargo profile to use in building GreptimeDB.
@@ -243,7 +238,7 @@ jobs:
           version: ${{ needs.allocate-runners.outputs.version }}
           push-latest-tag: false # Don't push the latest tag to registry.
           dev-mode: true # Only build the standard images.
-
+          
       - name: Echo Docker image tag to step summary
         run: |
           echo "## Docker Image Tag" >> $GITHUB_STEP_SUMMARY
@@ -286,7 +281,7 @@ jobs:
           aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: ${{ inputs.upload_artifacts_to_s3 }}
+          upload-to-s3: false
           dev-mode: true                     # Only build the standard images(exclude centos images).
           push-latest-tag: false             # Don't push the latest tag to registry.
           update-version-info: false         # Don't update the version info in S3.

.github/workflows/develop.yml

@@ -195,7 +195,6 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
-      fail-fast: false
       matrix:
         target: [ "unstable_fuzz_create_table_standalone" ]
     steps:
@@ -250,11 +249,6 @@ jobs:
           name: unstable-fuzz-logs
           path: /tmp/unstable-greptime/
           retention-days: 3
-      - name: Describe pods
-        if: failure()
-        shell: bash
-        run: |
-          kubectl describe pod -n my-greptimedb
 
   build-greptime-ci:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
@@ -305,7 +299,6 @@ jobs:
     needs:  build-greptime-ci
     timeout-minutes: 60
     strategy:
-      fail-fast: false
       matrix:
         target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
         mode:
@@ -410,11 +403,6 @@ jobs:
         shell: bash
         run: |
           kubectl describe nodes
-      - name: Describe pod
-        if: failure()
-        shell: bash
-        run: |
-          kubectl describe pod -n my-greptimedb
       - name: Export kind logs
         if: failure()
         shell: bash
@@ -443,7 +431,6 @@ jobs:
     needs:  build-greptime-ci
     timeout-minutes: 60
     strategy:
-      fail-fast: false
       matrix:
         target: ["fuzz_migrate_mito_regions", "fuzz_migrate_metric_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
         mode:
@@ -564,11 +551,6 @@ jobs:
         shell: bash
         run: |
           kubectl describe nodes
-      - name: Describe pods
-        if: failure()
-        shell: bash
-        run: |
-          kubectl describe pod -n my-greptimedb
       - name: Export kind logs
         if: failure()
         shell: bash
@@ -596,7 +578,6 @@ jobs:
     needs: build
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
       matrix:
         os: [ ubuntu-latest ]
         mode:

.github/workflows/nightly-ci.yml

@@ -124,7 +124,9 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - uses: cachix/install-nix-action@v31
-      - run: nix develop --command cargo check --bin greptime
+        with:
+          nix_path: nixpkgs=channel:nixos-24.11
+      - run: nix develop --command cargo build --bin greptime
         env:
           CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
 

.github/workflows/pr-labeling.yaml

@@ -1,42 +0,0 @@
-name: 'PR Labeling'
-
-on:
-  pull_request_target:
-    types:
-      - opened
-      - synchronize
-      - reopened
-
-permissions:
-  contents: read
-  pull-requests: write
-  issues: write
-
-jobs:
-  labeler:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v4
-
-      - uses: actions/labeler@v5
-        with:
-          configuration-path: ".github/labeler.yaml"
-          repo-token: "${{ secrets.GITHUB_TOKEN }}"
-
-  size-label:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: pascalgn/size-label-action@v0.5.5
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        with:
-          sizes: >
-            {
-              "0": "XS",
-              "100": "S",
-              "300": "M",
-              "1000": "L",
-              "1500": "XL",
-              "2000": "XXL"
-            }

.github/workflows/release.yml

@@ -110,8 +110,6 @@ jobs:
 
       # The 'version' use as the global tag name of the release workflow.
       version: ${{ steps.create-version.outputs.version }}
-
-      should-push-latest-tag: ${{ steps.check-version.outputs.should-push-latest-tag }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -137,11 +135,6 @@ jobs:
           GITHUB_REF_NAME: ${{ github.ref_name }}
           NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
 
-      - name: Check version
-        id: check-version
-        run: |
-          ./.github/scripts/check-version.sh "${{ steps.create-version.outputs.version }}"
-
       - name: Allocate linux-amd64 runner
         if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
         uses: ./.github/actions/start-runner
@@ -321,7 +314,7 @@ jobs:
           image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          push-latest-tag: ${{ needs.allocate-runners.outputs.should-push-latest-tag == 'true' && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
+          push-latest-tag: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
 
       - name: Set build image result
         id: set-build-image-result
@@ -339,7 +332,7 @@ jobs:
       build-windows-artifacts,
       release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -368,7 +361,7 @@ jobs:
           dev-mode: false
           upload-to-s3: true
           update-version-info: true
-          push-latest-tag: ${{ needs.allocate-runners.outputs.should-push-latest-tag == 'true' && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
+          push-latest-tag: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
 
   publish-github-release:
     name: Create GitHub release and upload artifacts
@@ -448,8 +441,8 @@ jobs:
           aws-region: ${{ vars.EC2_RUNNER_REGION }}
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
 
-  bump-downstream-repo-versions:
-    name: Bump downstream repo versions
+  bump-doc-version:
+    name: Bump doc version
     if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }}
     needs: [allocate-runners, publish-github-release]
     runs-on: ubuntu-latest
@@ -463,16 +456,36 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
-      - name: Bump downstream repo versions
+      - name: Bump doc version
         working-directory: cyborg
-        run: pnpm tsx bin/bump-versions.ts
+        run: pnpm tsx bin/bump-doc-version.ts
+        env:
+          VERSION: ${{ needs.allocate-runners.outputs.version }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCS_REPO_TOKEN: ${{ secrets.DOCS_REPO_TOKEN }}
+
+  bump-website-version:
+    name: Bump website version
+    if: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
+    needs: [allocate-runners, publish-github-release]
+    runs-on: ubuntu-latest
+    # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+    permissions:
+      issues: write # Allows the action to create issues for cyborg.
+      contents: write # Allows the action to create a release.
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: ./.github/actions/setup-cyborg
+      - name: Bump website version
+        working-directory: cyborg
+        run: pnpm tsx bin/bump-website-version.ts
         env:
-          TARGET_REPOS: website,docs,demo
           VERSION: ${{ needs.allocate-runners.outputs.version }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           WEBSITE_REPO_TOKEN: ${{ secrets.WEBSITE_REPO_TOKEN }}
-          DOCS_REPO_TOKEN: ${{ secrets.DOCS_REPO_TOKEN }}
-          DEMO_REPO_TOKEN: ${{ secrets.DEMO_REPO_TOKEN }}
 
   bump-helm-charts-version:
     name: Bump helm charts version

.github/workflows/semantic-pull-request.yml

@@ -11,17 +11,16 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
-permissions:
-  issues: write
-  contents: write
-  pull-requests: write
-
 jobs:
   check:
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write  # Add permissions to modify PRs
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - name: Check Pull Request
         working-directory: cyborg

.gitignore

@@ -60,7 +60,4 @@ tests-fuzz/corpus/
 greptimedb_data
 
 # github
-!/.github
-
-# Claude code
-CLAUDE.md
+!/.github

AUTHOR.md

@@ -10,10 +10,12 @@
 * [NiwakaDev](https://github.com/NiwakaDev)
 * [tisonkun](https://github.com/tisonkun)
 
+
 ## Team Members (in alphabetical order)
 
 * [apdong2022](https://github.com/apdong2022)
 * [beryl678](https://github.com/beryl678)
+* [Breeze-P](https://github.com/Breeze-P)
 * [daviderli614](https://github.com/daviderli614)
 * [discord9](https://github.com/discord9)
 * [evenyag](https://github.com/evenyag)

CONTRIBUTING.md

@@ -108,7 +108,7 @@ of what you were trying to do and what went wrong. You can also reach for help i
 The core team will be thrilled if you would like to participate in any way you like. When you are stuck, try to ask for help by filing an issue, with a detailed description of what you were trying to do and what went wrong. If you have any questions or if you would like to get involved in our community, please check out:
 
 - [GreptimeDB Community Slack](https://greptime.com/slack)
-- [GreptimeDB GitHub Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
+- [GreptimeDB Github Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)
 
 Also, see some extra GreptimeDB content:
 

Cargo.toml

@@ -13,7 +13,6 @@ members = [
     "src/common/datasource",
     "src/common/decimal",
     "src/common/error",
-    "src/common/event-recorder",
     "src/common/frontend",
     "src/common/function",
     "src/common/greptimedb-telemetry",
@@ -31,8 +30,6 @@ members = [
     "src/common/recordbatch",
     "src/common/runtime",
     "src/common/session",
-    "src/common/sql",
-    "src/common/stat",
     "src/common/substrait",
     "src/common/telemetry",
     "src/common/test-util",
@@ -51,7 +48,6 @@ members = [
     "src/meta-client",
     "src/meta-srv",
     "src/metric-engine",
-    "src/mito-codec",
     "src/mito2",
     "src/object-store",
     "src/operator",
@@ -73,13 +69,11 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.16.0"
+version = "0.15.0"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.lints]
-clippy.print_stdout = "warn"
-clippy.print_stderr = "warn"
 clippy.dbg_macro = "warn"
 clippy.implicit_clone = "warn"
 clippy.result_large_err = "allow"
@@ -121,26 +115,24 @@ clap = { version = "4.4", features = ["derive"] }
 config = "0.13.0"
 crossbeam-utils = "0.8"
 dashmap = "6.1"
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-functions = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-functions-aggregate-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-physical-plan = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "12c0381babd52c681043957e9d6ee083a03f7646" }
+datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
+datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
+datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
+datafusion-functions = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
+datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
+datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
+datafusion-physical-plan = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
+datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
+datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" }
 deadpool = "0.12"
 deadpool-postgres = "0.14"
 derive_builder = "0.20"
 dotenv = "0.15"
-either = "1.15"
 etcd-client = "0.14"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "7fcaa3e413947a7a28d9af95812af26c1939ce78" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "7668a882d57ca6a2333146e0574b8f0c9d5008ae" }
 hex = "0.4"
 http = "1"
 humantime = "2.1"
@@ -156,7 +148,6 @@ meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev =
 mockall = "0.13"
 moka = "0.12"
 nalgebra = "0.33"
-nix = { version = "0.30.1", default-features = false, features = ["event", "fs", "process"] }
 notify = "8.0"
 num_cpus = "1.16"
 object_store_opendal = "0.50"
@@ -168,13 +159,14 @@ opentelemetry-proto = { version = "0.27", features = [
     "with-serde",
     "logs",
 ] }
-ordered-float = { version = "4.3", features = ["serde"] }
 parking_lot = "0.12"
 parquet = { version = "54.2", default-features = false, features = ["arrow", "async", "object_store"] }
 paste = "1.0"
 pin-project = "1.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-promql-parser = { version = "0.6", features = ["ser"] }
+promql-parser = { git = "https://github.com/GreptimeTeam/promql-parser.git", rev = "0410e8b459dda7cb222ce9596f8bf3971bd07bd2", features = [
+    "ser",
+] }
 prost = { version = "0.13", features = ["no-recursion-limit"] }
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.9"
@@ -225,13 +217,10 @@ tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
 tonic = { version = "0.12", features = ["tls", "gzip", "zstd"] }
 tower = "0.5"
-tower-http = "0.6"
-tracing = "0.1"
 tracing-appender = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
 typetag = "0.2"
 uuid = { version = "1.7", features = ["serde", "v4", "fast-rng"] }
-vrl = "0.25"
 zstd = "0.13"
 # DO_NOT_REMOVE_THIS: END_OF_EXTERNAL_DEPENDENCIES
 
@@ -249,7 +238,6 @@ common-config = { path = "src/common/config" }
 common-datasource = { path = "src/common/datasource" }
 common-decimal = { path = "src/common/decimal" }
 common-error = { path = "src/common/error" }
-common-event-recorder = { path = "src/common/event-recorder" }
 common-frontend = { path = "src/common/frontend" }
 common-function = { path = "src/common/function" }
 common-greptimedb-telemetry = { path = "src/common/greptimedb-telemetry" }
@@ -267,7 +255,6 @@ common-query = { path = "src/common/query" }
 common-recordbatch = { path = "src/common/recordbatch" }
 common-runtime = { path = "src/common/runtime" }
 common-session = { path = "src/common/session" }
-common-sql = { path = "src/common/sql" }
 common-telemetry = { path = "src/common/telemetry" }
 common-test-util = { path = "src/common/test-util" }
 common-time = { path = "src/common/time" }
@@ -285,7 +272,6 @@ log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
 meta-srv = { path = "src/meta-srv" }
 metric-engine = { path = "src/metric-engine" }
-mito-codec = { path = "src/mito-codec" }
 mito2 = { path = "src/mito2" }
 object-store = { path = "src/object-store" }
 operator = { path = "src/operator" }
@@ -301,7 +287,6 @@ query = { path = "src/query" }
 servers = { path = "src/servers" }
 session = { path = "src/session" }
 sql = { path = "src/sql" }
-stat = { path = "src/common/stat" }
 store-api = { path = "src/store-api" }
 substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }

README.md

@@ -75,9 +75,9 @@
 | --------- | ----------- |
 | [Unified Observability Data](https://docs.greptime.com/user-guide/concepts/why-greptimedb) | Store metrics, logs, and traces as timestamped, contextual wide events. Query via [SQL](https://docs.greptime.com/user-guide/query-data/sql), [PromQL](https://docs.greptime.com/user-guide/query-data/promql), and [streaming](https://docs.greptime.com/user-guide/flow-computation/overview). |
 | [High Performance & Cost Effective](https://docs.greptime.com/user-guide/manage-data/data-index) | Written in Rust, with a distributed query engine, [rich indexing](https://docs.greptime.com/user-guide/manage-data/data-index), and optimized columnar storage, delivering sub-second responses at PB scale. |
-| [Cloud-Native Architecture](https://docs.greptime.com/user-guide/concepts/architecture) | Designed for [Kubernetes](https://docs.greptime.com/user-guide/deployments-administration/deploy-on-kubernetes/greptimedb-operator-management), with compute/storage separation, native object storage (AWS S3, Azure Blob, etc.) and seamless cross-cloud access. |
+| [Cloud-Native Architecture](https://docs.greptime.com/user-guide/concepts/architecture) | Designed for [Kubernetes](https://docs.greptime.com/user-guide/deployments/deploy-on-kubernetes/greptimedb-operator-management), with compute/storage separation, native object storage (AWS S3, Azure Blob, etc.) and seamless cross-cloud access. |
 | [Developer-Friendly](https://docs.greptime.com/user-guide/protocols/overview) | Access via SQL/PromQL interfaces, REST API, MySQL/PostgreSQL protocols, and popular ingestion [protocols](https://docs.greptime.com/user-guide/protocols/overview). |
-| [Flexible Deployment](https://docs.greptime.com/user-guide/deployments-administration/overview) | Deploy anywhere: edge (including ARM/[Android](https://docs.greptime.com/user-guide/deployments-administration/run-on-android)) or cloud, with unified APIs and efficient data sync. |
+| [Flexible Deployment](https://docs.greptime.com/user-guide/deployments/overview) | Deploy anywhere: edge (including ARM/[Android](https://docs.greptime.com/user-guide/deployments/run-on-android)) or cloud, with unified APIs and efficient data sync. |
 
 Learn more in [Why GreptimeDB](https://docs.greptime.com/user-guide/concepts/why-greptimedb) and [Observability 2.0 and the Database for It](https://greptime.com/blogs/2025-04-25-greptimedb-observability2-new-database).
 
@@ -189,8 +189,7 @@ We invite you to engage and contribute!
 - [Official Website](https://greptime.com/)
 - [Blog](https://greptime.com/blogs/)
 - [LinkedIn](https://www.linkedin.com/company/greptime/)
-- [X (Twitter)](https://X.com/greptime)
-- [YouTube](https://www.youtube.com/@greptime)
+- [Twitter](https://twitter.com/greptime)
 
 ## License
 

config/config.md

@@ -27,7 +27,6 @@
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
 | `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
-| `http.prom_validation_mode` | String | `strict` | Whether to enable validation for Prometheus remote write requests.<br/>Available options:<br/>- strict: deny invalid UTF-8 strings (default).<br/>- lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).<br/>- unchecked: do not valid strings. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.bind_addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
@@ -100,7 +99,7 @@
 | `query` | -- | -- | The query engine options. |
 | `query.parallelism` | Integer | `0` | Parallelism of the query engine.<br/>Default to 0, which means the number of CPU cores. |
 | `storage` | -- | -- | The data storage options. |
-| `storage.data_home` | String | `./greptimedb_data` | The working home directory. |
+| `storage.data_home` | String | `./greptimedb_data/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
 | `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}`. An empty string means disabling. |
 | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. |
@@ -123,7 +122,6 @@
 | `storage.http_client.connect_timeout` | String | `30s` | The timeout for only the connect phase of a http client. |
 | `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.<br/>Also considered a total deadline. |
 | `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. |
-| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification<br/>**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
 | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. |
 | `region_engine.mito` | -- | -- | The Mito engine options. |
 | `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
@@ -185,11 +183,10 @@
 | `logging.dir` | String | `./greptimedb_data/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
-| `logging.otlp_endpoint` | String | `http://localhost:4318` | The OTLP tracing endpoint. |
+| `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `slow_query` | -- | -- | The slow query log options. |
@@ -197,13 +194,13 @@
 | `slow_query.record_type` | String | Unset | The record type of slow queries. It can be `system_table` or `log`. |
 | `slow_query.threshold` | String | Unset | The threshold of slow query. |
 | `slow_query.sample_ratio` | Float | Unset | The sampling ratio of slow query log. The value should be in the range of (0, 1]. |
-| `export_metrics` | -- | -- | The standalone can export its metrics and send to Prometheus compatible service (e.g. `greptimedb`) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
+| `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
 | `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommended to collect metrics generated by itself<br/>You must create the database before enabling it. |
 | `export_metrics.self_import.db` | String | Unset | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
-| `export_metrics.remote_write.url` | String | `""` | The prometheus remote write endpoint that the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
+| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
@@ -229,12 +226,10 @@
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
 | `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
-| `http.prom_validation_mode` | String | `strict` | Whether to enable validation for Prometheus remote write requests.<br/>Available options:<br/>- strict: deny invalid UTF-8 strings (default).<br/>- lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).<br/>- unchecked: do not valid strings. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.bind_addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
 | `grpc.server_addr` | String | `127.0.0.1:4001` | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
-| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:<br/>- `none`: disable all compression<br/>- `transport`: only enable gRPC transport compression (zstd)<br/>- `arrow_ipc`: only enable Arrow IPC compression (lz4)<br/>- `all`: enable all compression.<br/>Default to `none` |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
 | `grpc.tls.cert_path` | String | Unset | Certificate file path. |
@@ -289,11 +284,10 @@
 | `logging.dir` | String | `./greptimedb_data/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
-| `logging.otlp_endpoint` | String | `http://localhost:4318` | The OTLP tracing endpoint. |
+| `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `slow_query` | -- | -- | The slow query log options. |
@@ -302,11 +296,13 @@
 | `slow_query.threshold` | String | `30s` | The threshold of slow query. It can be human readable time string, for example: `10s`, `100ms`, `1s`. |
 | `slow_query.sample_ratio` | Float | `1.0` | The sampling ratio of slow query log. The value should be in the range of (0, 1]. For example, `0.1` means 10% of the slow queries will be logged and `1.0` means all slow queries will be logged. |
 | `slow_query.ttl` | String | `30d` | The TTL of the `slow_queries` system table. Default is `30d` when `record_type` is `system_table`. |
-| `export_metrics` | -- | -- | The frontend can export its metrics and send to Prometheus compatible service (e.g. `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
+| `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
+| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
+| `export_metrics.self_import.db` | String | Unset | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
-| `export_metrics.remote_write.url` | String | `""` | The prometheus remote write endpoint that the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
+| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
@@ -316,32 +312,23 @@
 
 | Key | Type | Default | Descriptions |
 | --- | -----| ------- | ----------- |
-| `data_home` | String | `./greptimedb_data` | The working home directory. |
+| `data_home` | String | `./greptimedb_data/metasrv/` | The working home directory. |
+| `bind_addr` | String | `127.0.0.1:3002` | The bind address of metasrv. |
+| `server_addr` | String | `127.0.0.1:3002` | The communication server address for the frontend and datanode to connect to metasrv.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `bind_addr`. |
 | `store_addrs` | Array | -- | Store server address default to etcd store.<br/>For postgres store, the format is:<br/>"password=password dbname=postgres user=postgres host=localhost port=5432"<br/>For etcd store, the format is:<br/>"127.0.0.1:2379" |
 | `store_key_prefix` | String | `""` | If it's not empty, the metasrv will store all data with this key prefix. |
-| `backend` | String | `etcd_store` | The datastore for meta server.<br/>Available values:<br/>- `etcd_store` (default value)<br/>- `memory_store`<br/>- `postgres_store`<br/>- `mysql_store` |
+| `backend` | String | `etcd_store` | The datastore for meta server.<br/>Available values:<br/>- `etcd_store` (default value)<br/>- `memory_store`<br/>- `postgres_store` |
 | `meta_table_name` | String | `greptime_metakv` | Table name in RDS to store metadata. Effect when using a RDS kvbackend.<br/>**Only used when backend is `postgres_store`.** |
 | `meta_election_lock_id` | Integer | `1` | Advisory lock id in PostgreSQL for election. Effect when using PostgreSQL as kvbackend<br/>Only used when backend is `postgres_store`. |
 | `selector` | String | `round_robin` | Datanode selector type.<br/>- `round_robin` (default value)<br/>- `lease_based`<br/>- `load_based`<br/>For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector". |
 | `use_memory_store` | Bool | `false` | Store data in memory. |
 | `enable_region_failover` | Bool | `false` | Whether to enable region failover.<br/>This feature is only available on GreptimeDB running on cluster mode and<br/>- Using Remote WAL<br/>- Using shared storage (e.g., s3). |
-| `region_failure_detector_initialization_delay` | String | `10m` | The delay before starting region failure detection.<br/>This delay helps prevent Metasrv from triggering unnecessary region failovers before all Datanodes are fully started.<br/>Especially useful when the cluster is not deployed with GreptimeDB Operator and maintenance mode is not enabled. |
 | `allow_region_failover_on_local_wal` | Bool | `false` | Whether to allow region failover on local WAL.<br/>**This option is not recommended to be set to true, because it may lead to data loss during failover.** |
 | `node_max_idle_time` | String | `24hours` | Max allowed idle time before removing node info from metasrv memory. |
 | `enable_telemetry` | Bool | `true` | Whether to enable greptimedb telemetry. Enabled by default. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
-| `grpc` | -- | -- | The gRPC server options. |
-| `grpc.bind_addr` | String | `127.0.0.1:3002` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:3002` | The communication server address for the frontend and datanode to connect to metasrv.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `bind_addr`. |
-| `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
-| `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
-| `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
-| `http` | -- | -- | The HTTP server options. |
-| `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
-| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout. |
-| `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `procedure` | -- | -- | Procedure storage options. |
 | `procedure.max_retry_times` | Integer | `12` | Procedure max retry time. |
 | `procedure.retry_delay` | String | `500ms` | Initial retry delay of procedures, increases exponentially |
@@ -373,18 +360,19 @@
 | `logging.dir` | String | `./greptimedb_data/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
-| `logging.otlp_endpoint` | String | `http://localhost:4318` | The OTLP tracing endpoint. |
+| `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
-| `export_metrics` | -- | -- | The metasrv can export its metrics and send to Prometheus compatible service (e.g. `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
+| `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
+| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
+| `export_metrics.self_import.db` | String | Unset | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
-| `export_metrics.remote_write.url` | String | `""` | The prometheus remote write endpoint that the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
+| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
@@ -410,7 +398,6 @@
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
-| `grpc.flight_compression` | String | `arrow_ipc` | Compression mode for datanode side Arrow IPC service. Available options:<br/>- `none`: disable all compression<br/>- `transport`: only enable gRPC transport compression (zstd)<br/>- `arrow_ipc`: only enable Arrow IPC compression (lz4)<br/>- `all`: enable all compression.<br/>Default to `none` |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
 | `grpc.tls.cert_path` | String | Unset | Certificate file path. |
@@ -436,8 +423,8 @@
 | `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka. |
 | `wal.dir` | String | Unset | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.file_size` | String | `128MB` | The size of the WAL segment file.<br/>**It's only used when the provider is `raft_engine`**. |
-| `wal.purge_threshold` | String | `1GB` | The threshold of the WAL size to trigger a purge.<br/>**It's only used when the provider is `raft_engine`**. |
-| `wal.purge_interval` | String | `1m` | The interval to trigger a purge.<br/>**It's only used when the provider is `raft_engine`**. |
+| `wal.purge_threshold` | String | `1GB` | The threshold of the WAL size to trigger a flush.<br/>**It's only used when the provider is `raft_engine`**. |
+| `wal.purge_interval` | String | `1m` | The interval to trigger a flush.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.read_batch_size` | Integer | `128` | The read batch size.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.sync_write` | Bool | `false` | Whether to use sync write.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.enable_log_recycle` | Bool | `true` | Whether to reuse logically truncated log files.<br/>**It's only used when the provider is `raft_engine`**. |
@@ -453,7 +440,7 @@
 | `query` | -- | -- | The query engine options. |
 | `query.parallelism` | Integer | `0` | Parallelism of the query engine.<br/>Default to 0, which means the number of CPU cores. |
 | `storage` | -- | -- | The data storage options. |
-| `storage.data_home` | String | `./greptimedb_data` | The working home directory. |
+| `storage.data_home` | String | `./greptimedb_data/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
 | `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}`. An empty string means disabling. |
 | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. |
@@ -476,7 +463,6 @@
 | `storage.http_client.connect_timeout` | String | `30s` | The timeout for only the connect phase of a http client. |
 | `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.<br/>Also considered a total deadline. |
 | `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. |
-| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification<br/>**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
 | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. |
 | `region_engine.mito` | -- | -- | The Mito engine options. |
 | `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
@@ -538,18 +524,19 @@
 | `logging.dir` | String | `./greptimedb_data/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
-| `logging.otlp_endpoint` | String | `http://localhost:4318` | The OTLP tracing endpoint. |
+| `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
-| `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
+| `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
+| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
+| `export_metrics.self_import.db` | String | Unset | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
-| `export_metrics.remote_write.url` | String | `""` | The prometheus remote write endpoint that the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
+| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
@@ -562,16 +549,6 @@
 | `node_id` | Integer | Unset | The flownode identifier and should be unique in the cluster. |
 | `flow` | -- | -- | flow engine options. |
 | `flow.num_workers` | Integer | `0` | The number of flow worker in flownode.<br/>Not setting(or set to 0) this value will use the number of CPU cores divided by 2. |
-| `flow.batching_mode` | -- | -- | -- |
-| `flow.batching_mode.query_timeout` | String | `600s` | The default batching engine query timeout is 10 minutes. |
-| `flow.batching_mode.slow_query_threshold` | String | `60s` | will output a warn log for any query that runs for more that this threshold |
-| `flow.batching_mode.experimental_min_refresh_duration` | String | `5s` | The minimum duration between two queries execution by batching mode task |
-| `flow.batching_mode.grpc_conn_timeout` | String | `5s` | The gRPC connection timeout |
-| `flow.batching_mode.experimental_grpc_max_retries` | Integer | `3` | The gRPC max retry number |
-| `flow.batching_mode.experimental_frontend_scan_timeout` | String | `30s` | Flow wait for available frontend timeout,<br/>if failed to find available frontend after frontend_scan_timeout elapsed, return error<br/>which prevent flownode from starting |
-| `flow.batching_mode.experimental_frontend_activity_timeout` | String | `60s` | Frontend activity timeout<br/>if frontend is down(not sending heartbeat) for more than frontend_activity_timeout,<br/>it will be removed from the list that flownode use to connect |
-| `flow.batching_mode.experimental_max_filter_num_per_query` | Integer | `20` | Maximum number of filters allowed in a single query |
-| `flow.batching_mode.experimental_time_window_merge_threshold` | Integer | `3` | Time window merge distance |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.bind_addr` | String | `127.0.0.1:6800` | The address to bind the gRPC server. |
 | `grpc.server_addr` | String | `127.0.0.1:6800` | The address advertised to the metasrv,<br/>and used for connections from outside the host |
@@ -599,14 +576,11 @@
 | `logging.dir` | String | `./greptimedb_data/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
-| `logging.otlp_endpoint` | String | `http://localhost:4318` | The OTLP tracing endpoint. |
+| `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
-| `query` | -- | -- | -- |
-| `query.parallelism` | Integer | `1` | Parallelism of the query engine for query sent by flownode.<br/>Default to 1, so it won't use too much cpu or memory |

config/datanode.example.toml

@@ -44,13 +44,6 @@ runtime_size = 8
 max_recv_message_size = "512MB"
 ## The maximum send message size for gRPC server.
 max_send_message_size = "512MB"
-## Compression mode for datanode side Arrow IPC service. Available options:
-## - `none`: disable all compression
-## - `transport`: only enable gRPC transport compression (zstd)
-## - `arrow_ipc`: only enable Arrow IPC compression (lz4)
-## - `all`: enable all compression.
-## Default to `none`
-flight_compression = "arrow_ipc"
 
 ## gRPC server TLS options, see `mysql.tls` section.
 [grpc.tls]
@@ -129,11 +122,11 @@ dir = "./greptimedb_data/wal"
 ## **It's only used when the provider is `raft_engine`**.
 file_size = "128MB"
 
-## The threshold of the WAL size to trigger a purge.
+## The threshold of the WAL size to trigger a flush.
 ## **It's only used when the provider is `raft_engine`**.
 purge_threshold = "1GB"
 
-## The interval to trigger a purge.
+## The interval to trigger a flush.
 ## **It's only used when the provider is `raft_engine`**.
 purge_interval = "1m"
 
@@ -259,7 +252,7 @@ parallelism = 0
 ## The data storage options.
 [storage]
 ## The working home directory.
-data_home = "./greptimedb_data"
+data_home = "./greptimedb_data/"
 
 ## The storage type used to store the data.
 ## - `File`: the data is stored in the local file system.
@@ -367,10 +360,6 @@ timeout = "30s"
 ## The timeout for idle sockets being kept-alive.
 pool_idle_timeout = "90s"
 
-## To skip the ssl verification
-## **Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks.
-skip_ssl_validation = false
-
 # Custom storage options
 # [[storage.providers]]
 # name = "S3"
@@ -629,7 +618,7 @@ level = "info"
 enable_otlp_tracing = false
 
 ## The OTLP tracing endpoint.
-otlp_endpoint = "http://localhost:4318"
+otlp_endpoint = "http://localhost:4317"
 
 ## Whether to append logs to stdout.
 append_stdout = true
@@ -640,25 +629,30 @@ log_format = "text"
 ## The maximum amount of log files.
 max_log_files = 720
 
-## The OTLP tracing export protocol. Can be `grpc`/`http`.
-otlp_export_protocol = "http"
-
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
 [logging.tracing_sample_ratio]
 default_ratio = 1.0
 
-## The datanode can export its metrics and send to Prometheus compatible service (e.g. `greptimedb` itself) from remote-write API.
+## The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
 ## This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
 [export_metrics]
+
 ## whether enable export metrics.
 enable = false
+
 ## The interval of export metrics.
 write_interval = "30s"
 
+## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
+## You must create the database before enabling it.
+[export_metrics.self_import]
+## @toml2docs:none-default
+db = "greptime_metrics"
+
 [export_metrics.remote_write]
-## The prometheus remote write endpoint that the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
+## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
 url = ""
 
 ## HTTP headers of Prometheus remote-write carry.

config/flownode.example.toml

@@ -7,29 +7,6 @@ node_id = 14
 ## The number of flow worker in flownode.
 ## Not setting(or set to 0) this value will use the number of CPU cores divided by 2.
 #+num_workers=0
-[flow.batching_mode]
-## The default batching engine query timeout is 10 minutes.
-#+query_timeout="600s"
-## will output a warn log for any query that runs for more that this threshold
-#+slow_query_threshold="60s"
-## The minimum duration between two queries execution by batching mode task
-#+experimental_min_refresh_duration="5s"
-## The gRPC connection timeout
-#+grpc_conn_timeout="5s"
-## The gRPC max retry number
-#+experimental_grpc_max_retries=3
-## Flow wait for available frontend timeout,
-## if failed to find available frontend after frontend_scan_timeout elapsed, return error
-## which prevent flownode from starting
-#+experimental_frontend_scan_timeout="30s"
-## Frontend activity timeout
-## if frontend is down(not sending heartbeat) for more than frontend_activity_timeout,
-## it will be removed from the list that flownode use to connect
-#+experimental_frontend_activity_timeout="60s"
-## Maximum number of filters allowed in a single query
-#+experimental_max_filter_num_per_query=20
-## Time window merge distance
-#+experimental_time_window_merge_threshold=3
 
 ## The gRPC server options.
 [grpc]
@@ -106,7 +83,7 @@ level = "info"
 enable_otlp_tracing = false
 
 ## The OTLP tracing endpoint.
-otlp_endpoint = "http://localhost:4318"
+otlp_endpoint = "http://localhost:4317"
 
 ## Whether to append logs to stdout.
 append_stdout = true
@@ -117,9 +94,6 @@ log_format = "text"
 ## The maximum amount of log files.
 max_log_files = 720
 
-## The OTLP tracing export protocol. Can be `grpc`/`http`.
-otlp_export_protocol = "http"
-
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
@@ -131,8 +105,3 @@ default_ratio = 1.0
 ## The tokio console address.
 ## @toml2docs:none-default
 #+ tokio_console_addr = "127.0.0.1"
-
-[query]
-## Parallelism of the query engine for query sent by flownode.
-## Default to 1, so it won't use too much cpu or memory
-parallelism = 1

config/frontend.example.toml

@@ -37,12 +37,6 @@ enable_cors = true
 ## Customize allowed origins for HTTP CORS.
 ## @toml2docs:none-default
 cors_allowed_origins = ["https://example.com"]
-## Whether to enable validation for Prometheus remote write requests.
-## Available options:
-## - strict: deny invalid UTF-8 strings (default).
-## - lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).
-## - unchecked: do not valid strings.
-prom_validation_mode = "strict"
 
 ## The gRPC server options.
 [grpc]
@@ -54,13 +48,6 @@ bind_addr = "127.0.0.1:4001"
 server_addr = "127.0.0.1:4001"
 ## The number of server worker threads.
 runtime_size = 8
-## Compression mode for frontend side Arrow IPC service. Available options:
-## - `none`: disable all compression
-## - `transport`: only enable gRPC transport compression (zstd)
-## - `arrow_ipc`: only enable Arrow IPC compression (lz4)
-## - `all`: enable all compression.
-## Default to `none`
-flight_compression = "arrow_ipc"
 
 ## gRPC server TLS options, see `mysql.tls` section.
 [grpc.tls]
@@ -218,7 +205,7 @@ level = "info"
 enable_otlp_tracing = false
 
 ## The OTLP tracing endpoint.
-otlp_endpoint = "http://localhost:4318"
+otlp_endpoint = "http://localhost:4317"
 
 ## Whether to append logs to stdout.
 append_stdout = true
@@ -229,9 +216,6 @@ log_format = "text"
 ## The maximum amount of log files.
 max_log_files = 720
 
-## The OTLP tracing export protocol. Can be `grpc`/`http`.
-otlp_export_protocol = "http"
-
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
@@ -257,16 +241,24 @@ sample_ratio = 1.0
 ## The TTL of the `slow_queries` system table. Default is `30d` when `record_type` is `system_table`.
 ttl = "30d"
 
-## The frontend can export its metrics and send to Prometheus compatible service (e.g. `greptimedb` itself) from remote-write API.
+## The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
 ## This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
 [export_metrics]
+
 ## whether enable export metrics.
 enable = false
+
 ## The interval of export metrics.
 write_interval = "30s"
 
+## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
+## You must create the database before enabling it.
+[export_metrics.self_import]
+## @toml2docs:none-default
+db = "greptime_metrics"
+
 [export_metrics.remote_write]
-## The prometheus remote write endpoint that the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
+## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
 url = ""
 
 ## HTTP headers of Prometheus remote-write carry.

config/metasrv.example.toml

@@ -1,5 +1,13 @@
 ## The working home directory.
-data_home = "./greptimedb_data"
+data_home = "./greptimedb_data/metasrv/"
+
+## The bind address of metasrv.
+bind_addr = "127.0.0.1:3002"
+
+## The communication server address for the frontend and datanode to connect to metasrv.
+## If left empty or unset, the server will automatically use the IP address of the first network interface
+## on the host, with the same port number as the one specified in `bind_addr`.
+server_addr = "127.0.0.1:3002"
 
 ## Store server address default to etcd store.
 ## For postgres store, the format is:
@@ -16,7 +24,6 @@ store_key_prefix = ""
 ## - `etcd_store` (default value)
 ## - `memory_store`
 ## - `postgres_store`
-## - `mysql_store`
 backend = "etcd_store"
 
 ## Table name in RDS to store metadata. Effect when using a RDS kvbackend.
@@ -43,11 +50,6 @@ use_memory_store = false
 ## - Using shared storage (e.g., s3).
 enable_region_failover = false
 
-## The delay before starting region failure detection.
-## This delay helps prevent Metasrv from triggering unnecessary region failovers before all Datanodes are fully started.
-## Especially useful when the cluster is not deployed with GreptimeDB Operator and maintenance mode is not enabled.
-region_failure_detector_initialization_delay = '10m'
-
 ## Whether to allow region failover on local WAL.
 ## **This option is not recommended to be set to true, because it may lead to data loss during failover.**
 allow_region_failover_on_local_wal = false
@@ -65,32 +67,6 @@ node_max_idle_time = "24hours"
 ## The number of threads to execute the runtime for global write operations.
 #+ compact_rt_size = 4
 
-## The gRPC server options.
-[grpc]
-## The address to bind the gRPC server.
-bind_addr = "127.0.0.1:3002"
-## The communication server address for the frontend and datanode to connect to metasrv.
-## If left empty or unset, the server will automatically use the IP address of the first network interface
-## on the host, with the same port number as the one specified in `bind_addr`.
-server_addr = "127.0.0.1:3002"
-## The number of server worker threads.
-runtime_size = 8
-## The maximum receive message size for gRPC server.
-max_recv_message_size = "512MB"
-## The maximum send message size for gRPC server.
-max_send_message_size = "512MB"
-
-## The HTTP server options.
-[http]
-## The address to bind the HTTP server.
-addr = "127.0.0.1:4000"
-## HTTP request timeout. Set to 0 to disable timeout.
-timeout = "0s"
-## HTTP request body limit.
-## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
-## Set to 0 to disable limit.
-body_limit = "64MB"
-
 ## Procedure storage options.
 [procedure]
 
@@ -225,7 +201,7 @@ level = "info"
 enable_otlp_tracing = false
 
 ## The OTLP tracing endpoint.
-otlp_endpoint = "http://localhost:4318"
+otlp_endpoint = "http://localhost:4317"
 
 ## Whether to append logs to stdout.
 append_stdout = true
@@ -236,25 +212,30 @@ log_format = "text"
 ## The maximum amount of log files.
 max_log_files = 720
 
-## The OTLP tracing export protocol. Can be `grpc`/`http`.
-otlp_export_protocol = "http"
-
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
 [logging.tracing_sample_ratio]
 default_ratio = 1.0
 
-## The metasrv can export its metrics and send to Prometheus compatible service (e.g. `greptimedb` itself) from remote-write API.
+## The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
 ## This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
 [export_metrics]
+
 ## whether enable export metrics.
 enable = false
+
 ## The interval of export metrics.
 write_interval = "30s"
 
+## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
+## You must create the database before enabling it.
+[export_metrics.self_import]
+## @toml2docs:none-default
+db = "greptime_metrics"
+
 [export_metrics.remote_write]
-## The prometheus remote write endpoint that the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
+## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
 url = ""
 
 ## HTTP headers of Prometheus remote-write carry.

config/standalone.example.toml

@@ -43,13 +43,6 @@ enable_cors = true
 ## @toml2docs:none-default
 cors_allowed_origins = ["https://example.com"]
 
-## Whether to enable validation for Prometheus remote write requests.
-## Available options:
-## - strict: deny invalid UTF-8 strings (default).
-## - lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).
-## - unchecked: do not valid strings.
-prom_validation_mode = "strict"
-
 ## The gRPC server options.
 [grpc]
 ## The address to bind the gRPC server.
@@ -350,7 +343,7 @@ parallelism = 0
 ## The data storage options.
 [storage]
 ## The working home directory.
-data_home = "./greptimedb_data"
+data_home = "./greptimedb_data/"
 
 ## The storage type used to store the data.
 ## - `File`: the data is stored in the local file system.
@@ -458,10 +451,6 @@ timeout = "30s"
 ## The timeout for idle sockets being kept-alive.
 pool_idle_timeout = "90s"
 
-## To skip the ssl verification
-## **Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks.
-skip_ssl_validation = false
-
 # Custom storage options
 # [[storage.providers]]
 # name = "S3"
@@ -720,7 +709,7 @@ level = "info"
 enable_otlp_tracing = false
 
 ## The OTLP tracing endpoint.
-otlp_endpoint = "http://localhost:4318"
+otlp_endpoint = "http://localhost:4317"
 
 ## Whether to append logs to stdout.
 append_stdout = true
@@ -731,9 +720,6 @@ log_format = "text"
 ## The maximum amount of log files.
 max_log_files = 720
 
-## The OTLP tracing export protocol. Can be `grpc`/`http`.
-otlp_export_protocol = "http"
-
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
@@ -757,11 +743,13 @@ default_ratio = 1.0
 ## @toml2docs:none-default
 #+ sample_ratio = 1.0
 
-## The standalone can export its metrics and send to Prometheus compatible service (e.g. `greptimedb`) from remote-write API.
+## The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
 ## This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
 [export_metrics]
+
 ## whether enable export metrics.
 enable = false
+
 ## The interval of export metrics.
 write_interval = "30s"
 
@@ -772,7 +760,7 @@ write_interval = "30s"
 db = "greptime_metrics"
 
 [export_metrics.remote_write]
-## The prometheus remote write endpoint that the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
+## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
 url = ""
 
 ## HTTP headers of Prometheus remote-write carry.

cyborg/bin/bump-doc-version.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2023 Greptime Team
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as core from "@actions/core";
+import {obtainClient} from "@/common";
+
+async function triggerWorkflow(workflowId: string, version: string) {
+  const docsClient = obtainClient("DOCS_REPO_TOKEN")
+  try {
+    await docsClient.rest.actions.createWorkflowDispatch({
+      owner: "GreptimeTeam",
+      repo: "docs",
+      workflow_id: workflowId,
+      ref: "main",
+      inputs: {
+        version,
+      },
+    });
+    console.log(`Successfully triggered ${workflowId} workflow with version ${version}`);
+  } catch (error) {
+    core.setFailed(`Failed to trigger workflow: ${error.message}`);
+  }
+}
+
+function determineWorkflow(version: string): [string, string] {
+  // Check if it's a nightly version
+  if (version.includes('nightly')) {
+    return ['bump-nightly-version.yml', version];
+  }
+
+  const parts = version.split('.');
+
+  if (parts.length !== 3) {
+    throw new Error('Invalid version format');
+  }
+
+  // If patch version (last number) is 0, it's a major version
+  // Return only major.minor version
+  if (parts[2] === '0') {
+    return ['bump-version.yml', `${parts[0]}.${parts[1]}`];
+  }
+
+  // Otherwise it's a patch version, use full version
+  return ['bump-patch-version.yml', version];
+}
+
+const version = process.env.VERSION;
+if (!version) {
+  core.setFailed("VERSION environment variable is required");
+  process.exit(1);
+}
+
+// Remove 'v' prefix if exists
+const cleanVersion = version.startsWith('v') ? version.slice(1) : version;
+
+try {
+  const [workflowId, apiVersion] = determineWorkflow(cleanVersion);
+  triggerWorkflow(workflowId, apiVersion);
+} catch (error) {
+  core.setFailed(`Error processing version: ${error.message}`);
+  process.exit(1);
+}

cyborg/bin/bump-versions.ts

@@ -1,156 +0,0 @@
-/*
- * Copyright 2023 Greptime Team
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import * as core from "@actions/core";
-import {obtainClient} from "@/common";
-
-interface RepoConfig {
-  tokenEnv: string;
-  repo: string;
-  workflowLogic: (version: string) => [string, string] | null;
-}
-
-const REPO_CONFIGS: Record<string, RepoConfig> = {
-  website: {
-    tokenEnv: "WEBSITE_REPO_TOKEN",
-    repo: "website",
-    workflowLogic: (version: string) => {
-      // Skip nightly versions for website
-      if (version.includes('nightly')) {
-        console.log('Nightly version detected for website, skipping workflow trigger.');
-        return null;
-      }
-      return ['bump-patch-version.yml', version];
-    }
-  },
-  demo: {
-    tokenEnv: "DEMO_REPO_TOKEN",
-    repo: "demo-scene",
-    workflowLogic: (version: string) => {
-      // Skip nightly versions for demo
-      if (version.includes('nightly')) {
-        console.log('Nightly version detected for demo, skipping workflow trigger.');
-        return null;
-      }
-      return ['bump-patch-version.yml', version];
-    }
-  },
-  docs: {
-    tokenEnv: "DOCS_REPO_TOKEN",
-    repo: "docs",
-    workflowLogic: (version: string) => {
-      // Check if it's a nightly version
-      if (version.includes('nightly')) {
-        return ['bump-nightly-version.yml', version];
-      }
-
-      const parts = version.split('.');
-      if (parts.length !== 3) {
-        throw new Error('Invalid version format');
-      }
-
-      // If patch version (last number) is 0, it's a major version
-      // Return only major.minor version
-      if (parts[2] === '0') {
-        return ['bump-version.yml', `${parts[0]}.${parts[1]}`];
-      }
-
-      // Otherwise it's a patch version, use full version
-      return ['bump-patch-version.yml', version];
-    }
-  }
-};
-
-async function triggerWorkflow(repoConfig: RepoConfig, workflowId: string, version: string) {
-  const client = obtainClient(repoConfig.tokenEnv);
-  try {
-    await client.rest.actions.createWorkflowDispatch({
-      owner: "GreptimeTeam",
-      repo: repoConfig.repo,
-      workflow_id: workflowId,
-      ref: "main",
-      inputs: {
-        version,
-      },
-    });
-    console.log(`Successfully triggered ${workflowId} workflow for ${repoConfig.repo} with version ${version}`);
-  } catch (error) {
-    core.setFailed(`Failed to trigger workflow for ${repoConfig.repo}: ${error.message}`);
-    throw error;
-  }
-}
-
-async function processRepo(repoName: string, version: string) {
-  const repoConfig = REPO_CONFIGS[repoName];
-  if (!repoConfig) {
-    throw new Error(`Unknown repository: ${repoName}`);
-  }
-
-  try {
-    const workflowResult = repoConfig.workflowLogic(version);
-    if (workflowResult === null) {
-      // Skip this repo (e.g., nightly version for website)
-      return;
-    }
-
-    const [workflowId, apiVersion] = workflowResult;
-    await triggerWorkflow(repoConfig, workflowId, apiVersion);
-  } catch (error) {
-    core.setFailed(`Error processing ${repoName} with version ${version}: ${error.message}`);
-    throw error;
-  }
-}
-
-async function main() {
-  const version = process.env.VERSION;
-  if (!version) {
-    core.setFailed("VERSION environment variable is required");
-    process.exit(1);
-  }
-
-  // Remove 'v' prefix if exists
-  const cleanVersion = version.startsWith('v') ? version.slice(1) : version;
-
-  // Get target repositories from environment variable
-  // Default to both if not specified
-  const targetRepos = process.env.TARGET_REPOS?.split(',').map(repo => repo.trim()) || ['website', 'docs'];
-
-  console.log(`Processing version ${cleanVersion} for repositories: ${targetRepos.join(', ')}`);
-
-  const errors: string[] = [];
-
-  // Process each repository
-  for (const repo of targetRepos) {
-    try {
-      await processRepo(repo, cleanVersion);
-    } catch (error) {
-      errors.push(`${repo}: ${error.message}`);
-    }
-  }
-
-  if (errors.length > 0) {
-    core.setFailed(`Failed to process some repositories: ${errors.join('; ')}`);
-    process.exit(1);
-  }
-
-  console.log('All repositories processed successfully');
-}
-
-// Execute main function
-main().catch((error) => {
-  core.setFailed(`Unexpected error: ${error.message}`);
-  process.exit(1);
-});

cyborg/bin/bump-website-version.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2023 Greptime Team
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as core from "@actions/core";
+import {obtainClient} from "@/common";
+
+async function triggerWorkflow(workflowId: string, version: string) {
+  const websiteClient = obtainClient("WEBSITE_REPO_TOKEN")
+  try {
+    await websiteClient.rest.actions.createWorkflowDispatch({
+      owner: "GreptimeTeam",
+      repo: "website",
+      workflow_id: workflowId,
+      ref: "main",
+      inputs: {
+        version,
+      },
+    });
+    console.log(`Successfully triggered ${workflowId} workflow with version ${version}`);
+  } catch (error) {
+    core.setFailed(`Failed to trigger workflow: ${error.message}`);
+  }
+}
+
+const version = process.env.VERSION;
+if (!version) {
+  core.setFailed("VERSION environment variable is required");
+  process.exit(1);
+}
+
+// Remove 'v' prefix if exists
+const cleanVersion = version.startsWith('v') ? version.slice(1) : version;
+
+if (cleanVersion.includes('nightly')) {
+  console.log('Nightly version detected, skipping workflow trigger.');
+  process.exit(0);
+}
+
+try {
+  triggerWorkflow('bump-patch-version.yml', cleanVersion);
+} catch (error) {
+  core.setFailed(`Error processing version: ${error.message}`);
+  process.exit(1);
+}

cyborg/bin/follow-up-docs-issue.ts

@@ -55,25 +55,12 @@ async function main() {
         await client.rest.issues.addLabels({
             owner, repo, issue_number: number, labels: [labelDocsRequired],
         })
-
-        // Get available assignees for the docs repo
-        const assigneesResponse = await docsClient.rest.issues.listAssignees({
-            owner: 'GreptimeTeam',
-            repo: 'docs',
-        })
-        const validAssignees = assigneesResponse.data.map(assignee => assignee.login)
-        core.info(`Available assignees: ${validAssignees.join(', ')}`)
-
-        // Check if the actor is a valid assignee, otherwise fallback to fengjiachun
-        const assignee = validAssignees.includes(actor) ? actor : 'fengjiachun'
-        core.info(`Assigning issue to: ${assignee}`)
-
         await docsClient.rest.issues.create({
             owner: 'GreptimeTeam',
             repo: 'docs',
             title: `Update docs for ${title}`,
             body: `A document change request is generated from ${html_url}`,
-            assignee: assignee,
+            assignee: actor,
         }).then((res) => {
             core.info(`Created issue ${res.data}`)
         })

docs/benchmarks/log/README.md

@@ -48,4 +48,4 @@ Please refer to [SQL query](./query.sql) for GreptimeDB and Clickhouse, and [que
 
 ## Addition
 - You can tune GreptimeDB's configuration to get better performance.
-- You can setup GreptimeDB to use S3 as storage, see [here](https://docs.greptime.com/user-guide/deployments-administration/configuration#storage-options).
+- You can setup GreptimeDB to use S3 as storage, see [here](https://docs.greptime.com/user-guide/deployments/configuration#storage-options).

docs/how-to/how-to-profile-memory.md

@@ -1,6 +1,6 @@
 # Profile memory usage of GreptimeDB
 
-This crate provides an easy approach to dump memory profiling info. A set of ready to use scripts is provided in [docs/how-to/memory-profile-scripts](./memory-profile-scripts/scripts).
+This crate provides an easy approach to dump memory profiling info. A set of ready to use scripts is provided in [docs/how-to/memory-profile-scripts](docs/how-to/memory-profile-scripts).
 
 ## Prerequisites
 ### jemalloc

docs/how-to/how-to-write-aggregate-function.md

@@ -1,4 +1,4 @@
-Currently, our query engine is based on DataFusion, so all aggregate function is executed by DataFusion, through its UDAF interface. You can find DataFusion's UDAF example [here](https://github.com/apache/datafusion/tree/main/datafusion-examples/examples/simple_udaf.rs). Basically, we provide the same way as DataFusion to write aggregate functions: both are centered in a struct called "Accumulator" to accumulates states along the way in aggregation.
+Currently, our query engine is based on DataFusion, so all aggregate function is executed by DataFusion, through its UDAF interface. You can find DataFusion's UDAF example [here](https://github.com/apache/arrow-datafusion/blob/arrow2/datafusion-examples/examples/simple_udaf.rs). Basically, we provide the same way as DataFusion to write aggregate functions: both are centered in a struct called "Accumulator" to accumulates states along the way in aggregation.
 
 However, DataFusion's UDAF implementation has a huge restriction, that it requires user to provide a concrete "Accumulator". Take `Median` aggregate function for example, to aggregate a `u32` datatype column, you have to write a `MedianU32`, and use `SELECT MEDIANU32(x)` in SQL. `MedianU32` cannot be used to aggregate a `i32` datatype column. Or, there's another way: you can use a special type that can hold all kinds of data (like our `Value` enum or Arrow's `ScalarValue`), and `match` all the way up to do aggregate calculations. It might work, though rather tedious. (But I think it's DataFusion's preferred way to write UDAF.)
 

docs/rfcs/2023-02-01-table-compaction.md

@@ -76,7 +76,7 @@ pub trait CompactionStrategy {
 ```
 
 The most suitable compaction strategy for time-series scenario would be 
-a hybrid strategy that combines time window compaction with size-tired compaction, just like [Cassandra](https://cassandra.apache.org/doc/latest/cassandra/managing/operating/compaction/twcs.html) and [ScyllaDB](https://docs.scylladb.com/stable/architecture/compaction/compaction-strategies.html#time-window-compaction-strategy-twcs) does.
+a hybrid strategy that combines time window compaction with size-tired compaction, just like [Cassandra](https://cassandra.apache.org/doc/latest/cassandra/operating/compaction/twcs.html) and [ScyllaDB](https://docs.scylladb.com/stable/architecture/compaction/compaction-strategies.html#time-window-compaction-strategy-twcs) does.
 
 We can first group SSTs in level n into buckets according to some predefined time window. Within that window, 
 SSTs are compacted in a size-tired manner (find SSTs with similar size and compact them to level n+1). 

docs/rfcs/2024-01-17-dataflow-framework.md

@@ -28,7 +28,7 @@ In order to do those things while maintaining a low memory footprint, you need t
 - Greptime Flow's is built on top of [Hydroflow](https://github.com/hydro-project/hydroflow).
 - We have three choices for the Dataflow/Streaming process framework for our simple continuous aggregation feature:
 1. Based on the timely/differential dataflow crate that [materialize](https://github.com/MaterializeInc/materialize) based on. Later, it's proved too obscure for a simple usage, and is hard to customize memory usage control.
-2. Based on a simple dataflow framework that we write from ground up, like what [arroyo](https://www.arroyo.dev/) or [risingwave](https://www.risingwave.dev/) did, for example the core streaming logic of [arroyo](https://github.com/ArroyoSystems/arroyo/blob/master/crates/arroyo-datastream/src/lib.rs) only takes up to 2000 line of codes. However, it means maintaining another layer of dataflow framework, which might seem easy in the beginning, but I fear it might be too burdensome to maintain once we need more features.
+2. Based on a simple dataflow framework that we write from ground up, like what [arroyo](https://www.arroyo.dev/) or [risingwave](https://www.risingwave.dev/) did, for example the core streaming logic of [arroyo](https://github.com/ArroyoSystems/arroyo/blob/master/arroyo-datastream/src/lib.rs) only takes up to 2000 line of codes. However, it means maintaining another layer of dataflow framework, which might seem easy in the beginning, but I fear it might be too burdensome to maintain once we need more features.
 3. Based on a simple and lower level dataflow framework that someone else write, like [hydroflow](https://github.com/hydro-project/hydroflow), this approach combines the best of both worlds. Firstly, it boasts ease of comprehension and customization. Secondly, the dataflow framework offers precisely the necessary features for crafting uncomplicated single-node dataflow programs while delivering decent performance.
 
 Hence, we choose the third option, and use a simple logical plan that's anagonistic to the underlying dataflow framework, as it only describe how the dataflow graph should be doing, not how it do that. And we built operator in hydroflow to execute the plan. And the result hydroflow graph is wrapped in a engine that only support data in/out and tick event to flush and compute the result. This provide a thin middle layer that's easy to maintain and allow switching to other dataflow framework if necessary.

flake.lock

@@ -41,16 +41,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1748162331,
-        "narHash": "sha256-rqc2RKYTxP3tbjA+PB3VMRQNnjesrT0pEofXQTrMsS8=",
+        "lastModified": 1745487689,
+        "narHash": "sha256-FQoi3R0NjQeBAsEOo49b5tbDPcJSMWc3QhhaIi9eddw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7c43f080a7f28b2774f3b3f43234ca11661bf334",
+        "rev": "5630cf13cceac06cefe9fc607e8dfa8fb342dde3",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-25.05",
+        "ref": "nixos-24.11",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -2,7 +2,7 @@
   description = "Development environment flake";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
     fenix = {
       url = "github:nix-community/fenix";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -51,7 +51,6 @@
           ];
 
           LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;
-          NIX_HARDENING_ENABLE = "";
         };
       });
 }

grafana/README.md

@@ -9,7 +9,7 @@ We highly recommend using the self-monitoring feature provided by [GreptimeDB Op
 - **Metrics Dashboards**
 
   - `dashboards/metrics/cluster/dashboard.json`: The Grafana dashboard for the GreptimeDB cluster. Read the [dashboard.md](./dashboards/metrics/cluster/dashboard.md) for more details.
-
+  
   - `dashboards/metrics/standalone/dashboard.json`: The Grafana dashboard for the standalone GreptimeDB instance. **It's generated from the `cluster/dashboard.json` by removing the instance filter through the `make dashboards` command**. Read the [dashboard.md](./dashboards/metrics/standalone/dashboard.md) for more details.
 
 - **Logs Dashboard**
@@ -83,7 +83,7 @@ If you use the [Helm Chart](https://github.com/GreptimeTeam/helm-charts) to depl
 - `monitoring.enabled=true`: Deploys a standalone GreptimeDB instance dedicated to monitoring the cluster;
 - `grafana.enabled=true`: Deploys Grafana and automatically imports the monitoring dashboard;
 
-The standalone GreptimeDB instance will collect metrics from your cluster, and the dashboard will be available in the Grafana UI. For detailed deployment instructions, please refer to our [Kubernetes deployment guide](https://docs.greptime.com/user-guide/deployments-administration/deploy-on-kubernetes/overview).
+The standalone GreptimeDB instance will collect metrics from your cluster, and the dashboard will be available in the Grafana UI. For detailed deployment instructions, please refer to our [Kubernetes deployment guide](https://docs.greptime.com/nightly/user-guide/deployments/deploy-on-kubernetes/getting-started).
 
 ### Self-host Prometheus and import dashboards manually
 

grafana/dashboards/metrics/cluster/dashboard.md

@@ -60,7 +60,7 @@
 | Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
 | Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
 | Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
-| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
+| Compaction P99 per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
 | Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` |
 | WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
 | Cached Bytes per Instance | `greptime_mito_cache_bytes{instance=~"$datanode"}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
@@ -69,39 +69,27 @@
 | Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
 | Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
 | Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`<br/>`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` |
-| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`<br/>`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
-| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`<br/>`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
-| Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`<br/>`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
+| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`<br/>`sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
 # OpenDAL
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
 | QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read\|Reader::read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Read QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
+| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation="read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-{{scheme}}` |
+| Write QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="write"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-{{scheme}}` |
+| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
 | List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
 | List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
 | Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list\|Writer::write\|Writer::close\|Reader::read"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
 | Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
 | OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
 # Metasrv
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
-| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`<br/>`greptime_meta_region_migration_stat{datanode_type="desc"}` | `status-history` | Counter of region migration by source and destination | `prometheus` | -- | `from-datanode-{{datanode_id}}` |
-| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
-| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `binBps` | `Datanode-{{datanode_id}}-writeload` |
-| Rate of SQL Executions (RDS) | `rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])` | `timeseries` | Displays the rate of SQL executions processed by the Meta service using the RDS backend. | `prometheus` | `none` | `{{pod}} {{op}} {{type}} {{result}} ` |
-| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))` | `timeseries` | Measures the response time of SQL executions via the RDS backend.  | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
-| Handler Execution Latency | `histogram_quantile(0.90, sum by(pod, le, name) (
-  rate(greptime_meta_handler_execute_bucket[$__rate_interval])
-))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.<br/> | `prometheus` | `s` | `{{pod}} {{name}} p90` |
-| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.<br/> | `prometheus` | `bytes` | `{{pod}}` |
-| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}` |
-| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
-| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
-| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
+| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`<br/>`greptime_meta_region_migration_stat{datanode_type="desc"}` | `state-timeline` | Counter of region migration by source and destination | `prometheus` | `none` | `from-datanode-{{datanode_id}}` |
+| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `__auto` |
+| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `__auto` |
 # Flownode
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |

grafana/dashboards/metrics/cluster/dashboard.yaml

@@ -487,7 +487,7 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{ instance }}]-[{{pod}}]'
-        - title: Compaction Elapsed Time per Instance by Stage
+        - title: Compaction P99 per Instance by Stage
           type: timeseries
           description: Compaction latency by stage
           unit: s
@@ -497,11 +497,6 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99'
-            - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg'
         - title: Compaction P99 per Instance
           type: timeseries
           description: Compaction P99 per Instance.
@@ -607,37 +602,7 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-P95'
-            - expr: sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG'
-        - title: Active Series and Field Builders Count
-          type: timeseries
-          description: Compaction oinput output bytes
-          unit: none
-          queries:
-            - expr: sum by(instance, pod) (greptime_mito_memtable_active_series_count)
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-series'
-            - expr: sum by(instance, pod) (greptime_mito_memtable_field_builder_count)
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-field_builders'
-        - title: Region Worker Convert Requests
-          type: timeseries
-          description: Per-stage elapsed time for region worker to decode requests.
-          unit: s
-          queries:
-            - expr: histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-P95'
-            - expr: sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))
+            - expr: sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
@@ -659,41 +624,41 @@ groups:
           description: Read QPS per Instance.
           unit: ops
           queries:
-            - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval]))
+            - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="read"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
         - title: Read P99 per Instance
           type: timeseries
           description: Read P99 per Instance.
           unit: s
           queries:
-            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read|Reader::read"}[$__rate_interval])))
+            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation="read"}[$__rate_interval])))
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+              legendFormat: '[{{instance}}]-[{{pod}}]-{{scheme}}'
         - title: Write QPS per Instance
           type: timeseries
           description: Write QPS per Instance.
           unit: ops
           queries:
-            - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval]))
+            - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="write"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+              legendFormat: '[{{instance}}]-[{{pod}}]-{{scheme}}'
         - title: Write P99 per Instance
           type: timeseries
           description: Write P99 per Instance.
           unit: s
           queries:
-            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write|Writer::close|write"}[$__rate_interval])))
+            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="write"}[$__rate_interval])))
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
         - title: List QPS per Instance
           type: timeseries
           description: List QPS per Instance.
@@ -729,7 +694,7 @@ groups:
           description: Other Request P99 per Instance.
           unit: s
           queries:
-            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read|write|list|Writer::write|Writer::close|Reader::read"}[$__rate_interval])))
+            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read|write|list"}[$__rate_interval])))
               datasource:
                 type: prometheus
                 uid: ${metrics}
@@ -756,8 +721,9 @@ groups:
     - title: Metasrv
       panels:
         - title: Region migration datanode
-          type: status-history
+          type: state-timeline
           description: Counter of region migration by source and destination
+          unit: none
           queries:
             - expr: greptime_meta_region_migration_stat{datanode_type="src"}
               datasource:
@@ -778,127 +744,17 @@ groups:
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '{{pod}}-{{state}}-{{error_type}}'
+              legendFormat: __auto
         - title: Datanode load
           type: timeseries
           description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: binBps
+          unit: none
           queries:
             - expr: greptime_datanode_load
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: Datanode-{{datanode_id}}-writeload
-        - title: Rate of SQL Executions (RDS)
-          type: timeseries
-          description: Displays the rate of SQL executions processed by the Meta service using the RDS backend.
-          unit: none
-          queries:
-            - expr: rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}} {{op}} {{type}} {{result}} '
-        - title: SQL Execution Latency (RDS)
-          type: timeseries
-          description: 'Measures the response time of SQL executions via the RDS backend. '
-          unit: ms
-          queries:
-            - expr: histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}} {{op}} {{type}} {{result}} p90'
-        - title: Handler Execution Latency
-          type: timeseries
-          description: |
-            Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
-          unit: s
-          queries:
-            - expr: |-
-                histogram_quantile(0.90, sum by(pod, le, name) (
-                  rate(greptime_meta_handler_execute_bucket[$__rate_interval])
-                ))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}} {{name}} p90'
-        - title: Heartbeat Packet Size
-          type: timeseries
-          description: |
-            Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
-          unit: bytes
-          queries:
-            - expr: histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}}'
-        - title: Meta Heartbeat Receive Rate
-          type: timeseries
-          description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: s
-          queries:
-            - expr: rate(greptime_meta_heartbeat_rate[$__rate_interval])
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}}'
-        - title: Meta KV Ops Latency
-          type: timeseries
-          description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: s
-          queries:
-            - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}}-{{op}} p99'
-        - title: Rate of meta KV Ops
-          type: timeseries
-          description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: none
-          queries:
-            - expr: rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}}-{{op}} p99'
-        - title: DDL Latency
-          type: timeseries
-          description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: s
-          queries:
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: CreateLogicalTables-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: CreateTable-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: CreateView-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: CreateFlow-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: DropTable-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: AlterTable-{{step}} p90
+              legendFormat: __auto
     - title: Flownode
       panels:
         - title: Flow Ingest / Output Rate

grafana/dashboards/metrics/standalone/dashboard.md

@@ -60,7 +60,7 @@
 | Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
 | Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
 | Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
-| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))`<br/>`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
+| Compaction P99 per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
 | Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` |
 | WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`<br/>`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
 | Cached Bytes per Instance | `greptime_mito_cache_bytes{}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
@@ -69,39 +69,27 @@
 | Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
 | Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
 | Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`<br/>`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` |
-| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`<br/>`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
-| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`<br/>`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
-| Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`<br/>`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
+| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`<br/>`sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
 # OpenDAL
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
 | QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read\|Reader::read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Read QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
+| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-{{scheme}}` |
+| Write QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="write"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-{{scheme}}` |
+| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
 | List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
 | List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
 | Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list\|Writer::write\|Writer::close\|Reader::read"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
 | Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
 | OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
 # Metasrv
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
-| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`<br/>`greptime_meta_region_migration_stat{datanode_type="desc"}` | `status-history` | Counter of region migration by source and destination | `prometheus` | -- | `from-datanode-{{datanode_id}}` |
-| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
-| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `binBps` | `Datanode-{{datanode_id}}-writeload` |
-| Rate of SQL Executions (RDS) | `rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])` | `timeseries` | Displays the rate of SQL executions processed by the Meta service using the RDS backend. | `prometheus` | `none` | `{{pod}} {{op}} {{type}} {{result}} ` |
-| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))` | `timeseries` | Measures the response time of SQL executions via the RDS backend.  | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
-| Handler Execution Latency | `histogram_quantile(0.90, sum by(pod, le, name) (
-  rate(greptime_meta_handler_execute_bucket[$__rate_interval])
-))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.<br/> | `prometheus` | `s` | `{{pod}} {{name}} p90` |
-| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.<br/> | `prometheus` | `bytes` | `{{pod}}` |
-| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}` |
-| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
-| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
-| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))`<br/>`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
+| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`<br/>`greptime_meta_region_migration_stat{datanode_type="desc"}` | `state-timeline` | Counter of region migration by source and destination | `prometheus` | `none` | `from-datanode-{{datanode_id}}` |
+| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `__auto` |
+| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `__auto` |
 # Flownode
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |

grafana/dashboards/metrics/standalone/dashboard.yaml

@@ -487,7 +487,7 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{ instance }}]-[{{pod}}]'
-        - title: Compaction Elapsed Time per Instance by Stage
+        - title: Compaction P99 per Instance by Stage
           type: timeseries
           description: Compaction latency by stage
           unit: s
@@ -497,11 +497,6 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99'
-            - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg'
         - title: Compaction P99 per Instance
           type: timeseries
           description: Compaction P99 per Instance.
@@ -607,37 +602,7 @@ groups:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-P95'
-            - expr: sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG'
-        - title: Active Series and Field Builders Count
-          type: timeseries
-          description: Compaction oinput output bytes
-          unit: none
-          queries:
-            - expr: sum by(instance, pod) (greptime_mito_memtable_active_series_count)
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-series'
-            - expr: sum by(instance, pod) (greptime_mito_memtable_field_builder_count)
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-field_builders'
-        - title: Region Worker Convert Requests
-          type: timeseries
-          description: Per-stage elapsed time for region worker to decode requests.
-          unit: s
-          queries:
-            - expr: histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-P95'
-            - expr: sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))
+            - expr: sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
@@ -659,41 +624,41 @@ groups:
           description: Read QPS per Instance.
           unit: ops
           queries:
-            - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"read|Reader::read"}[$__rate_interval]))
+            - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="read"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
         - title: Read P99 per Instance
           type: timeseries
           description: Read P99 per Instance.
           unit: s
           queries:
-            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read|Reader::read"}[$__rate_interval])))
+            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="read"}[$__rate_interval])))
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+              legendFormat: '[{{instance}}]-[{{pod}}]-{{scheme}}'
         - title: Write QPS per Instance
           type: timeseries
           description: Write QPS per Instance.
           unit: ops
           queries:
-            - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"write|Writer::write|Writer::close"}[$__rate_interval]))
+            - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="write"}[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+              legendFormat: '[{{instance}}]-[{{pod}}]-{{scheme}}'
         - title: Write P99 per Instance
           type: timeseries
           description: Write P99 per Instance.
           unit: s
           queries:
-            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ "Writer::write|Writer::close|write"}[$__rate_interval])))
+            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="write"}[$__rate_interval])))
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+              legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
         - title: List QPS per Instance
           type: timeseries
           description: List QPS per Instance.
@@ -729,7 +694,7 @@ groups:
           description: Other Request P99 per Instance.
           unit: s
           queries:
-            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read|write|list|Writer::write|Writer::close|Reader::read"}[$__rate_interval])))
+            - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read|write|list"}[$__rate_interval])))
               datasource:
                 type: prometheus
                 uid: ${metrics}
@@ -756,8 +721,9 @@ groups:
     - title: Metasrv
       panels:
         - title: Region migration datanode
-          type: status-history
+          type: state-timeline
           description: Counter of region migration by source and destination
+          unit: none
           queries:
             - expr: greptime_meta_region_migration_stat{datanode_type="src"}
               datasource:
@@ -778,127 +744,17 @@ groups:
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: '{{pod}}-{{state}}-{{error_type}}'
+              legendFormat: __auto
         - title: Datanode load
           type: timeseries
           description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: binBps
+          unit: none
           queries:
             - expr: greptime_datanode_load
               datasource:
                 type: prometheus
                 uid: ${metrics}
-              legendFormat: Datanode-{{datanode_id}}-writeload
-        - title: Rate of SQL Executions (RDS)
-          type: timeseries
-          description: Displays the rate of SQL executions processed by the Meta service using the RDS backend.
-          unit: none
-          queries:
-            - expr: rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}} {{op}} {{type}} {{result}} '
-        - title: SQL Execution Latency (RDS)
-          type: timeseries
-          description: 'Measures the response time of SQL executions via the RDS backend. '
-          unit: ms
-          queries:
-            - expr: histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}} {{op}} {{type}} {{result}} p90'
-        - title: Handler Execution Latency
-          type: timeseries
-          description: |
-            Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
-          unit: s
-          queries:
-            - expr: |-
-                histogram_quantile(0.90, sum by(pod, le, name) (
-                  rate(greptime_meta_handler_execute_bucket[$__rate_interval])
-                ))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}} {{name}} p90'
-        - title: Heartbeat Packet Size
-          type: timeseries
-          description: |
-            Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
-          unit: bytes
-          queries:
-            - expr: histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}}'
-        - title: Meta Heartbeat Receive Rate
-          type: timeseries
-          description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: s
-          queries:
-            - expr: rate(greptime_meta_heartbeat_rate[$__rate_interval])
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}}'
-        - title: Meta KV Ops Latency
-          type: timeseries
-          description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: s
-          queries:
-            - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}}-{{op}} p99'
-        - title: Rate of meta KV Ops
-          type: timeseries
-          description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: none
-          queries:
-            - expr: rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: '{{pod}}-{{op}} p99'
-        - title: DDL Latency
-          type: timeseries
-          description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
-          unit: s
-          queries:
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: CreateLogicalTables-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: CreateTable-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: CreateView-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: CreateFlow-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: DropTable-{{step}} p90
-            - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))
-              datasource:
-                type: prometheus
-                uid: ${metrics}
-              legendFormat: AlterTable-{{step}} p90
+              legendFormat: __auto
     - title: Flownode
       panels:
         - title: Flow Ingest / Output Rate

grafana/scripts/gen-dashboards.sh

@@ -6,7 +6,7 @@ DAC_IMAGE=ghcr.io/zyy17/dac:20250423-522bd35
 
 remove_instance_filters() {
   # Remove the instance filters for the standalone dashboards.
-  sed -E 's/instance=~\\"(\$datanode|\$frontend|\$metasrv|\$flownode)\\",?//g' "$CLUSTER_DASHBOARD_DIR/dashboard.json" > "$STANDALONE_DASHBOARD_DIR/dashboard.json"
+  sed 's/instance=~\\"$datanode\\",//; s/instance=~\\"$datanode\\"//; s/instance=~\\"$frontend\\",//; s/instance=~\\"$frontend\\"//; s/instance=~\\"$metasrv\\",//; s/instance=~\\"$metasrv\\"//; s/instance=~\\"$flownode\\",//; s/instance=~\\"$flownode\\"//;' $CLUSTER_DASHBOARD_DIR/dashboard.json > $STANDALONE_DASHBOARD_DIR/dashboard.json
 }
 
 generate_intermediate_dashboards_and_docs() {

licenserc.toml

@@ -26,15 +26,6 @@ excludes = [
     "src/common/base/src/secrets.rs",
     "src/servers/src/repeated_field.rs",
     "src/servers/src/http/test_helpers.rs",
-    # enterprise
-    "src/common/meta/src/rpc/ddl/trigger.rs",
-    "src/operator/src/expr_helper/trigger.rs",
-    "src/sql/src/statements/create/trigger.rs",
-    "src/sql/src/statements/show/trigger.rs",
-    "src/sql/src/statements/drop/trigger.rs",
-    "src/sql/src/parsers/create_parser/trigger.rs",
-    "src/sql/src/parsers/show_parser/trigger.rs",
-    "src/mito2/src/extension.rs",
 ]
 
 [properties]

scripts/check-super-imports.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 import os
+import re
 from multiprocessing import Pool
-from pathlib import Path
 
 
 def find_rust_files(directory):
@@ -24,10 +24,6 @@ def find_rust_files(directory):
         if "test" in root.lower():
             continue
 
-        # Skip the target directory
-        if "target" in Path(root).parts:
-            continue
-
         for file in files:
             # Skip files with "test" in the filename
             if "test" in file.lower():

scripts/install.sh

@@ -53,54 +53,6 @@ get_arch_type() {
   esac
 }
 
-# Verify SHA256 checksum
-verify_sha256() {
-  file="$1"
-  expected_sha256="$2"
-
-  if command -v sha256sum >/dev/null 2>&1; then
-    actual_sha256=$(sha256sum "$file" | cut -d' ' -f1)
-  elif command -v shasum >/dev/null 2>&1; then
-    actual_sha256=$(shasum -a 256 "$file" | cut -d' ' -f1)
-  else
-    echo "Warning: No SHA256 verification tool found (sha256sum or shasum). Skipping checksum verification."
-    return 0
-  fi
-
-  if [ "$actual_sha256" = "$expected_sha256" ]; then
-    echo "SHA256 checksum verified successfully."
-    return 0
-  else
-    echo "Error: SHA256 checksum verification failed!"
-    echo "Expected: $expected_sha256"
-    echo "Actual: $actual_sha256"
-    return 1
-  fi
-}
-
-# Prompt for user confirmation (compatible with different shells)
-prompt_confirmation() {
-  message="$1"
-  printf "%s (y/N): " "$message"
-
-  # Try to read user input, fallback if read fails
-  answer=""
-  if read answer </dev/tty 2>/dev/null; then
-    case "$answer" in
-      [Yy]|[Yy][Ee][Ss])
-        return 0
-        ;;
-      *)
-        return 1
-        ;;
-    esac
-  else
-    echo ""
-    echo "Cannot read user input. Defaulting to No."
-    return 1
-  fi
-}
-
 download_artifact() {
   if [ -n "${OS_TYPE}" ] && [ -n "${ARCH_TYPE}" ]; then
     # Use the latest stable released version.
@@ -119,104 +71,17 @@ download_artifact() {
     fi
 
     echo "Downloading ${BIN}, OS: ${OS_TYPE}, Arch: ${ARCH_TYPE}, Version: ${VERSION}"
-    PKG_NAME="${BIN}-${OS_TYPE}-${ARCH_TYPE}-${VERSION}"
-    PACKAGE_NAME="${PKG_NAME}.tar.gz"
-    SHA256_FILE="${PKG_NAME}.sha256sum"
+    PACKAGE_NAME="${BIN}-${OS_TYPE}-${ARCH_TYPE}-${VERSION}.tar.gz"
 
     if [ -n "${PACKAGE_NAME}" ]; then
-      # Check if files already exist and prompt for override
-      if [ -f "${PACKAGE_NAME}" ]; then
-        echo "File ${PACKAGE_NAME} already exists."
-        if prompt_confirmation "Do you want to override it?"; then
-          echo "Overriding existing file..."
-          rm -f "${PACKAGE_NAME}"
-        else
-          echo "Skipping download. Using existing file."
-        fi
-      fi
-
-      if [ -f "${BIN}" ]; then
-        echo "Binary ${BIN} already exists."
-        if prompt_confirmation "Do you want to override it?"; then
-          echo "Will override existing binary..."
-          rm -f "${BIN}"
-        else
-          echo "Installation cancelled."
-          exit 0
-        fi
-      fi
-
-      # Download package if not exists
-      if [ ! -f "${PACKAGE_NAME}" ]; then
-        echo "Downloading ${PACKAGE_NAME}..."
-        # Use curl instead of wget for better compatibility
-        if command -v curl >/dev/null 2>&1; then
-          if ! curl -L -o "${PACKAGE_NAME}" "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${PACKAGE_NAME}"; then
-            echo "Error: Failed to download ${PACKAGE_NAME}"
-            exit 1
-          fi
-        elif command -v wget >/dev/null 2>&1; then
-          if ! wget -O "${PACKAGE_NAME}" "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${PACKAGE_NAME}"; then
-            echo "Error: Failed to download ${PACKAGE_NAME}"
-            exit 1
-          fi
-        else
-          echo "Error: Neither curl nor wget is available for downloading."
-          exit 1
-        fi
-      fi
-
-      # Download and verify SHA256 checksum
-      echo "Downloading SHA256 checksum..."
-      sha256_download_success=0
-      if command -v curl >/dev/null 2>&1; then
-        if curl -L -s -o "${SHA256_FILE}" "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${SHA256_FILE}" 2>/dev/null; then
-          sha256_download_success=1
-        fi
-      elif command -v wget >/dev/null 2>&1; then
-        if wget -q -O "${SHA256_FILE}" "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${SHA256_FILE}" 2>/dev/null; then
-          sha256_download_success=1
-        fi
-      fi
-
-      if [ $sha256_download_success -eq 1 ] && [ -f "${SHA256_FILE}" ]; then
-        expected_sha256=$(cat "${SHA256_FILE}" | cut -d' ' -f1)
-        if [ -n "$expected_sha256" ]; then
-          if ! verify_sha256 "${PACKAGE_NAME}" "${expected_sha256}"; then
-            echo "SHA256 verification failed. Removing downloaded file."
-            rm -f "${PACKAGE_NAME}" "${SHA256_FILE}"
-            exit 1
-          fi
-        else
-          echo "Warning: Could not parse SHA256 checksum from file."
-        fi
-        rm -f "${SHA256_FILE}"
-      else
-        echo "Warning: Could not download SHA256 checksum file. Skipping verification."
-      fi
+      wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${PACKAGE_NAME}"
 
       # Extract the binary and clean the rest.
-      echo "Extracting ${PACKAGE_NAME}..."
-      if ! tar xf "${PACKAGE_NAME}"; then
-        echo "Error: Failed to extract ${PACKAGE_NAME}"
-        exit 1
-      fi
-
-      # Find the binary in the extracted directory
-      extracted_dir="${PACKAGE_NAME%.tar.gz}"
-      if [ -f "${extracted_dir}/${BIN}" ]; then
-        mv "${extracted_dir}/${BIN}" "${PWD}/"
-        rm -f "${PACKAGE_NAME}"
-        rm -rf "${extracted_dir}"
-        chmod +x "${BIN}"
-        echo "Installation completed successfully!"
-        echo "Run './${BIN} --help' to get started"
-      else
-        echo "Error: Binary ${BIN} not found in extracted archive"
-        rm -f "${PACKAGE_NAME}"
-        rm -rf "${extracted_dir}"
-        exit 1
-      fi
+      tar xvf "${PACKAGE_NAME}" && \
+      mv "${PACKAGE_NAME%.tar.gz}/${BIN}" "${PWD}" && \
+      rm -r "${PACKAGE_NAME}" && \
+      rm -r "${PACKAGE_NAME%.tar.gz}" && \
+      echo "Run './${BIN} --help' to get started"
     fi
   fi
 }

src/api/src/region.rs

@@ -22,7 +22,6 @@ use greptime_proto::v1::region::RegionResponse as RegionResponseV1;
 pub struct RegionResponse {
     pub affected_rows: AffectedRows,
     pub extensions: HashMap<String, Vec<u8>>,
-    pub metadata: Vec<u8>,
 }
 
 impl RegionResponse {
@@ -30,7 +29,6 @@ impl RegionResponse {
         Self {
             affected_rows: region_response.affected_rows as _,
             extensions: region_response.extensions,
-            metadata: region_response.metadata,
         }
     }
 
@@ -39,16 +37,6 @@ impl RegionResponse {
         Self {
             affected_rows,
             extensions: Default::default(),
-            metadata: Vec::new(),
-        }
-    }
-
-    /// Creates one response with metadata.
-    pub fn from_metadata(metadata: Vec<u8>) -> Self {
-        Self {
-            affected_rows: 0,
-            extensions: Default::default(),
-            metadata,
         }
     }
 }

src/api/src/v1/column_def.rs

@@ -226,20 +226,18 @@ mod tests {
         assert!(options.is_none());
 
         let mut schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true)
-            .with_fulltext_options(FulltextOptions::new_unchecked(
-                true,
-                FulltextAnalyzer::English,
-                false,
-                FulltextBackend::Bloom,
-                10240,
-                0.01,
-            ))
+            .with_fulltext_options(FulltextOptions {
+                enable: true,
+                analyzer: FulltextAnalyzer::English,
+                case_sensitive: false,
+                backend: FulltextBackend::Bloom,
+            })
             .unwrap();
         schema.set_inverted_index(true);
         let options = options_from_column_schema(&schema).unwrap();
         assert_eq!(
             options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
-            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
+            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}"
         );
         assert_eq!(
             options.options.get(INVERTED_INDEX_GRPC_KEY).unwrap(),
@@ -249,18 +247,16 @@ mod tests {
 
     #[test]
     fn test_options_with_fulltext() {
-        let fulltext = FulltextOptions::new_unchecked(
-            true,
-            FulltextAnalyzer::English,
-            false,
-            FulltextBackend::Bloom,
-            10240,
-            0.01,
-        );
+        let fulltext = FulltextOptions {
+            enable: true,
+            analyzer: FulltextAnalyzer::English,
+            case_sensitive: false,
+            backend: FulltextBackend::Bloom,
+        };
         let options = options_from_fulltext(&fulltext).unwrap().unwrap();
         assert_eq!(
             options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
-            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
+            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}"
         );
     }
 

src/catalog/Cargo.toml

@@ -5,7 +5,6 @@ edition.workspace = true
 license.workspace = true
 
 [features]
-enterprise = []
 testing = []
 
 [lints]
@@ -18,10 +17,8 @@ arrow-schema.workspace = true
 async-stream.workspace = true
 async-trait.workspace = true
 bytes.workspace = true
-common-base.workspace = true
 common-catalog.workspace = true
 common-error.workspace = true
-common-frontend.workspace = true
 common-macro.workspace = true
 common-meta.workspace = true
 common-procedure.workspace = true

src/catalog/src/error.rs

@@ -277,26 +277,6 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-
-    #[snafu(display("Failed to invoke frontend services"))]
-    InvokeFrontend {
-        source: common_frontend::error::Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Meta client is not provided"))]
-    MetaClientMissing {
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to find frontend node: {}", addr))]
-    FrontendNotFound {
-        addr: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
 }
 
 impl Error {
@@ -365,10 +345,6 @@ impl ErrorExt for Error {
             Error::GetViewCache { source, .. } | Error::GetTableCache { source, .. } => {
                 source.status_code()
             }
-            Error::InvokeFrontend { source, .. } => source.status_code(),
-            Error::FrontendNotFound { .. } | Error::MetaClientMissing { .. } => {
-                StatusCode::Unexpected
-            }
         }
     }
 

src/catalog/src/kvbackend.rs

@@ -14,11 +14,9 @@
 
 pub use client::{CachedKvBackend, CachedKvBackendBuilder, MetaKvBackend};
 
-mod builder;
 mod client;
 mod manager;
 mod table_cache;
 
-pub use builder::KvBackendCatalogManagerBuilder;
 pub use manager::KvBackendCatalogManager;
 pub use table_cache::{new_table_cache, TableCache, TableCacheRef};

src/catalog/src/kvbackend/builder.rs

@@ -1,131 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use common_catalog::consts::DEFAULT_CATALOG_NAME;
-use common_meta::cache::LayeredCacheRegistryRef;
-use common_meta::key::flow::FlowMetadataManager;
-use common_meta::key::TableMetadataManager;
-use common_meta::kv_backend::KvBackendRef;
-use common_procedure::ProcedureManagerRef;
-use moka::sync::Cache;
-use partition::manager::PartitionRuleManager;
-
-#[cfg(feature = "enterprise")]
-use crate::information_schema::InformationSchemaTableFactoryRef;
-use crate::information_schema::{InformationExtensionRef, InformationSchemaProvider};
-use crate::kvbackend::manager::{SystemCatalog, CATALOG_CACHE_MAX_CAPACITY};
-use crate::kvbackend::KvBackendCatalogManager;
-use crate::process_manager::ProcessManagerRef;
-use crate::system_schema::pg_catalog::PGCatalogProvider;
-
-pub struct KvBackendCatalogManagerBuilder {
-    information_extension: InformationExtensionRef,
-    backend: KvBackendRef,
-    cache_registry: LayeredCacheRegistryRef,
-    procedure_manager: Option<ProcedureManagerRef>,
-    process_manager: Option<ProcessManagerRef>,
-    #[cfg(feature = "enterprise")]
-    extra_information_table_factories:
-        std::collections::HashMap<String, InformationSchemaTableFactoryRef>,
-}
-
-impl KvBackendCatalogManagerBuilder {
-    pub fn new(
-        information_extension: InformationExtensionRef,
-        backend: KvBackendRef,
-        cache_registry: LayeredCacheRegistryRef,
-    ) -> Self {
-        Self {
-            information_extension,
-            backend,
-            cache_registry,
-            procedure_manager: None,
-            process_manager: None,
-            #[cfg(feature = "enterprise")]
-            extra_information_table_factories: std::collections::HashMap::new(),
-        }
-    }
-
-    pub fn with_procedure_manager(mut self, procedure_manager: ProcedureManagerRef) -> Self {
-        self.procedure_manager = Some(procedure_manager);
-        self
-    }
-
-    pub fn with_process_manager(mut self, process_manager: ProcessManagerRef) -> Self {
-        self.process_manager = Some(process_manager);
-        self
-    }
-
-    /// Sets the extra information tables.
-    #[cfg(feature = "enterprise")]
-    pub fn with_extra_information_table_factories(
-        mut self,
-        factories: std::collections::HashMap<String, InformationSchemaTableFactoryRef>,
-    ) -> Self {
-        self.extra_information_table_factories = factories;
-        self
-    }
-
-    pub fn build(self) -> Arc<KvBackendCatalogManager> {
-        let Self {
-            information_extension,
-            backend,
-            cache_registry,
-            procedure_manager,
-            process_manager,
-            #[cfg(feature = "enterprise")]
-            extra_information_table_factories,
-        } = self;
-        Arc::new_cyclic(|me| KvBackendCatalogManager {
-            information_extension,
-            partition_manager: Arc::new(PartitionRuleManager::new(
-                backend.clone(),
-                cache_registry
-                    .get()
-                    .expect("Failed to get table_route_cache"),
-            )),
-            table_metadata_manager: Arc::new(TableMetadataManager::new(backend.clone())),
-            system_catalog: SystemCatalog {
-                catalog_manager: me.clone(),
-                catalog_cache: Cache::new(CATALOG_CACHE_MAX_CAPACITY),
-                pg_catalog_cache: Cache::new(CATALOG_CACHE_MAX_CAPACITY),
-                information_schema_provider: {
-                    let provider = InformationSchemaProvider::new(
-                        DEFAULT_CATALOG_NAME.to_string(),
-                        me.clone(),
-                        Arc::new(FlowMetadataManager::new(backend.clone())),
-                        process_manager.clone(),
-                        backend.clone(),
-                    );
-                    #[cfg(feature = "enterprise")]
-                    let provider = provider
-                        .with_extra_table_factories(extra_information_table_factories.clone());
-                    Arc::new(provider)
-                },
-                pg_catalog_provider: Arc::new(PGCatalogProvider::new(
-                    DEFAULT_CATALOG_NAME.to_string(),
-                    me.clone(),
-                )),
-                backend,
-                process_manager,
-                #[cfg(feature = "enterprise")]
-                extra_information_table_factories,
-            },
-            cache_registry,
-            procedure_manager,
-        })
-    }
-}

src/catalog/src/kvbackend/manager.rs

@@ -22,24 +22,21 @@ use common_catalog::consts::{
     PG_CATALOG_NAME,
 };
 use common_error::ext::BoxedError;
-use common_meta::cache::{
-    LayeredCacheRegistryRef, TableRoute, TableRouteCacheRef, ViewInfoCacheRef,
-};
+use common_meta::cache::{LayeredCacheRegistryRef, ViewInfoCacheRef};
 use common_meta::key::catalog_name::CatalogNameKey;
 use common_meta::key::flow::FlowMetadataManager;
 use common_meta::key::schema_name::SchemaNameKey;
-use common_meta::key::table_info::{TableInfoManager, TableInfoValue};
+use common_meta::key::table_info::TableInfoValue;
 use common_meta::key::table_name::TableNameKey;
-use common_meta::key::TableMetadataManagerRef;
+use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
 use common_procedure::ProcedureManagerRef;
 use futures_util::stream::BoxStream;
 use futures_util::{StreamExt, TryStreamExt};
 use moka::sync::Cache;
-use partition::manager::PartitionRuleManagerRef;
+use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
 use session::context::{Channel, QueryContext};
 use snafu::prelude::*;
-use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
 use table::dist_table::DistTable;
 use table::metadata::TableId;
 use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
@@ -52,11 +49,8 @@ use crate::error::{
     CacheNotFoundSnafu, GetTableCacheSnafu, InvalidTableInfoInCatalogSnafu, ListCatalogsSnafu,
     ListSchemasSnafu, ListTablesSnafu, Result, TableMetadataManagerSnafu,
 };
-#[cfg(feature = "enterprise")]
-use crate::information_schema::InformationSchemaTableFactoryRef;
 use crate::information_schema::{InformationExtensionRef, InformationSchemaProvider};
 use crate::kvbackend::TableCacheRef;
-use crate::process_manager::ProcessManagerRef;
 use crate::system_schema::pg_catalog::PGCatalogProvider;
 use crate::system_schema::SystemSchemaProvider;
 use crate::CatalogManager;
@@ -69,22 +63,57 @@ use crate::CatalogManager;
 #[derive(Clone)]
 pub struct KvBackendCatalogManager {
     /// Provides the extension methods for the `information_schema` tables
-    pub(super) information_extension: InformationExtensionRef,
+    information_extension: InformationExtensionRef,
     /// Manages partition rules.
-    pub(super) partition_manager: PartitionRuleManagerRef,
+    partition_manager: PartitionRuleManagerRef,
     /// Manages table metadata.
-    pub(super) table_metadata_manager: TableMetadataManagerRef,
+    table_metadata_manager: TableMetadataManagerRef,
     /// A sub-CatalogManager that handles system tables
-    pub(super) system_catalog: SystemCatalog,
+    system_catalog: SystemCatalog,
     /// Cache registry for all caches.
-    pub(super) cache_registry: LayeredCacheRegistryRef,
+    cache_registry: LayeredCacheRegistryRef,
     /// Only available in `Standalone` mode.
-    pub(super) procedure_manager: Option<ProcedureManagerRef>,
+    procedure_manager: Option<ProcedureManagerRef>,
 }
 
-pub(super) const CATALOG_CACHE_MAX_CAPACITY: u64 = 128;
+const CATALOG_CACHE_MAX_CAPACITY: u64 = 128;
 
 impl KvBackendCatalogManager {
+    pub fn new(
+        information_extension: InformationExtensionRef,
+        backend: KvBackendRef,
+        cache_registry: LayeredCacheRegistryRef,
+        procedure_manager: Option<ProcedureManagerRef>,
+    ) -> Arc<Self> {
+        Arc::new_cyclic(|me| Self {
+            information_extension,
+            partition_manager: Arc::new(PartitionRuleManager::new(
+                backend.clone(),
+                cache_registry
+                    .get()
+                    .expect("Failed to get table_route_cache"),
+            )),
+            table_metadata_manager: Arc::new(TableMetadataManager::new(backend.clone())),
+            system_catalog: SystemCatalog {
+                catalog_manager: me.clone(),
+                catalog_cache: Cache::new(CATALOG_CACHE_MAX_CAPACITY),
+                pg_catalog_cache: Cache::new(CATALOG_CACHE_MAX_CAPACITY),
+                information_schema_provider: Arc::new(InformationSchemaProvider::new(
+                    DEFAULT_CATALOG_NAME.to_string(),
+                    me.clone(),
+                    Arc::new(FlowMetadataManager::new(backend.clone())),
+                )),
+                pg_catalog_provider: Arc::new(PGCatalogProvider::new(
+                    DEFAULT_CATALOG_NAME.to_string(),
+                    me.clone(),
+                )),
+                backend,
+            },
+            cache_registry,
+            procedure_manager,
+        })
+    }
+
     pub fn view_info_cache(&self) -> Result<ViewInfoCacheRef> {
         self.cache_registry.get().context(CacheNotFoundSnafu {
             name: "view_info_cache",
@@ -107,61 +136,6 @@ impl KvBackendCatalogManager {
     pub fn procedure_manager(&self) -> Option<ProcedureManagerRef> {
         self.procedure_manager.clone()
     }
-
-    // Override logical table's partition key indices with physical table's.
-    async fn override_logical_table_partition_key_indices(
-        table_route_cache: &TableRouteCacheRef,
-        table_info_manager: &TableInfoManager,
-        table: TableRef,
-    ) -> Result<TableRef> {
-        // If the table is not a metric table, return the table directly.
-        if table.table_info().meta.engine != METRIC_ENGINE_NAME {
-            return Ok(table);
-        }
-
-        if let Some(table_route_value) = table_route_cache
-            .get(table.table_info().table_id())
-            .await
-            .context(TableMetadataManagerSnafu)?
-            && let TableRoute::Logical(logical_route) = &*table_route_value
-            && let Some(physical_table_info_value) = table_info_manager
-                .get(logical_route.physical_table_id())
-                .await
-                .context(TableMetadataManagerSnafu)?
-        {
-            let mut new_table_info = (*table.table_info()).clone();
-
-            // Remap partition key indices from physical table to logical table
-            new_table_info.meta.partition_key_indices = physical_table_info_value
-                .table_info
-                .meta
-                .partition_key_indices
-                .iter()
-                .filter_map(|&physical_index| {
-                    // Get the column name from the physical table using the physical index
-                    physical_table_info_value
-                        .table_info
-                        .meta
-                        .schema
-                        .column_schemas
-                        .get(physical_index)
-                        .and_then(|physical_column| {
-                            // Find the corresponding index in the logical table schema
-                            new_table_info
-                                .meta
-                                .schema
-                                .column_index_by_name(physical_column.name.as_str())
-                        })
-                })
-                .collect();
-
-            let new_table = DistTable::table(Arc::new(new_table_info));
-
-            return Ok(new_table);
-        }
-
-        Ok(table)
-    }
 }
 
 #[async_trait::async_trait]
@@ -288,28 +262,16 @@ impl CatalogManager for KvBackendCatalogManager {
         let table_cache: TableCacheRef = self.cache_registry.get().context(CacheNotFoundSnafu {
             name: "table_cache",
         })?;
-
-        let table = table_cache
+        if let Some(table) = table_cache
             .get_by_ref(&TableName {
                 catalog_name: catalog_name.to_string(),
                 schema_name: schema_name.to_string(),
                 table_name: table_name.to_string(),
             })
             .await
-            .context(GetTableCacheSnafu)?;
-
-        if let Some(table) = table {
-            let table_route_cache: TableRouteCacheRef =
-                self.cache_registry.get().context(CacheNotFoundSnafu {
-                    name: "table_route_cache",
-                })?;
-            return Self::override_logical_table_partition_key_indices(
-                &table_route_cache,
-                self.table_metadata_manager.table_info_manager(),
-                table,
-            )
-            .await
-            .map(Some);
+            .context(GetTableCacheSnafu)?
+        {
+            return Ok(Some(table));
         }
 
         if channel == Channel::Postgres {
@@ -322,7 +284,7 @@ impl CatalogManager for KvBackendCatalogManager {
             }
         }
 
-        Ok(None)
+        return Ok(None);
     }
 
     async fn tables_by_ids(
@@ -374,20 +336,8 @@ impl CatalogManager for KvBackendCatalogManager {
         let catalog = catalog.to_string();
         let schema = schema.to_string();
         let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
-        let table_route_cache: Result<TableRouteCacheRef> =
-            self.cache_registry.get().context(CacheNotFoundSnafu {
-                name: "table_route_cache",
-            });
 
         common_runtime::spawn_global(async move {
-            let table_route_cache = match table_route_cache {
-                Ok(table_route_cache) => table_route_cache,
-                Err(e) => {
-                    let _ = tx.send(Err(e)).await;
-                    return;
-                }
-            };
-
             let table_id_stream = metadata_manager
                 .table_name_manager()
                 .tables(&catalog, &schema)
@@ -414,7 +364,6 @@ impl CatalogManager for KvBackendCatalogManager {
                 let metadata_manager = metadata_manager.clone();
                 let tx = tx.clone();
                 let semaphore = semaphore.clone();
-                let table_route_cache = table_route_cache.clone();
                 common_runtime::spawn_global(async move {
                     // we don't explicitly close the semaphore so just ignore the potential error.
                     let _ = semaphore.acquire().await;
@@ -432,16 +381,6 @@ impl CatalogManager for KvBackendCatalogManager {
                     };
 
                     for table in table_info_values.into_values().map(build_table) {
-                        let table = if let Ok(table) = table {
-                            Self::override_logical_table_partition_key_indices(
-                                &table_route_cache,
-                                metadata_manager.table_info_manager(),
-                                table,
-                            )
-                            .await
-                        } else {
-                            table
-                        };
                         if tx.send(table).await.is_err() {
                             return;
                         }
@@ -471,19 +410,15 @@ fn build_table(table_info_value: TableInfoValue) -> Result<TableRef> {
 /// - information_schema.{tables}
 /// - pg_catalog.{tables}
 #[derive(Clone)]
-pub(super) struct SystemCatalog {
-    pub(super) catalog_manager: Weak<KvBackendCatalogManager>,
-    pub(super) catalog_cache: Cache<String, Arc<InformationSchemaProvider>>,
-    pub(super) pg_catalog_cache: Cache<String, Arc<PGCatalogProvider>>,
+struct SystemCatalog {
+    catalog_manager: Weak<KvBackendCatalogManager>,
+    catalog_cache: Cache<String, Arc<InformationSchemaProvider>>,
+    pg_catalog_cache: Cache<String, Arc<PGCatalogProvider>>,
 
     // system_schema_provider for default catalog
-    pub(super) information_schema_provider: Arc<InformationSchemaProvider>,
-    pub(super) pg_catalog_provider: Arc<PGCatalogProvider>,
-    pub(super) backend: KvBackendRef,
-    pub(super) process_manager: Option<ProcessManagerRef>,
-    #[cfg(feature = "enterprise")]
-    pub(super) extra_information_table_factories:
-        std::collections::HashMap<String, InformationSchemaTableFactoryRef>,
+    information_schema_provider: Arc<InformationSchemaProvider>,
+    pg_catalog_provider: Arc<PGCatalogProvider>,
+    backend: KvBackendRef,
 }
 
 impl SystemCatalog {
@@ -547,17 +482,11 @@ impl SystemCatalog {
         if schema == INFORMATION_SCHEMA_NAME {
             let information_schema_provider =
                 self.catalog_cache.get_with_by_ref(catalog, move || {
-                    let provider = InformationSchemaProvider::new(
+                    Arc::new(InformationSchemaProvider::new(
                         catalog.to_string(),
                         self.catalog_manager.clone(),
                         Arc::new(FlowMetadataManager::new(self.backend.clone())),
-                        self.process_manager.clone(),
-                        self.backend.clone(),
-                    );
-                    #[cfg(feature = "enterprise")]
-                    let provider = provider
-                        .with_extra_table_factories(self.extra_information_table_factories.clone());
-                    Arc::new(provider)
+                    ))
                 });
             information_schema_provider.table(table_name)
         } else if schema == PG_CATALOG_NAME && channel == Channel::Postgres {

src/catalog/src/lib.rs

@@ -14,7 +14,6 @@
 
 #![feature(assert_matches)]
 #![feature(try_blocks)]
-#![feature(let_chains)]
 
 use std::any::Any;
 use std::fmt::{Debug, Formatter};
@@ -41,7 +40,6 @@ pub mod information_schema {
     pub use crate::system_schema::information_schema::*;
 }
 
-pub mod process_manager;
 pub mod table_source;
 
 #[async_trait::async_trait]

src/catalog/src/memory/manager.rs

@@ -352,13 +352,10 @@ impl MemoryCatalogManager {
     }
 
     fn create_catalog_entry(self: &Arc<Self>, catalog: String) -> SchemaEntries {
-        let backend = Arc::new(MemoryKvBackend::new());
         let information_schema_provider = InformationSchemaProvider::new(
             catalog,
             Arc::downgrade(self) as Weak<dyn CatalogManager>,
-            Arc::new(FlowMetadataManager::new(backend.clone())),
-            None, // we don't need ProcessManager on regions server.
-            backend,
+            Arc::new(FlowMetadataManager::new(Arc::new(MemoryKvBackend::new()))),
         );
         let information_schema = information_schema_provider.tables().clone();
 

src/catalog/src/metrics.rs

@@ -34,20 +34,4 @@ lazy_static! {
         register_histogram!("greptime_catalog_kv_get", "catalog kv get").unwrap();
     pub static ref METRIC_CATALOG_KV_BATCH_GET: Histogram =
         register_histogram!("greptime_catalog_kv_batch_get", "catalog kv batch get").unwrap();
-
-    /// Count of running process in each catalog.
-    pub static ref PROCESS_LIST_COUNT: IntGaugeVec = register_int_gauge_vec!(
-        "greptime_process_list_count",
-        "Running process count per catalog",
-        &["catalog"]
-    )
-    .unwrap();
-
-    /// Count of killed process in each catalog.
-    pub static ref PROCESS_KILL_COUNT: IntCounterVec = register_int_counter_vec!(
-        "greptime_process_kill_count",
-        "Completed kill process requests count",
-        &["catalog"]
-    )
-    .unwrap();
 }

src/catalog/src/process_manager.rs

@@ -1,500 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::hash_map::Entry;
-use std::collections::HashMap;
-use std::fmt::{Debug, Formatter};
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::{Arc, RwLock};
-
-use api::v1::frontend::{KillProcessRequest, ListProcessRequest, ProcessInfo};
-use common_base::cancellation::CancellationHandle;
-use common_frontend::selector::{FrontendSelector, MetaClientSelector};
-use common_telemetry::{debug, info, warn};
-use common_time::util::current_time_millis;
-use meta_client::MetaClientRef;
-use snafu::{ensure, OptionExt, ResultExt};
-
-use crate::error;
-use crate::metrics::{PROCESS_KILL_COUNT, PROCESS_LIST_COUNT};
-
-pub type ProcessId = u32;
-pub type ProcessManagerRef = Arc<ProcessManager>;
-
-/// Query process manager.
-pub struct ProcessManager {
-    /// Local frontend server address,
-    server_addr: String,
-    /// Next process id for local queries.
-    next_id: AtomicU32,
-    /// Running process per catalog.
-    catalogs: RwLock<HashMap<String, HashMap<ProcessId, CancellableProcess>>>,
-    /// Frontend selector to locate frontend nodes.
-    frontend_selector: Option<MetaClientSelector>,
-}
-
-impl ProcessManager {
-    /// Create a [ProcessManager] instance with server address and kv client.
-    pub fn new(server_addr: String, meta_client: Option<MetaClientRef>) -> Self {
-        let frontend_selector = meta_client.map(MetaClientSelector::new);
-        Self {
-            server_addr,
-            next_id: Default::default(),
-            catalogs: Default::default(),
-            frontend_selector,
-        }
-    }
-}
-
-impl ProcessManager {
-    /// Registers a submitted query. Use the provided id if present.
-    #[must_use]
-    pub fn register_query(
-        self: &Arc<Self>,
-        catalog: String,
-        schemas: Vec<String>,
-        query: String,
-        client: String,
-        query_id: Option<ProcessId>,
-    ) -> Ticket {
-        let id = query_id.unwrap_or_else(|| self.next_id.fetch_add(1, Ordering::Relaxed));
-        let process = ProcessInfo {
-            id,
-            catalog: catalog.clone(),
-            schemas,
-            query,
-            start_timestamp: current_time_millis(),
-            client,
-            frontend: self.server_addr.clone(),
-        };
-        let cancellation_handle = Arc::new(CancellationHandle::default());
-        let cancellable_process = CancellableProcess::new(cancellation_handle.clone(), process);
-
-        self.catalogs
-            .write()
-            .unwrap()
-            .entry(catalog.clone())
-            .or_default()
-            .insert(id, cancellable_process);
-
-        Ticket {
-            catalog,
-            manager: self.clone(),
-            id,
-            cancellation_handle,
-        }
-    }
-
-    /// Generates the next process id.
-    pub fn next_id(&self) -> u32 {
-        self.next_id.fetch_add(1, Ordering::Relaxed)
-    }
-
-    /// De-register a query from process list.
-    pub fn deregister_query(&self, catalog: String, id: ProcessId) {
-        if let Entry::Occupied(mut o) = self.catalogs.write().unwrap().entry(catalog) {
-            let process = o.get_mut().remove(&id);
-            debug!("Deregister process: {:?}", process);
-            if o.get().is_empty() {
-                o.remove();
-            }
-        }
-    }
-
-    /// List local running processes in given catalog.
-    pub fn local_processes(&self, catalog: Option<&str>) -> error::Result<Vec<ProcessInfo>> {
-        let catalogs = self.catalogs.read().unwrap();
-        let result = if let Some(catalog) = catalog {
-            if let Some(catalogs) = catalogs.get(catalog) {
-                catalogs.values().map(|p| p.process.clone()).collect()
-            } else {
-                vec![]
-            }
-        } else {
-            catalogs
-                .values()
-                .flat_map(|v| v.values().map(|p| p.process.clone()))
-                .collect()
-        };
-        Ok(result)
-    }
-
-    pub async fn list_all_processes(
-        &self,
-        catalog: Option<&str>,
-    ) -> error::Result<Vec<ProcessInfo>> {
-        let mut processes = vec![];
-        if let Some(remote_frontend_selector) = self.frontend_selector.as_ref() {
-            let frontends = remote_frontend_selector
-                .select(|node| node.peer.addr != self.server_addr)
-                .await
-                .context(error::InvokeFrontendSnafu)?;
-            for mut f in frontends {
-                let result = f
-                    .list_process(ListProcessRequest {
-                        catalog: catalog.unwrap_or_default().to_string(),
-                    })
-                    .await
-                    .context(error::InvokeFrontendSnafu);
-                match result {
-                    Ok(resp) => {
-                        processes.extend(resp.processes);
-                    }
-                    Err(e) => {
-                        warn!(e; "Skipping failing node: {:?}", f)
-                    }
-                }
-            }
-        }
-        processes.extend(self.local_processes(catalog)?);
-        Ok(processes)
-    }
-
-    /// Kills query with provided catalog and id.
-    pub async fn kill_process(
-        &self,
-        server_addr: String,
-        catalog: String,
-        id: ProcessId,
-    ) -> error::Result<bool> {
-        if server_addr == self.server_addr {
-            self.kill_local_process(catalog, id).await
-        } else {
-            let mut nodes = self
-                .frontend_selector
-                .as_ref()
-                .context(error::MetaClientMissingSnafu)?
-                .select(|node| node.peer.addr == server_addr)
-                .await
-                .context(error::InvokeFrontendSnafu)?;
-            ensure!(
-                !nodes.is_empty(),
-                error::FrontendNotFoundSnafu { addr: server_addr }
-            );
-
-            let request = KillProcessRequest {
-                server_addr,
-                catalog,
-                process_id: id,
-            };
-            nodes[0]
-                .kill_process(request)
-                .await
-                .context(error::InvokeFrontendSnafu)?;
-            Ok(true)
-        }
-    }
-
-    /// Kills local query with provided catalog and id.
-    pub async fn kill_local_process(&self, catalog: String, id: ProcessId) -> error::Result<bool> {
-        if let Some(catalogs) = self.catalogs.write().unwrap().get_mut(&catalog) {
-            if let Some(process) = catalogs.remove(&id) {
-                process.handle.cancel();
-                info!(
-                    "Killed process, catalog: {}, id: {:?}",
-                    process.process.catalog, process.process.id
-                );
-                PROCESS_KILL_COUNT.with_label_values(&[&catalog]).inc();
-                Ok(true)
-            } else {
-                debug!("Failed to kill process, id not found: {}", id);
-                Ok(false)
-            }
-        } else {
-            debug!("Failed to kill process, catalog not found: {}", catalog);
-            Ok(false)
-        }
-    }
-}
-
-pub struct Ticket {
-    pub(crate) catalog: String,
-    pub(crate) manager: ProcessManagerRef,
-    pub(crate) id: ProcessId,
-    pub cancellation_handle: Arc<CancellationHandle>,
-}
-
-impl Drop for Ticket {
-    fn drop(&mut self) {
-        self.manager
-            .deregister_query(std::mem::take(&mut self.catalog), self.id);
-    }
-}
-
-struct CancellableProcess {
-    handle: Arc<CancellationHandle>,
-    process: ProcessInfo,
-}
-
-impl Drop for CancellableProcess {
-    fn drop(&mut self) {
-        PROCESS_LIST_COUNT
-            .with_label_values(&[&self.process.catalog])
-            .dec();
-    }
-}
-
-impl CancellableProcess {
-    fn new(handle: Arc<CancellationHandle>, process: ProcessInfo) -> Self {
-        PROCESS_LIST_COUNT
-            .with_label_values(&[&process.catalog])
-            .inc();
-        Self { handle, process }
-    }
-}
-
-impl Debug for CancellableProcess {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("CancellableProcess")
-            .field("cancelled", &self.handle.is_cancelled())
-            .field("process", &self.process)
-            .finish()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use crate::process_manager::ProcessManager;
-
-    #[tokio::test]
-    async fn test_register_query() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-        let ticket = process_manager.clone().register_query(
-            "public".to_string(),
-            vec!["test".to_string()],
-            "SELECT * FROM table".to_string(),
-            "".to_string(),
-            None,
-        );
-
-        let running_processes = process_manager.local_processes(None).unwrap();
-        assert_eq!(running_processes.len(), 1);
-        assert_eq!(&running_processes[0].frontend, "127.0.0.1:8000");
-        assert_eq!(running_processes[0].id, ticket.id);
-        assert_eq!(&running_processes[0].query, "SELECT * FROM table");
-
-        drop(ticket);
-        assert_eq!(process_manager.local_processes(None).unwrap().len(), 0);
-    }
-
-    #[tokio::test]
-    async fn test_register_query_with_custom_id() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-        let custom_id = 12345;
-
-        let ticket = process_manager.clone().register_query(
-            "public".to_string(),
-            vec!["test".to_string()],
-            "SELECT * FROM table".to_string(),
-            "client1".to_string(),
-            Some(custom_id),
-        );
-
-        assert_eq!(ticket.id, custom_id);
-
-        let running_processes = process_manager.local_processes(None).unwrap();
-        assert_eq!(running_processes.len(), 1);
-        assert_eq!(running_processes[0].id, custom_id);
-        assert_eq!(&running_processes[0].client, "client1");
-    }
-
-    #[tokio::test]
-    async fn test_multiple_queries_same_catalog() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-
-        let ticket1 = process_manager.clone().register_query(
-            "public".to_string(),
-            vec!["schema1".to_string()],
-            "SELECT * FROM table1".to_string(),
-            "client1".to_string(),
-            None,
-        );
-
-        let ticket2 = process_manager.clone().register_query(
-            "public".to_string(),
-            vec!["schema2".to_string()],
-            "SELECT * FROM table2".to_string(),
-            "client2".to_string(),
-            None,
-        );
-
-        let running_processes = process_manager.local_processes(Some("public")).unwrap();
-        assert_eq!(running_processes.len(), 2);
-
-        // Verify both processes are present
-        let ids: Vec<u32> = running_processes.iter().map(|p| p.id).collect();
-        assert!(ids.contains(&ticket1.id));
-        assert!(ids.contains(&ticket2.id));
-    }
-
-    #[tokio::test]
-    async fn test_multiple_catalogs() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-
-        let _ticket1 = process_manager.clone().register_query(
-            "catalog1".to_string(),
-            vec!["schema1".to_string()],
-            "SELECT * FROM table1".to_string(),
-            "client1".to_string(),
-            None,
-        );
-
-        let _ticket2 = process_manager.clone().register_query(
-            "catalog2".to_string(),
-            vec!["schema2".to_string()],
-            "SELECT * FROM table2".to_string(),
-            "client2".to_string(),
-            None,
-        );
-
-        // Test listing processes for specific catalog
-        let catalog1_processes = process_manager.local_processes(Some("catalog1")).unwrap();
-        assert_eq!(catalog1_processes.len(), 1);
-        assert_eq!(&catalog1_processes[0].catalog, "catalog1");
-
-        let catalog2_processes = process_manager.local_processes(Some("catalog2")).unwrap();
-        assert_eq!(catalog2_processes.len(), 1);
-        assert_eq!(&catalog2_processes[0].catalog, "catalog2");
-
-        // Test listing all processes
-        let all_processes = process_manager.local_processes(None).unwrap();
-        assert_eq!(all_processes.len(), 2);
-    }
-
-    #[tokio::test]
-    async fn test_deregister_query() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-
-        let ticket = process_manager.clone().register_query(
-            "public".to_string(),
-            vec!["test".to_string()],
-            "SELECT * FROM table".to_string(),
-            "client1".to_string(),
-            None,
-        );
-        assert_eq!(process_manager.local_processes(None).unwrap().len(), 1);
-        process_manager.deregister_query("public".to_string(), ticket.id);
-        assert_eq!(process_manager.local_processes(None).unwrap().len(), 0);
-    }
-
-    #[tokio::test]
-    async fn test_cancellation_handle() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-
-        let ticket = process_manager.clone().register_query(
-            "public".to_string(),
-            vec!["test".to_string()],
-            "SELECT * FROM table".to_string(),
-            "client1".to_string(),
-            None,
-        );
-
-        assert!(!ticket.cancellation_handle.is_cancelled());
-        ticket.cancellation_handle.cancel();
-        assert!(ticket.cancellation_handle.is_cancelled());
-    }
-
-    #[tokio::test]
-    async fn test_kill_local_process() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-
-        let ticket = process_manager.clone().register_query(
-            "public".to_string(),
-            vec!["test".to_string()],
-            "SELECT * FROM table".to_string(),
-            "client1".to_string(),
-            None,
-        );
-        assert!(!ticket.cancellation_handle.is_cancelled());
-        let killed = process_manager
-            .kill_process(
-                "127.0.0.1:8000".to_string(),
-                "public".to_string(),
-                ticket.id,
-            )
-            .await
-            .unwrap();
-
-        assert!(killed);
-        assert_eq!(process_manager.local_processes(None).unwrap().len(), 0);
-    }
-
-    #[tokio::test]
-    async fn test_kill_nonexistent_process() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-        let killed = process_manager
-            .kill_process("127.0.0.1:8000".to_string(), "public".to_string(), 999)
-            .await
-            .unwrap();
-        assert!(!killed);
-    }
-
-    #[tokio::test]
-    async fn test_kill_process_nonexistent_catalog() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-        let killed = process_manager
-            .kill_process("127.0.0.1:8000".to_string(), "nonexistent".to_string(), 1)
-            .await
-            .unwrap();
-        assert!(!killed);
-    }
-
-    #[tokio::test]
-    async fn test_process_info_fields() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-
-        let _ticket = process_manager.clone().register_query(
-            "test_catalog".to_string(),
-            vec!["schema1".to_string(), "schema2".to_string()],
-            "SELECT COUNT(*) FROM users WHERE age > 18".to_string(),
-            "test_client".to_string(),
-            Some(42),
-        );
-
-        let processes = process_manager.local_processes(None).unwrap();
-        assert_eq!(processes.len(), 1);
-
-        let process = &processes[0];
-        assert_eq!(process.id, 42);
-        assert_eq!(&process.catalog, "test_catalog");
-        assert_eq!(process.schemas, vec!["schema1", "schema2"]);
-        assert_eq!(&process.query, "SELECT COUNT(*) FROM users WHERE age > 18");
-        assert_eq!(&process.client, "test_client");
-        assert_eq!(&process.frontend, "127.0.0.1:8000");
-        assert!(process.start_timestamp > 0);
-    }
-
-    #[tokio::test]
-    async fn test_ticket_drop_deregisters_process() {
-        let process_manager = Arc::new(ProcessManager::new("127.0.0.1:8000".to_string(), None));
-
-        {
-            let _ticket = process_manager.clone().register_query(
-                "public".to_string(),
-                vec!["test".to_string()],
-                "SELECT * FROM table".to_string(),
-                "client1".to_string(),
-                None,
-            );
-
-            // Process should be registered
-            assert_eq!(process_manager.local_processes(None).unwrap().len(), 1);
-        } // ticket goes out of scope here
-
-        // Process should be automatically deregistered
-        assert_eq!(process_manager.local_processes(None).unwrap().len(), 0);
-    }
-}

src/catalog/src/system_schema.rs

@@ -15,7 +15,7 @@
 pub mod information_schema;
 mod memory_table;
 pub mod pg_catalog;
-pub mod predicate;
+mod predicate;
 mod utils;
 
 use std::collections::HashMap;
@@ -96,7 +96,7 @@ trait SystemSchemaProviderInner {
     }
 }
 
-pub trait SystemTable {
+pub(crate) trait SystemTable {
     fn table_id(&self) -> TableId;
 
     fn table_name(&self) -> &'static str;
@@ -110,7 +110,7 @@ pub trait SystemTable {
     }
 }
 
-pub type SystemTableRef = Arc<dyn SystemTable + Send + Sync>;
+pub(crate) type SystemTableRef = Arc<dyn SystemTable + Send + Sync>;
 
 struct SystemTableDataSource {
     table: SystemTableRef,

src/catalog/src/system_schema/information_schema.rs

@@ -19,7 +19,6 @@ mod information_memory_table;
 pub mod key_column_usage;
 mod partitions;
 mod procedure_info;
-pub mod process_list;
 pub mod region_peers;
 mod region_statistics;
 mod runtime_metrics;
@@ -38,13 +37,11 @@ use common_meta::cluster::NodeInfo;
 use common_meta::datanode::RegionStat;
 use common_meta::key::flow::flow_state::FlowStat;
 use common_meta::key::flow::FlowMetadataManager;
-use common_meta::kv_backend::KvBackendRef;
 use common_procedure::ProcedureInfo;
 use common_recordbatch::SendableRecordBatchStream;
 use datatypes::schema::SchemaRef;
 use lazy_static::lazy_static;
 use paste::paste;
-use process_list::InformationSchemaProcessList;
 use store_api::storage::{ScanRequest, TableId};
 use table::metadata::TableType;
 use table::TableRef;
@@ -53,7 +50,6 @@ use views::InformationSchemaViews;
 
 use self::columns::InformationSchemaColumns;
 use crate::error::{Error, Result};
-use crate::process_manager::ProcessManagerRef;
 use crate::system_schema::information_schema::cluster_info::InformationSchemaClusterInfo;
 use crate::system_schema::information_schema::flows::InformationSchemaFlows;
 use crate::system_schema::information_schema::information_memory_table::get_schema_columns;
@@ -113,36 +109,12 @@ macro_rules! setup_memory_table {
     };
 }
 
-#[cfg(feature = "enterprise")]
-pub struct MakeInformationTableRequest {
-    pub catalog_name: String,
-    pub catalog_manager: Weak<dyn CatalogManager>,
-    pub kv_backend: KvBackendRef,
-}
-
-/// A factory trait for making information schema tables.
-///
-/// This trait allows for extensibility of the information schema by providing
-/// a way to dynamically create custom information schema tables.
-#[cfg(feature = "enterprise")]
-pub trait InformationSchemaTableFactory {
-    fn make_information_table(&self, req: MakeInformationTableRequest) -> SystemTableRef;
-}
-
-#[cfg(feature = "enterprise")]
-pub type InformationSchemaTableFactoryRef = Arc<dyn InformationSchemaTableFactory + Send + Sync>;
-
 /// The `information_schema` tables info provider.
 pub struct InformationSchemaProvider {
     catalog_name: String,
     catalog_manager: Weak<dyn CatalogManager>,
-    process_manager: Option<ProcessManagerRef>,
     flow_metadata_manager: Arc<FlowMetadataManager>,
     tables: HashMap<String, TableRef>,
-    #[allow(dead_code)]
-    kv_backend: KvBackendRef,
-    #[cfg(feature = "enterprise")]
-    extra_table_factories: HashMap<String, InformationSchemaTableFactoryRef>,
 }
 
 impl SystemSchemaProvider for InformationSchemaProvider {
@@ -152,7 +124,6 @@ impl SystemSchemaProvider for InformationSchemaProvider {
         &self.tables
     }
 }
-
 impl SystemSchemaProviderInner for InformationSchemaProvider {
     fn catalog_name(&self) -> &str {
         &self.catalog_name
@@ -162,16 +133,6 @@ impl SystemSchemaProviderInner for InformationSchemaProvider {
     }
 
     fn system_table(&self, name: &str) -> Option<SystemTableRef> {
-        #[cfg(feature = "enterprise")]
-        if let Some(factory) = self.extra_table_factories.get(name) {
-            let req = MakeInformationTableRequest {
-                catalog_name: self.catalog_name.clone(),
-                catalog_manager: self.catalog_manager.clone(),
-                kv_backend: self.kv_backend.clone(),
-            };
-            return Some(factory.make_information_table(req));
-        }
-
         match name.to_ascii_lowercase().as_str() {
             TABLES => Some(Arc::new(InformationSchemaTables::new(
                 self.catalog_name.clone(),
@@ -246,10 +207,6 @@ impl SystemSchemaProviderInner for InformationSchemaProvider {
                     self.catalog_manager.clone(),
                 ),
             ) as _),
-            PROCESS_LIST => self
-                .process_manager
-                .as_ref()
-                .map(|p| Arc::new(InformationSchemaProcessList::new(p.clone())) as _),
             _ => None,
         }
     }
@@ -260,18 +217,12 @@ impl InformationSchemaProvider {
         catalog_name: String,
         catalog_manager: Weak<dyn CatalogManager>,
         flow_metadata_manager: Arc<FlowMetadataManager>,
-        process_manager: Option<ProcessManagerRef>,
-        kv_backend: KvBackendRef,
     ) -> Self {
         let mut provider = Self {
             catalog_name,
             catalog_manager,
             flow_metadata_manager,
-            process_manager,
             tables: HashMap::new(),
-            kv_backend,
-            #[cfg(feature = "enterprise")]
-            extra_table_factories: HashMap::new(),
         };
 
         provider.build_tables();
@@ -279,16 +230,6 @@ impl InformationSchemaProvider {
         provider
     }
 
-    #[cfg(feature = "enterprise")]
-    pub(crate) fn with_extra_table_factories(
-        mut self,
-        factories: HashMap<String, InformationSchemaTableFactoryRef>,
-    ) -> Self {
-        self.extra_table_factories = factories;
-        self.build_tables();
-        self
-    }
-
     fn build_tables(&mut self) {
         let mut tables = HashMap::new();
 
@@ -336,22 +277,16 @@ impl InformationSchemaProvider {
             self.build_table(TABLE_CONSTRAINTS).unwrap(),
         );
         tables.insert(FLOWS.to_string(), self.build_table(FLOWS).unwrap());
-        if let Some(process_list) = self.build_table(PROCESS_LIST) {
-            tables.insert(PROCESS_LIST.to_string(), process_list);
-        }
-        #[cfg(feature = "enterprise")]
-        for name in self.extra_table_factories.keys() {
-            tables.insert(name.to_string(), self.build_table(name).expect(name));
-        }
         // Add memory tables
         for name in MEMORY_TABLES.iter() {
             tables.insert((*name).to_string(), self.build_table(name).expect(name));
         }
+
         self.tables = tables;
     }
 }
 
-pub trait InformationTable {
+trait InformationTable {
     fn table_id(&self) -> TableId;
 
     fn table_name(&self) -> &'static str;

src/catalog/src/system_schema/information_schema/information_memory_table.rs

@@ -15,8 +15,7 @@
 use std::sync::Arc;
 
 use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE};
-use datatypes::data_type::ConcreteDataType;
-use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::schema::{Schema, SchemaRef};
 use datatypes::vectors::{Int64Vector, StringVector, VectorRef};
 
 use crate::system_schema::information_schema::table_names::*;
@@ -368,18 +367,28 @@ pub(super) fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec<VectorRef>
 
         TRIGGERS => (
             vec![
+                string_column("TRIGGER_CATALOG"),
+                string_column("TRIGGER_SCHEMA"),
                 string_column("TRIGGER_NAME"),
-                ColumnSchema::new(
-                    "trigger_id",
-                    ConcreteDataType::uint64_datatype(),
-                    false,
-                ),
-                string_column("TRIGGER_DEFINITION"),
-                ColumnSchema::new(
-                    "flownode_id",
-                    ConcreteDataType::uint64_datatype(),
-                    true,
-                ),
+                string_column("EVENT_MANIPULATION"),
+                string_column("EVENT_OBJECT_CATALOG"),
+                string_column("EVENT_OBJECT_SCHEMA"),
+                string_column("EVENT_OBJECT_TABLE"),
+                bigint_column("ACTION_ORDER"),
+                string_column("ACTION_CONDITION"),
+                string_column("ACTION_STATEMENT"),
+                string_column("ACTION_ORIENTATION"),
+                string_column("ACTION_TIMING"),
+                string_column("ACTION_REFERENCE_OLD_TABLE"),
+                string_column("ACTION_REFERENCE_NEW_TABLE"),
+                string_column("ACTION_REFERENCE_OLD_ROW"),
+                string_column("ACTION_REFERENCE_NEW_ROW"),
+                timestamp_micro_column("CREATED"),
+                string_column("SQL_MODE"),
+                string_column("DEFINER"),
+                string_column("CHARACTER_SET_CLIENT"),
+                string_column("COLLATION_CONNECTION"),
+                string_column("DATABASE_COLLATION"),
             ],
             vec![],
         ),

src/catalog/src/system_schema/information_schema/partitions.rs

@@ -329,8 +329,13 @@ impl InformationSchemaPartitionsBuilder {
             self.partition_names.push(Some(&partition_name));
             self.partition_ordinal_positions
                 .push(Some((index + 1) as i64));
-            let expression = partition.partition_expr.as_ref().map(|e| e.to_string());
-            self.partition_expressions.push(expression.as_deref());
+            let expressions = if partition.partition.partition_columns().is_empty() {
+                None
+            } else {
+                Some(partition.partition.to_string())
+            };
+
+            self.partition_expressions.push(expressions.as_deref());
             self.create_times.push(Some(TimestampMicrosecond::from(
                 table_info.meta.created_on.timestamp_millis(),
             )));

src/catalog/src/system_schema/information_schema/process_list.rs

@@ -1,189 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use common_catalog::consts::INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID;
-use common_error::ext::BoxedError;
-use common_frontend::DisplayProcessId;
-use common_recordbatch::adapter::RecordBatchStreamAdapter;
-use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use common_time::util::current_time_millis;
-use common_time::{Duration, Timestamp};
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
-use datatypes::prelude::ConcreteDataType as CDT;
-use datatypes::scalars::ScalarVectorBuilder;
-use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
-use datatypes::value::Value;
-use datatypes::vectors::{
-    DurationMillisecondVectorBuilder, StringVectorBuilder, TimestampMillisecondVectorBuilder,
-    VectorRef,
-};
-use snafu::ResultExt;
-use store_api::storage::{ScanRequest, TableId};
-
-use crate::error::{self, InternalSnafu};
-use crate::information_schema::Predicates;
-use crate::process_manager::ProcessManagerRef;
-use crate::system_schema::information_schema::InformationTable;
-
-/// Column names of `information_schema.process_list`
-pub const ID: &str = "id";
-pub const CATALOG: &str = "catalog";
-pub const SCHEMAS: &str = "schemas";
-pub const QUERY: &str = "query";
-pub const CLIENT: &str = "client";
-pub const FRONTEND: &str = "frontend";
-pub const START_TIMESTAMP: &str = "start_timestamp";
-pub const ELAPSED_TIME: &str = "elapsed_time";
-
-/// `information_schema.process_list` table implementation that tracks running
-/// queries in current cluster.
-pub struct InformationSchemaProcessList {
-    schema: SchemaRef,
-    process_manager: ProcessManagerRef,
-}
-
-impl InformationSchemaProcessList {
-    pub fn new(process_manager: ProcessManagerRef) -> Self {
-        Self {
-            schema: Self::schema(),
-            process_manager,
-        }
-    }
-
-    fn schema() -> SchemaRef {
-        Arc::new(Schema::new(vec![
-            ColumnSchema::new(ID, CDT::string_datatype(), false),
-            ColumnSchema::new(CATALOG, CDT::string_datatype(), false),
-            ColumnSchema::new(SCHEMAS, CDT::string_datatype(), false),
-            ColumnSchema::new(QUERY, CDT::string_datatype(), false),
-            ColumnSchema::new(CLIENT, CDT::string_datatype(), false),
-            ColumnSchema::new(FRONTEND, CDT::string_datatype(), false),
-            ColumnSchema::new(
-                START_TIMESTAMP,
-                CDT::timestamp_millisecond_datatype(),
-                false,
-            ),
-            ColumnSchema::new(ELAPSED_TIME, CDT::duration_millisecond_datatype(), false),
-        ]))
-    }
-}
-
-impl InformationTable for InformationSchemaProcessList {
-    fn table_id(&self) -> TableId {
-        INFORMATION_SCHEMA_PROCESS_LIST_TABLE_ID
-    }
-
-    fn table_name(&self) -> &'static str {
-        "process_list"
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn to_stream(&self, request: ScanRequest) -> error::Result<SendableRecordBatchStream> {
-        let process_manager = self.process_manager.clone();
-        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
-            self.schema.arrow_schema().clone(),
-            futures::stream::once(async move {
-                make_process_list(process_manager, request)
-                    .await
-                    .map(RecordBatch::into_df_record_batch)
-                    .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e)))
-            }),
-        ));
-
-        Ok(Box::pin(
-            RecordBatchStreamAdapter::try_new(stream)
-                .map_err(BoxedError::new)
-                .context(InternalSnafu)?,
-        ))
-    }
-}
-
-/// Build running process list.
-async fn make_process_list(
-    process_manager: ProcessManagerRef,
-    request: ScanRequest,
-) -> error::Result<RecordBatch> {
-    let predicates = Predicates::from_scan_request(&Some(request));
-    let current_time = current_time_millis();
-    // todo(hl): find a way to extract user catalog to filter queries from other users.
-    let queries = process_manager.list_all_processes(None).await?;
-
-    let mut id_builder = StringVectorBuilder::with_capacity(queries.len());
-    let mut catalog_builder = StringVectorBuilder::with_capacity(queries.len());
-    let mut schemas_builder = StringVectorBuilder::with_capacity(queries.len());
-    let mut query_builder = StringVectorBuilder::with_capacity(queries.len());
-    let mut client_builder = StringVectorBuilder::with_capacity(queries.len());
-    let mut frontend_builder = StringVectorBuilder::with_capacity(queries.len());
-    let mut start_time_builder = TimestampMillisecondVectorBuilder::with_capacity(queries.len());
-    let mut elapsed_time_builder = DurationMillisecondVectorBuilder::with_capacity(queries.len());
-
-    for process in queries {
-        let display_id = DisplayProcessId {
-            server_addr: process.frontend.to_string(),
-            id: process.id,
-        }
-        .to_string();
-        let schemas = process.schemas.join(",");
-        let id = Value::from(display_id);
-        let catalog = Value::from(process.catalog);
-        let schemas = Value::from(schemas);
-        let query = Value::from(process.query);
-        let client = Value::from(process.client);
-        let frontend = Value::from(process.frontend);
-        let start_timestamp = Value::from(Timestamp::new_millisecond(process.start_timestamp));
-        let elapsed_time = Value::from(Duration::new_millisecond(
-            current_time - process.start_timestamp,
-        ));
-        let row = [
-            (ID, &id),
-            (CATALOG, &catalog),
-            (SCHEMAS, &schemas),
-            (QUERY, &query),
-            (CLIENT, &client),
-            (FRONTEND, &frontend),
-            (START_TIMESTAMP, &start_timestamp),
-            (ELAPSED_TIME, &elapsed_time),
-        ];
-        if predicates.eval(&row) {
-            id_builder.push(id.as_string().as_deref());
-            catalog_builder.push(catalog.as_string().as_deref());
-            schemas_builder.push(schemas.as_string().as_deref());
-            query_builder.push(query.as_string().as_deref());
-            client_builder.push(client.as_string().as_deref());
-            frontend_builder.push(frontend.as_string().as_deref());
-            start_time_builder.push(start_timestamp.as_timestamp().map(|t| t.value().into()));
-            elapsed_time_builder.push(elapsed_time.as_duration().map(|d| d.value().into()));
-        }
-    }
-
-    RecordBatch::new(
-        InformationSchemaProcessList::schema(),
-        vec![
-            Arc::new(id_builder.finish()) as VectorRef,
-            Arc::new(catalog_builder.finish()) as VectorRef,
-            Arc::new(schemas_builder.finish()) as VectorRef,
-            Arc::new(query_builder.finish()) as VectorRef,
-            Arc::new(client_builder.finish()) as VectorRef,
-            Arc::new(frontend_builder.finish()) as VectorRef,
-            Arc::new(start_time_builder.finish()) as VectorRef,
-            Arc::new(elapsed_time_builder.finish()) as VectorRef,
-        ],
-    )
-    .context(error::CreateRecordBatchSnafu)
-}

src/catalog/src/system_schema/information_schema/region_statistics.rs

@@ -44,7 +44,6 @@ const DISK_SIZE: &str = "disk_size";
 const MEMTABLE_SIZE: &str = "memtable_size";
 const MANIFEST_SIZE: &str = "manifest_size";
 const SST_SIZE: &str = "sst_size";
-const SST_NUM: &str = "sst_num";
 const INDEX_SIZE: &str = "index_size";
 const ENGINE: &str = "engine";
 const REGION_ROLE: &str = "region_role";
@@ -88,7 +87,6 @@ impl InformationSchemaRegionStatistics {
             ColumnSchema::new(MEMTABLE_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(MANIFEST_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(SST_SIZE, ConcreteDataType::uint64_datatype(), true),
-            ColumnSchema::new(SST_NUM, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(INDEX_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(ENGINE, ConcreteDataType::string_datatype(), true),
             ColumnSchema::new(REGION_ROLE, ConcreteDataType::string_datatype(), true),
@@ -151,7 +149,6 @@ struct InformationSchemaRegionStatisticsBuilder {
     memtable_sizes: UInt64VectorBuilder,
     manifest_sizes: UInt64VectorBuilder,
     sst_sizes: UInt64VectorBuilder,
-    sst_nums: UInt64VectorBuilder,
     index_sizes: UInt64VectorBuilder,
     engines: StringVectorBuilder,
     region_roles: StringVectorBuilder,
@@ -170,7 +167,6 @@ impl InformationSchemaRegionStatisticsBuilder {
             memtable_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             manifest_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             sst_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
-            sst_nums: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             index_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             engines: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             region_roles: StringVectorBuilder::with_capacity(INIT_CAPACITY),
@@ -201,7 +197,6 @@ impl InformationSchemaRegionStatisticsBuilder {
             (MEMTABLE_SIZE, &Value::from(region_stat.memtable_size)),
             (MANIFEST_SIZE, &Value::from(region_stat.manifest_size)),
             (SST_SIZE, &Value::from(region_stat.sst_size)),
-            (SST_NUM, &Value::from(region_stat.sst_num)),
             (INDEX_SIZE, &Value::from(region_stat.index_size)),
             (ENGINE, &Value::from(region_stat.engine.as_str())),
             (REGION_ROLE, &Value::from(region_stat.role.to_string())),
@@ -220,7 +215,6 @@ impl InformationSchemaRegionStatisticsBuilder {
         self.memtable_sizes.push(Some(region_stat.memtable_size));
         self.manifest_sizes.push(Some(region_stat.manifest_size));
         self.sst_sizes.push(Some(region_stat.sst_size));
-        self.sst_nums.push(Some(region_stat.sst_num));
         self.index_sizes.push(Some(region_stat.index_size));
         self.engines.push(Some(&region_stat.engine));
         self.region_roles.push(Some(&region_stat.role.to_string()));
@@ -236,7 +230,6 @@ impl InformationSchemaRegionStatisticsBuilder {
             Arc::new(self.memtable_sizes.finish()),
             Arc::new(self.manifest_sizes.finish()),
             Arc::new(self.sst_sizes.finish()),
-            Arc::new(self.sst_nums.finish()),
             Arc::new(self.index_sizes.finish()),
             Arc::new(self.engines.finish()),
             Arc::new(self.region_roles.finish()),

src/catalog/src/system_schema/information_schema/table_names.rs

@@ -47,4 +47,3 @@ pub const VIEWS: &str = "views";
 pub const FLOWS: &str = "flows";
 pub const PROCEDURE_INFO: &str = "procedure_info";
 pub const REGION_STATISTICS: &str = "region_statistics";
-pub const PROCESS_LIST: &str = "process_list";

src/catalog/src/system_schema/pg_catalog/pg_class.rs

@@ -169,7 +169,7 @@ impl DfPartitionStream for PGClass {
 }
 
 /// Builds the `pg_catalog.pg_class` table row by row
-/// TODO(J0HN50N133): `relowner` is always the [`DUMMY_OWNER_ID`] because we don't have users.
+/// TODO(J0HN50N133): `relowner` is always the [`DUMMY_OWNER_ID`] cuz we don't have user.
 /// Once we have user system, make it the actual owner of the table.
 struct PGClassBuilder {
     schema: SchemaRef,

src/catalog/src/table_source.rs

@@ -207,7 +207,6 @@ mod tests {
     use session::context::QueryContext;
 
     use super::*;
-    use crate::kvbackend::KvBackendCatalogManagerBuilder;
     use crate::memory::MemoryCatalogManager;
 
     #[test]
@@ -324,13 +323,12 @@ mod tests {
             .build(),
         );
 
-        let catalog_manager = KvBackendCatalogManagerBuilder::new(
+        let catalog_manager = KvBackendCatalogManager::new(
             Arc::new(NoopInformationExtension),
             backend.clone(),
             layered_cache_registry,
-        )
-        .build();
-
+            None,
+        );
         let table_metadata_manager = TableMetadataManager::new(backend);
         let mut view_info = common_meta::key::test_utils::new_test_table_info(1024, vec![]);
         view_info.table_type = TableType::View;

src/cli/Cargo.toml

@@ -5,18 +5,13 @@ edition.workspace = true
 license.workspace = true
 
 [features]
-default = [
-    "pg_kvbackend",
-    "mysql_kvbackend",
-]
-pg_kvbackend = ["common-meta/pg_kvbackend", "meta-srv/pg_kvbackend"]
-mysql_kvbackend = ["common-meta/mysql_kvbackend", "meta-srv/mysql_kvbackend"]
+pg_kvbackend = ["common-meta/pg_kvbackend"]
+mysql_kvbackend = ["common-meta/mysql_kvbackend"]
 
 [lints]
 workspace = true
 
 [dependencies]
-async-stream.workspace = true
 async-trait.workspace = true
 auth.workspace = true
 base64.workspace = true
@@ -43,14 +38,16 @@ common-time.workspace = true
 common-version.workspace = true
 common-wal.workspace = true
 datatypes.workspace = true
+either = "1.8"
 etcd-client.workspace = true
 futures.workspace = true
 humantime.workspace = true
 meta-client.workspace = true
-meta-srv.workspace = true
 nu-ansi-term = "0.46"
-object-store.workspace = true
-operator.workspace = true
+opendal = { version = "0.51.1", features = [
+    "services-fs",
+    "services-s3",
+] }
 query.workspace = true
 rand.workspace = true
 reqwest.workspace = true
@@ -66,7 +63,6 @@ tokio.workspace = true
 tracing-appender.workspace = true
 
 [dev-dependencies]
-common-meta = { workspace = true, features = ["testing"] }
 common-version.workspace = true
 serde.workspace = true
 tempfile.workspace = true

src/cli/src/bench.rs

@@ -58,7 +58,6 @@ where
     info!("{desc}, average operation cost: {cost:.2} ms");
 }
 
-/// Command to benchmark table metadata operations.
 #[derive(Debug, Default, Parser)]
 pub struct BenchTableMetadataCommand {
     #[clap(long)]
@@ -160,7 +159,6 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
         options: Default::default(),
         region_numbers: (1..=100).collect(),
         partition_key_indices: vec![],
-        column_ids: vec![],
     };
 
     RawTableInfo {
@@ -188,7 +186,6 @@ fn create_region_routes(regions: Vec<RegionNumber>) -> Vec<RegionRoute> {
                 name: String::new(),
                 partition: None,
                 attrs: BTreeMap::new(),
-                partition_expr: Default::default(),
             },
             leader_peer: Some(Peer {
                 id: rng.random_range(0..10),

src/cli/src/data.rs

@@ -1,39 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-mod export;
-mod import;
-
-use clap::Subcommand;
-use common_error::ext::BoxedError;
-
-use crate::data::export::ExportCommand;
-use crate::data::import::ImportCommand;
-use crate::Tool;
-
-/// Command for data operations including exporting data from and importing data into GreptimeDB.
-#[derive(Subcommand)]
-pub enum DataCommand {
-    Export(ExportCommand),
-    Import(ImportCommand),
-}
-
-impl DataCommand {
-    pub async fn build(&self) -> std::result::Result<Box<dyn Tool>, BoxedError> {
-        match self {
-            DataCommand::Export(cmd) => cmd.build().await,
-            DataCommand::Import(cmd) => cmd.build().await,
-        }
-    }
-}

src/cli/src/error.rs

@@ -17,10 +17,7 @@ use std::any::Any;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
-use common_meta::peer::Peer;
-use object_store::Error as ObjectStoreError;
 use snafu::{Location, Snafu};
-use store_api::storage::TableId;
 
 #[derive(Snafu)]
 #[snafu(visibility(pub))]
@@ -32,7 +29,6 @@ pub enum Error {
         location: Location,
         msg: String,
     },
-
     #[snafu(display("Failed to create default catalog and schema"))]
     InitMetadata {
         #[snafu(implicit)]
@@ -75,20 +71,6 @@ pub enum Error {
         source: common_meta::error::Error,
     },
 
-    #[snafu(display("Failed to get table metadata"))]
-    TableMetadata {
-        #[snafu(implicit)]
-        location: Location,
-        source: common_meta::error::Error,
-    },
-
-    #[snafu(display("Unexpected error: {}", msg))]
-    Unexpected {
-        msg: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Missing config, msg: {}", msg))]
     MissingConfig {
         msg: String,
@@ -238,81 +220,23 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Table not found: {table_id}"))]
-    TableNotFound {
-        table_id: TableId,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("OpenDAL operator failed"))]
     OpenDal {
         #[snafu(implicit)]
         location: Location,
         #[snafu(source)]
-        error: ObjectStoreError,
+        error: opendal::Error,
     },
-
     #[snafu(display("S3 config need be set"))]
     S3ConfigNotSet {
         #[snafu(implicit)]
         location: Location,
     },
-
     #[snafu(display("Output directory not set"))]
     OutputDirNotSet {
         #[snafu(implicit)]
         location: Location,
     },
-
-    #[snafu(display("Empty store addresses"))]
-    EmptyStoreAddrs {
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Unsupported memory backend"))]
-    UnsupportedMemoryBackend {
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("File path invalid: {}", msg))]
-    InvalidFilePath {
-        msg: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Invalid arguments: {}", msg))]
-    InvalidArguments {
-        msg: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to init backend"))]
-    InitBackend {
-        #[snafu(implicit)]
-        location: Location,
-        #[snafu(source)]
-        error: ObjectStoreError,
-    },
-
-    #[snafu(display("Covert column schemas to defs failed"))]
-    CovertColumnSchemasToDefs {
-        #[snafu(implicit)]
-        location: Location,
-        source: operator::error::Error,
-    },
-
-    #[snafu(display("Failed to send request to datanode: {}", peer))]
-    SendRequestToDatanode {
-        peer: Peer,
-        #[snafu(implicit)]
-        location: Location,
-        source: common_meta::error::Error,
-    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -320,9 +244,9 @@ pub type Result<T> = std::result::Result<T, Error>;
 impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
-            Error::InitMetadata { source, .. }
-            | Error::InitDdlManager { source, .. }
-            | Error::TableMetadata { source, .. } => source.status_code(),
+            Error::InitMetadata { source, .. } | Error::InitDdlManager { source, .. } => {
+                source.status_code()
+            }
 
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
@@ -331,14 +255,8 @@ impl ErrorExt for Error {
             | Error::ConnectEtcd { .. }
             | Error::CreateDir { .. }
             | Error::EmptyResult { .. }
-            | Error::InvalidFilePath { .. }
-            | Error::UnsupportedMemoryBackend { .. }
-            | Error::InvalidArguments { .. }
             | Error::ParseProxyOpts { .. } => StatusCode::InvalidArguments,
 
-            Error::CovertColumnSchemasToDefs { source, .. } => source.status_code(),
-            Error::SendRequestToDatanode { source, .. } => source.status_code(),
-
             Error::StartProcedureManager { source, .. }
             | Error::StopProcedureManager { source, .. } => source.status_code(),
             Error::StartWalOptionsAllocator { source, .. } => source.status_code(),
@@ -346,7 +264,6 @@ impl ErrorExt for Error {
             Error::ParseSql { source, .. } | Error::PlanStatement { source, .. } => {
                 source.status_code()
             }
-            Error::Unexpected { .. } => StatusCode::Unexpected,
 
             Error::SerdeJson { .. }
             | Error::FileIo { .. }
@@ -355,16 +272,14 @@ impl ErrorExt for Error {
             | Error::BuildClient { .. } => StatusCode::Unexpected,
 
             Error::Other { source, .. } => source.status_code(),
-            Error::OpenDal { .. } | Error::InitBackend { .. } => StatusCode::Internal,
-            Error::S3ConfigNotSet { .. }
-            | Error::OutputDirNotSet { .. }
-            | Error::EmptyStoreAddrs { .. } => StatusCode::InvalidArguments,
+            Error::OpenDal { .. } => StatusCode::Internal,
+            Error::S3ConfigNotSet { .. } => StatusCode::InvalidArguments,
+            Error::OutputDirNotSet { .. } => StatusCode::InvalidArguments,
 
             Error::BuildRuntime { source, .. } => source.status_code(),
 
             Error::CacheRequired { .. } | Error::BuildCacheRegistry { .. } => StatusCode::Internal,
             Error::MetaClientInit { source, .. } => source.status_code(),
-            Error::TableNotFound { .. } => StatusCode::TableNotFound,
             Error::SchemaNotFound { .. } => StatusCode::DatabaseNotFound,
         }
     }

src/cli/src/export.rs

@@ -19,12 +19,10 @@ use std::time::Duration;
 
 use async_trait::async_trait;
 use clap::{Parser, ValueEnum};
-use common_base::secrets::{ExposeSecret, SecretString};
 use common_error::ext::BoxedError;
 use common_telemetry::{debug, error, info};
-use object_store::layers::LoggingLayer;
-use object_store::services::Oss;
-use object_store::{services, ObjectStore};
+use opendal::layers::LoggingLayer;
+use opendal::{services, Operator};
 use serde_json::Value;
 use snafu::{OptionExt, ResultExt};
 use tokio::sync::Semaphore;
@@ -50,7 +48,6 @@ enum ExportTarget {
     All,
 }
 
-/// Command for exporting data from the GreptimeDB.
 #[derive(Debug, Default, Parser)]
 pub struct ExportCommand {
     /// Server address to connect
@@ -113,15 +110,15 @@ pub struct ExportCommand {
     #[clap(long)]
     s3: bool,
 
-    /// if both `ddl_local_dir` and remote storage (s3/oss) are set, `ddl_local_dir` will be only used for
-    /// exported SQL files, and the data will be exported to remote storage.
+    /// if both `s3_ddl_local_dir` and `s3` are set, `s3_ddl_local_dir` will be only used for
+    /// exported SQL files, and the data will be exported to s3.
     ///
-    /// Note that `ddl_local_dir` export sql files to **LOCAL** file system, this is useful if export client don't have
-    /// direct access to remote storage.
+    /// Note that `s3_ddl_local_dir` export sql files to **LOCAL** file system, this is useful if export client don't have
+    /// direct access to s3.
     ///
-    /// if remote storage is set but `ddl_local_dir` is not set, both SQL&data will be exported to remote storage.
+    /// if `s3` is set but `s3_ddl_local_dir` is not set, both SQL&data will be exported to s3.
     #[clap(long)]
-    ddl_local_dir: Option<String>,
+    s3_ddl_local_dir: Option<String>,
 
     /// The s3 bucket name
     /// if s3 is set, this is required
@@ -152,30 +149,6 @@ pub struct ExportCommand {
     /// if s3 is set, this is required
     #[clap(long)]
     s3_region: Option<String>,
-
-    /// if export data to oss
-    #[clap(long)]
-    oss: bool,
-
-    /// The oss bucket name
-    /// if oss is set, this is required
-    #[clap(long)]
-    oss_bucket: Option<String>,
-
-    /// The oss endpoint
-    /// if oss is set, this is required
-    #[clap(long)]
-    oss_endpoint: Option<String>,
-
-    /// The oss access key id
-    /// if oss is set, this is required
-    #[clap(long)]
-    oss_access_key_id: Option<String>,
-
-    /// The oss access key secret
-    /// if oss is set, this is required
-    #[clap(long)]
-    oss_access_key_secret: Option<String>,
 }
 
 impl ExportCommand {
@@ -189,7 +162,7 @@ impl ExportCommand {
         {
             return Err(BoxedError::new(S3ConfigNotSetSnafu {}.build()));
         }
-        if !self.s3 && !self.oss && self.output_dir.is_none() {
+        if !self.s3 && self.output_dir.is_none() {
             return Err(BoxedError::new(OutputDirNotSetSnafu {}.build()));
         }
         let (catalog, schema) =
@@ -214,32 +187,13 @@ impl ExportCommand {
             start_time: self.start_time.clone(),
             end_time: self.end_time.clone(),
             s3: self.s3,
-            ddl_local_dir: self.ddl_local_dir.clone(),
+            s3_ddl_local_dir: self.s3_ddl_local_dir.clone(),
             s3_bucket: self.s3_bucket.clone(),
             s3_root: self.s3_root.clone(),
             s3_endpoint: self.s3_endpoint.clone(),
-            // Wrap sensitive values in SecretString
-            s3_access_key: self
-                .s3_access_key
-                .as_ref()
-                .map(|k| SecretString::from(k.clone())),
-            s3_secret_key: self
-                .s3_secret_key
-                .as_ref()
-                .map(|k| SecretString::from(k.clone())),
+            s3_access_key: self.s3_access_key.clone(),
+            s3_secret_key: self.s3_secret_key.clone(),
             s3_region: self.s3_region.clone(),
-            oss: self.oss,
-            oss_bucket: self.oss_bucket.clone(),
-            oss_endpoint: self.oss_endpoint.clone(),
-            // Wrap sensitive values in SecretString
-            oss_access_key_id: self
-                .oss_access_key_id
-                .as_ref()
-                .map(|k| SecretString::from(k.clone())),
-            oss_access_key_secret: self
-                .oss_access_key_secret
-                .as_ref()
-                .map(|k| SecretString::from(k.clone())),
         }))
     }
 }
@@ -255,30 +209,23 @@ pub struct Export {
     start_time: Option<String>,
     end_time: Option<String>,
     s3: bool,
-    ddl_local_dir: Option<String>,
+    s3_ddl_local_dir: Option<String>,
     s3_bucket: Option<String>,
     s3_root: Option<String>,
     s3_endpoint: Option<String>,
-    // Changed to SecretString for sensitive data
-    s3_access_key: Option<SecretString>,
-    s3_secret_key: Option<SecretString>,
+    s3_access_key: Option<String>,
+    s3_secret_key: Option<String>,
     s3_region: Option<String>,
-    oss: bool,
-    oss_bucket: Option<String>,
-    oss_endpoint: Option<String>,
-    // Changed to SecretString for sensitive data
-    oss_access_key_id: Option<SecretString>,
-    oss_access_key_secret: Option<SecretString>,
 }
 
 impl Export {
     fn catalog_path(&self) -> PathBuf {
-        if self.s3 || self.oss {
+        if self.s3 {
             PathBuf::from(&self.catalog)
         } else if let Some(dir) = &self.output_dir {
             PathBuf::from(dir).join(&self.catalog)
         } else {
-            unreachable!("catalog_path: output_dir must be set when not using remote storage")
+            unreachable!("catalog_path: output_dir must be set when not using s3")
         }
     }
 
@@ -480,7 +427,7 @@ impl Export {
                     .await?;
 
                 // Create directory if needed for file system storage
-                if !export_self.s3 && !export_self.oss {
+                if !export_self.s3 {
                     let db_dir = format!("{}/{}/", export_self.catalog, schema);
                     operator.create_dir(&db_dir).await.context(OpenDalSnafu)?;
                 }
@@ -523,35 +470,32 @@ impl Export {
         Ok(())
     }
 
-    async fn build_operator(&self) -> Result<ObjectStore> {
+    async fn build_operator(&self) -> Result<Operator> {
         if self.s3 {
             self.build_s3_operator().await
-        } else if self.oss {
-            self.build_oss_operator().await
         } else {
             self.build_fs_operator().await
         }
     }
 
     /// build operator with preference for file system
-    async fn build_prefer_fs_operator(&self) -> Result<ObjectStore> {
-        if (self.s3 || self.oss) && self.ddl_local_dir.is_some() {
-            let root = self.ddl_local_dir.as_ref().unwrap().clone();
-            let op = ObjectStore::new(services::Fs::default().root(&root))
+    async fn build_prefer_fs_operator(&self) -> Result<Operator> {
+        // is under s3 mode and s3_ddl_dir is set, use it as root
+        if self.s3 && self.s3_ddl_local_dir.is_some() {
+            let root = self.s3_ddl_local_dir.as_ref().unwrap().clone();
+            let op = Operator::new(services::Fs::default().root(&root))
                 .context(OpenDalSnafu)?
                 .layer(LoggingLayer::default())
                 .finish();
             Ok(op)
         } else if self.s3 {
             self.build_s3_operator().await
-        } else if self.oss {
-            self.build_oss_operator().await
         } else {
             self.build_fs_operator().await
         }
     }
 
-    async fn build_s3_operator(&self) -> Result<ObjectStore> {
+    async fn build_s3_operator(&self) -> Result<Operator> {
         let mut builder = services::S3::default().bucket(
             self.s3_bucket
                 .as_ref()
@@ -571,51 +515,27 @@ impl Export {
         }
 
         if let Some(key_id) = self.s3_access_key.as_ref() {
-            builder = builder.access_key_id(key_id.expose_secret());
+            builder = builder.access_key_id(key_id);
         }
 
         if let Some(secret_key) = self.s3_secret_key.as_ref() {
-            builder = builder.secret_access_key(secret_key.expose_secret());
+            builder = builder.secret_access_key(secret_key);
         }
 
-        let op = ObjectStore::new(builder)
+        let op = Operator::new(builder)
             .context(OpenDalSnafu)?
             .layer(LoggingLayer::default())
             .finish();
         Ok(op)
     }
 
-    async fn build_oss_operator(&self) -> Result<ObjectStore> {
-        let mut builder = Oss::default()
-            .bucket(self.oss_bucket.as_ref().expect("oss_bucket must be set"))
-            .endpoint(
-                self.oss_endpoint
-                    .as_ref()
-                    .expect("oss_endpoint must be set"),
-            );
-
-        // Use expose_secret() to access the actual secret value
-        if let Some(key_id) = self.oss_access_key_id.as_ref() {
-            builder = builder.access_key_id(key_id.expose_secret());
-        }
-        if let Some(secret_key) = self.oss_access_key_secret.as_ref() {
-            builder = builder.access_key_secret(secret_key.expose_secret());
-        }
-
-        let op = ObjectStore::new(builder)
-            .context(OpenDalSnafu)?
-            .layer(LoggingLayer::default())
-            .finish();
-        Ok(op)
-    }
-
-    async fn build_fs_operator(&self) -> Result<ObjectStore> {
+    async fn build_fs_operator(&self) -> Result<Operator> {
         let root = self
             .output_dir
             .as_ref()
             .context(OutputDirNotSetSnafu)?
             .clone();
-        let op = ObjectStore::new(services::Fs::default().root(&root))
+        let op = Operator::new(services::Fs::default().root(&root))
             .context(OpenDalSnafu)?
             .layer(LoggingLayer::default())
             .finish();
@@ -642,8 +562,8 @@ impl Export {
             tasks.push(async move {
                 let _permit = semaphore_moved.acquire().await.unwrap();
 
-                // Create directory if not using remote storage
-                if !export_self.s3 && !export_self.oss {
+                // Create directory if not using S3
+                if !export_self.s3 {
                     let db_dir = format!("{}/{}/", export_self.catalog, schema);
                     operator.create_dir(&db_dir).await.context(OpenDalSnafu)?;
                 }
@@ -655,11 +575,7 @@ impl Export {
                     r#"COPY DATABASE "{}"."{}" TO '{}' WITH ({}){};"#,
                     export_self.catalog, schema, path, with_options_clone, connection_part
                 );
-
-                // Log SQL command but mask sensitive information
-                let safe_sql = export_self.mask_sensitive_sql(&sql);
-                info!("Executing sql: {}", safe_sql);
-
+                info!("Executing sql: {sql}");
                 export_self.database_client.sql_in_public(&sql).await?;
                 info!(
                     "Finished exporting {}.{} data to {}",
@@ -699,29 +615,6 @@ impl Export {
         Ok(())
     }
 
-    /// Mask sensitive information in SQL commands for safe logging
-    fn mask_sensitive_sql(&self, sql: &str) -> String {
-        let mut masked_sql = sql.to_string();
-
-        // Mask S3 credentials
-        if let Some(access_key) = &self.s3_access_key {
-            masked_sql = masked_sql.replace(access_key.expose_secret(), "[REDACTED]");
-        }
-        if let Some(secret_key) = &self.s3_secret_key {
-            masked_sql = masked_sql.replace(secret_key.expose_secret(), "[REDACTED]");
-        }
-
-        // Mask OSS credentials
-        if let Some(access_key_id) = &self.oss_access_key_id {
-            masked_sql = masked_sql.replace(access_key_id.expose_secret(), "[REDACTED]");
-        }
-        if let Some(access_key_secret) = &self.oss_access_key_secret {
-            masked_sql = masked_sql.replace(access_key_secret.expose_secret(), "[REDACTED]");
-        }
-
-        masked_sql
-    }
-
     fn get_file_path(&self, schema: &str, file_name: &str) -> String {
         format!("{}/{}/{}", self.catalog, schema, file_name)
     }
@@ -738,13 +631,6 @@ impl Export {
                 },
                 file_path
             )
-        } else if self.oss {
-            format!(
-                "oss://{}/{}/{}",
-                self.oss_bucket.as_ref().unwrap_or(&String::new()),
-                self.catalog,
-                file_path
-            )
         } else {
             format!(
                 "{}/{}",
@@ -756,14 +642,11 @@ impl Export {
 
     async fn write_to_storage(
         &self,
-        op: &ObjectStore,
+        op: &Operator,
         file_path: &str,
         content: Vec<u8>,
     ) -> Result<()> {
-        op.write(file_path, content)
-            .await
-            .context(OpenDalSnafu)
-            .map(|_| ())
+        op.write(file_path, content).await.context(OpenDalSnafu)
     }
 
     fn get_storage_params(&self, schema: &str) -> (String, String) {
@@ -789,36 +672,15 @@ impl Export {
             };
 
             // Safety: All s3 options are required
-            // Use expose_secret() to access the actual secret values
             let connection_options = format!(
                 "ACCESS_KEY_ID='{}', SECRET_ACCESS_KEY='{}', REGION='{}'{}",
-                self.s3_access_key.as_ref().unwrap().expose_secret(),
-                self.s3_secret_key.as_ref().unwrap().expose_secret(),
+                self.s3_access_key.as_ref().unwrap(),
+                self.s3_secret_key.as_ref().unwrap(),
                 self.s3_region.as_ref().unwrap(),
                 endpoint_option
             );
 
             (s3_path, format!(" CONNECTION ({})", connection_options))
-        } else if self.oss {
-            let oss_path = format!(
-                "oss://{}/{}/{}/",
-                self.oss_bucket.as_ref().unwrap(),
-                self.catalog,
-                schema
-            );
-            let endpoint_option = if let Some(endpoint) = self.oss_endpoint.as_ref() {
-                format!(", ENDPOINT='{}'", endpoint)
-            } else {
-                String::new()
-            };
-
-            let connection_options = format!(
-                "ACCESS_KEY_ID='{}', ACCESS_KEY_SECRET='{}'{}",
-                self.oss_access_key_id.as_ref().unwrap().expose_secret(),
-                self.oss_access_key_secret.as_ref().unwrap().expose_secret(),
-                endpoint_option
-            );
-            (oss_path, format!(" CONNECTION ({})", connection_options))
         } else {
             (
                 self.catalog_path()

src/cli/src/import.rs

@@ -40,7 +40,6 @@ enum ImportTarget {
     All,
 }
 
-/// Command to import data from a directory into a GreptimeDB instance.
 #[derive(Debug, Default, Parser)]
 pub struct ImportCommand {
     /// Server address to connect

src/cli/src/lib.rs

@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![allow(clippy::print_stdout)]
 mod bench;
-mod data;
 mod database;
 pub mod error;
-mod metadata;
+mod export;
+mod import;
 
 use async_trait::async_trait;
 use clap::Parser;
@@ -26,8 +25,8 @@ pub use database::DatabaseClient;
 use error::Result;
 
 pub use crate::bench::BenchTableMetadataCommand;
-pub use crate::data::DataCommand;
-pub use crate::metadata::MetadataCommand;
+pub use crate::export::ExportCommand;
+pub use crate::import::ImportCommand;
 
 #[async_trait]
 pub trait Tool: Send + Sync {

src/cli/src/metadata.rs

@@ -1,52 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-mod common;
-mod control;
-mod repair;
-mod snapshot;
-mod utils;
-
-use clap::Subcommand;
-use common_error::ext::BoxedError;
-
-use crate::metadata::control::{DelCommand, GetCommand};
-use crate::metadata::repair::RepairLogicalTablesCommand;
-use crate::metadata::snapshot::SnapshotCommand;
-use crate::Tool;
-
-/// Command for managing metadata operations,
-/// including saving and restoring metadata snapshots,
-/// controlling metadata operations, and diagnosing and repairing metadata.
-#[derive(Subcommand)]
-pub enum MetadataCommand {
-    #[clap(subcommand)]
-    Snapshot(SnapshotCommand),
-    #[clap(subcommand)]
-    Get(GetCommand),
-    #[clap(subcommand)]
-    Del(DelCommand),
-    RepairLogicalTables(RepairLogicalTablesCommand),
-}
-
-impl MetadataCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        match self {
-            MetadataCommand::Snapshot(cmd) => cmd.build().await,
-            MetadataCommand::RepairLogicalTables(cmd) => cmd.build().await,
-            MetadataCommand::Get(cmd) => cmd.build().await,
-            MetadataCommand::Del(cmd) => cmd.build().await,
-        }
-    }
-}

src/cli/src/metadata/common.rs

@@ -1,116 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use clap::Parser;
-use common_error::ext::BoxedError;
-use common_meta::kv_backend::chroot::ChrootKvBackend;
-use common_meta::kv_backend::etcd::EtcdStore;
-use common_meta::kv_backend::KvBackendRef;
-use meta_srv::bootstrap::create_etcd_client;
-use meta_srv::metasrv::BackendImpl;
-
-use crate::error::{EmptyStoreAddrsSnafu, UnsupportedMemoryBackendSnafu};
-
-#[derive(Debug, Default, Parser)]
-pub(crate) struct StoreConfig {
-    /// The endpoint of store. one of etcd, postgres or mysql.
-    ///
-    /// For postgres store, the format is:
-    /// "password=password dbname=postgres user=postgres host=localhost port=5432"
-    ///
-    /// For etcd store, the format is:
-    /// "127.0.0.1:2379"
-    ///
-    /// For mysql store, the format is:
-    /// "mysql://user:password@ip:port/dbname"
-    #[clap(long, alias = "store-addr", value_delimiter = ',', num_args = 1..)]
-    store_addrs: Vec<String>,
-
-    /// The maximum number of operations in a transaction. Only used when using [etcd-store].
-    #[clap(long, default_value = "128")]
-    max_txn_ops: usize,
-
-    /// The metadata store backend.
-    #[clap(long, value_enum, default_value = "etcd-store")]
-    backend: BackendImpl,
-
-    /// The key prefix of the metadata store.
-    #[clap(long, default_value = "")]
-    store_key_prefix: String,
-
-    /// The table name in RDS to store metadata. Only used when using [postgres-store] or [mysql-store].
-    #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
-    #[clap(long, default_value = common_meta::kv_backend::DEFAULT_META_TABLE_NAME)]
-    meta_table_name: String,
-}
-
-impl StoreConfig {
-    /// Builds a [`KvBackendRef`] from the store configuration.
-    pub async fn build(&self) -> Result<KvBackendRef, BoxedError> {
-        let max_txn_ops = self.max_txn_ops;
-        let store_addrs = &self.store_addrs;
-        if store_addrs.is_empty() {
-            EmptyStoreAddrsSnafu.fail().map_err(BoxedError::new)
-        } else {
-            let kvbackend = match self.backend {
-                BackendImpl::EtcdStore => {
-                    let etcd_client = create_etcd_client(store_addrs)
-                        .await
-                        .map_err(BoxedError::new)?;
-                    Ok(EtcdStore::with_etcd_client(etcd_client, max_txn_ops))
-                }
-                #[cfg(feature = "pg_kvbackend")]
-                BackendImpl::PostgresStore => {
-                    let table_name = &self.meta_table_name;
-                    let pool = meta_srv::bootstrap::create_postgres_pool(store_addrs)
-                        .await
-                        .map_err(BoxedError::new)?;
-                    Ok(common_meta::kv_backend::rds::PgStore::with_pg_pool(
-                        pool,
-                        table_name,
-                        max_txn_ops,
-                    )
-                    .await
-                    .map_err(BoxedError::new)?)
-                }
-                #[cfg(feature = "mysql_kvbackend")]
-                BackendImpl::MysqlStore => {
-                    let table_name = &self.meta_table_name;
-                    let pool = meta_srv::bootstrap::create_mysql_pool(store_addrs)
-                        .await
-                        .map_err(BoxedError::new)?;
-                    Ok(common_meta::kv_backend::rds::MySqlStore::with_mysql_pool(
-                        pool,
-                        table_name,
-                        max_txn_ops,
-                    )
-                    .await
-                    .map_err(BoxedError::new)?)
-                }
-                BackendImpl::MemoryStore => UnsupportedMemoryBackendSnafu
-                    .fail()
-                    .map_err(BoxedError::new),
-            };
-            if self.store_key_prefix.is_empty() {
-                kvbackend
-            } else {
-                let chroot_kvbackend =
-                    ChrootKvBackend::new(self.store_key_prefix.as_bytes().to_vec(), kvbackend?);
-                Ok(Arc::new(chroot_kvbackend))
-            }
-        }
-    }
-}

src/cli/src/metadata/control.rs

@@ -1,22 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-mod del;
-mod get;
-#[cfg(test)]
-mod test_utils;
-mod utils;
-
-pub(crate) use del::DelCommand;
-pub(crate) use get::GetCommand;

src/cli/src/metadata/control/del.rs

@@ -1,42 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-mod key;
-mod table;
-
-use clap::Subcommand;
-use common_error::ext::BoxedError;
-
-use crate::metadata::control::del::key::DelKeyCommand;
-use crate::metadata::control::del::table::DelTableCommand;
-use crate::Tool;
-
-/// The prefix of the tombstone keys.
-pub(crate) const CLI_TOMBSTONE_PREFIX: &str = "__cli_tombstone/";
-
-/// Subcommand for deleting metadata from the metadata store.
-#[derive(Subcommand)]
-pub enum DelCommand {
-    Key(DelKeyCommand),
-    Table(DelTableCommand),
-}
-
-impl DelCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        match self {
-            DelCommand::Key(cmd) => cmd.build().await,
-            DelCommand::Table(cmd) => cmd.build().await,
-        }
-    }
-}

src/cli/src/metadata/control/del/key.rs

@@ -1,132 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use async_trait::async_trait;
-use clap::Parser;
-use common_error::ext::BoxedError;
-use common_meta::key::tombstone::TombstoneManager;
-use common_meta::kv_backend::KvBackendRef;
-use common_meta::rpc::store::RangeRequest;
-
-use crate::metadata::common::StoreConfig;
-use crate::metadata::control::del::CLI_TOMBSTONE_PREFIX;
-use crate::Tool;
-
-/// Delete key-value pairs logically from the metadata store.
-#[derive(Debug, Default, Parser)]
-pub struct DelKeyCommand {
-    /// The key to delete from the metadata store.
-    key: String,
-
-    /// Delete key-value pairs with the given prefix.
-    #[clap(long)]
-    prefix: bool,
-
-    #[clap(flatten)]
-    store: StoreConfig,
-}
-
-impl DelKeyCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        let kv_backend = self.store.build().await?;
-        Ok(Box::new(DelKeyTool {
-            key: self.key.to_string(),
-            prefix: self.prefix,
-            key_deleter: KeyDeleter::new(kv_backend),
-        }))
-    }
-}
-
-struct KeyDeleter {
-    kv_backend: KvBackendRef,
-    tombstone_manager: TombstoneManager,
-}
-
-impl KeyDeleter {
-    fn new(kv_backend: KvBackendRef) -> Self {
-        Self {
-            kv_backend: kv_backend.clone(),
-            tombstone_manager: TombstoneManager::new_with_prefix(kv_backend, CLI_TOMBSTONE_PREFIX),
-        }
-    }
-
-    async fn delete(&self, key: &str, prefix: bool) -> Result<usize, BoxedError> {
-        let mut req = RangeRequest::default().with_keys_only();
-        if prefix {
-            req = req.with_prefix(key.as_bytes());
-        } else {
-            req = req.with_key(key.as_bytes());
-        }
-        let resp = self.kv_backend.range(req).await.map_err(BoxedError::new)?;
-        let keys = resp.kvs.iter().map(|kv| kv.key.clone()).collect::<Vec<_>>();
-        self.tombstone_manager
-            .create(keys)
-            .await
-            .map_err(BoxedError::new)
-    }
-}
-
-struct DelKeyTool {
-    key: String,
-    prefix: bool,
-    key_deleter: KeyDeleter,
-}
-
-#[async_trait]
-impl Tool for DelKeyTool {
-    async fn do_work(&self) -> Result<(), BoxedError> {
-        let deleted = self.key_deleter.delete(&self.key, self.prefix).await?;
-        // Print the number of deleted keys.
-        println!("{}", deleted);
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use common_meta::kv_backend::chroot::ChrootKvBackend;
-    use common_meta::kv_backend::memory::MemoryKvBackend;
-    use common_meta::kv_backend::{KvBackend, KvBackendRef};
-    use common_meta::rpc::store::RangeRequest;
-
-    use crate::metadata::control::del::key::KeyDeleter;
-    use crate::metadata::control::del::CLI_TOMBSTONE_PREFIX;
-    use crate::metadata::control::test_utils::put_key;
-
-    #[tokio::test]
-    async fn test_delete_keys() {
-        let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
-        let key_deleter = KeyDeleter::new(kv_backend.clone());
-        put_key(&kv_backend, "foo", "bar").await;
-        put_key(&kv_backend, "foo/bar", "baz").await;
-        put_key(&kv_backend, "foo/baz", "qux").await;
-        let deleted = key_deleter.delete("foo", true).await.unwrap();
-        assert_eq!(deleted, 3);
-        let deleted = key_deleter.delete("foo/bar", false).await.unwrap();
-        assert_eq!(deleted, 0);
-
-        let chroot = ChrootKvBackend::new(CLI_TOMBSTONE_PREFIX.as_bytes().to_vec(), kv_backend);
-        let req = RangeRequest::default().with_prefix(b"foo");
-        let resp = chroot.range(req).await.unwrap();
-        assert_eq!(resp.kvs.len(), 3);
-        assert_eq!(resp.kvs[0].key, b"foo");
-        assert_eq!(resp.kvs[0].value, b"bar");
-        assert_eq!(resp.kvs[1].key, b"foo/bar");
-        assert_eq!(resp.kvs[1].value, b"baz");
-        assert_eq!(resp.kvs[2].key, b"foo/baz");
-        assert_eq!(resp.kvs[2].value, b"qux");
-    }
-}

src/cli/src/metadata/control/del/table.rs

@@ -1,235 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use async_trait::async_trait;
-use clap::Parser;
-use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use common_catalog::format_full_table_name;
-use common_error::ext::BoxedError;
-use common_meta::ddl::utils::get_region_wal_options;
-use common_meta::key::table_name::TableNameManager;
-use common_meta::key::TableMetadataManager;
-use common_meta::kv_backend::KvBackendRef;
-use store_api::storage::TableId;
-
-use crate::error::{InvalidArgumentsSnafu, TableNotFoundSnafu};
-use crate::metadata::common::StoreConfig;
-use crate::metadata::control::del::CLI_TOMBSTONE_PREFIX;
-use crate::metadata::control::utils::get_table_id_by_name;
-use crate::Tool;
-
-/// Delete table metadata logically from the metadata store.
-#[derive(Debug, Default, Parser)]
-pub struct DelTableCommand {
-    /// The table id to delete from the metadata store.
-    #[clap(long)]
-    table_id: Option<u32>,
-
-    /// The table name to delete from the metadata store.
-    #[clap(long)]
-    table_name: Option<String>,
-
-    /// The schema name of the table.
-    #[clap(long, default_value = DEFAULT_SCHEMA_NAME)]
-    schema_name: String,
-
-    /// The catalog name of the table.
-    #[clap(long, default_value = DEFAULT_CATALOG_NAME)]
-    catalog_name: String,
-
-    #[clap(flatten)]
-    store: StoreConfig,
-}
-
-impl DelTableCommand {
-    fn validate(&self) -> Result<(), BoxedError> {
-        if matches!(
-            (&self.table_id, &self.table_name),
-            (Some(_), Some(_)) | (None, None)
-        ) {
-            return Err(BoxedError::new(
-                InvalidArgumentsSnafu {
-                    msg: "You must specify either --table-id or --table-name.",
-                }
-                .build(),
-            ));
-        }
-        Ok(())
-    }
-}
-
-impl DelTableCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        self.validate()?;
-        let kv_backend = self.store.build().await?;
-        Ok(Box::new(DelTableTool {
-            table_id: self.table_id,
-            table_name: self.table_name.clone(),
-            schema_name: self.schema_name.clone(),
-            catalog_name: self.catalog_name.clone(),
-            table_name_manager: TableNameManager::new(kv_backend.clone()),
-            table_metadata_deleter: TableMetadataDeleter::new(kv_backend),
-        }))
-    }
-}
-
-struct DelTableTool {
-    table_id: Option<u32>,
-    table_name: Option<String>,
-    schema_name: String,
-    catalog_name: String,
-    table_name_manager: TableNameManager,
-    table_metadata_deleter: TableMetadataDeleter,
-}
-
-#[async_trait]
-impl Tool for DelTableTool {
-    async fn do_work(&self) -> Result<(), BoxedError> {
-        let table_id = if let Some(table_name) = &self.table_name {
-            let catalog_name = &self.catalog_name;
-            let schema_name = &self.schema_name;
-
-            let Some(table_id) = get_table_id_by_name(
-                &self.table_name_manager,
-                catalog_name,
-                schema_name,
-                table_name,
-            )
-            .await?
-            else {
-                println!(
-                    "Table({}) not found",
-                    format_full_table_name(catalog_name, schema_name, table_name)
-                );
-                return Ok(());
-            };
-            table_id
-        } else {
-            // Safety: we have validated that table_id or table_name is not None
-            self.table_id.unwrap()
-        };
-        self.table_metadata_deleter.delete(table_id).await?;
-        println!("Table({}) deleted", table_id);
-
-        Ok(())
-    }
-}
-
-struct TableMetadataDeleter {
-    table_metadata_manager: TableMetadataManager,
-}
-
-impl TableMetadataDeleter {
-    fn new(kv_backend: KvBackendRef) -> Self {
-        Self {
-            table_metadata_manager: TableMetadataManager::new_with_custom_tombstone_prefix(
-                kv_backend,
-                CLI_TOMBSTONE_PREFIX,
-            ),
-        }
-    }
-
-    async fn delete(&self, table_id: TableId) -> Result<(), BoxedError> {
-        let (table_info, table_route) = self
-            .table_metadata_manager
-            .get_full_table_info(table_id)
-            .await
-            .map_err(BoxedError::new)?;
-        let Some(table_info) = table_info else {
-            return Err(BoxedError::new(TableNotFoundSnafu { table_id }.build()));
-        };
-        let Some(table_route) = table_route else {
-            return Err(BoxedError::new(TableNotFoundSnafu { table_id }.build()));
-        };
-        let physical_table_id = self
-            .table_metadata_manager
-            .table_route_manager()
-            .get_physical_table_id(table_id)
-            .await
-            .map_err(BoxedError::new)?;
-
-        let table_name = table_info.table_name();
-        let region_wal_options = get_region_wal_options(
-            &self.table_metadata_manager,
-            &table_route,
-            physical_table_id,
-        )
-        .await
-        .map_err(BoxedError::new)?;
-
-        self.table_metadata_manager
-            .delete_table_metadata(table_id, &table_name, &table_route, &region_wal_options)
-            .await
-            .map_err(BoxedError::new)?;
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::collections::HashMap;
-    use std::sync::Arc;
-
-    use common_error::ext::ErrorExt;
-    use common_error::status_code::StatusCode;
-    use common_meta::key::table_route::TableRouteValue;
-    use common_meta::key::TableMetadataManager;
-    use common_meta::kv_backend::chroot::ChrootKvBackend;
-    use common_meta::kv_backend::memory::MemoryKvBackend;
-    use common_meta::kv_backend::{KvBackend, KvBackendRef};
-    use common_meta::rpc::store::RangeRequest;
-
-    use crate::metadata::control::del::table::TableMetadataDeleter;
-    use crate::metadata::control::del::CLI_TOMBSTONE_PREFIX;
-    use crate::metadata::control::test_utils::prepare_physical_table_metadata;
-
-    #[tokio::test]
-    async fn test_delete_table_not_found() {
-        let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
-
-        let table_metadata_deleter = TableMetadataDeleter::new(kv_backend);
-        let table_id = 1;
-        let err = table_metadata_deleter.delete(table_id).await.unwrap_err();
-        assert_eq!(err.status_code(), StatusCode::TableNotFound);
-    }
-
-    #[tokio::test]
-    async fn test_delete_table_metadata() {
-        let kv_backend = Arc::new(MemoryKvBackend::new());
-        let table_metadata_manager = TableMetadataManager::new(kv_backend.clone());
-        let table_id = 1024;
-        let (table_info, table_route) = prepare_physical_table_metadata("my_table", table_id).await;
-        table_metadata_manager
-            .create_table_metadata(
-                table_info,
-                TableRouteValue::Physical(table_route),
-                HashMap::new(),
-            )
-            .await
-            .unwrap();
-
-        let total_keys = kv_backend.len();
-        assert!(total_keys > 0);
-
-        let table_metadata_deleter = TableMetadataDeleter::new(kv_backend.clone());
-        table_metadata_deleter.delete(table_id).await.unwrap();
-
-        // Check the tombstone keys are deleted
-        let chroot =
-            ChrootKvBackend::new(CLI_TOMBSTONE_PREFIX.as_bytes().to_vec(), kv_backend.clone());
-        let req = RangeRequest::default().with_range(vec![0], vec![0]);
-        let resp = chroot.range(req).await.unwrap();
-        assert_eq!(resp.kvs.len(), total_keys);
-    }
-}

src/cli/src/metadata/control/get.rs

@@ -1,247 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::cmp::min;
-
-use async_trait::async_trait;
-use clap::{Parser, Subcommand};
-use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use common_catalog::format_full_table_name;
-use common_error::ext::BoxedError;
-use common_meta::key::table_info::TableInfoKey;
-use common_meta::key::table_route::TableRouteKey;
-use common_meta::key::TableMetadataManager;
-use common_meta::kv_backend::KvBackendRef;
-use common_meta::range_stream::{PaginationStream, DEFAULT_PAGE_SIZE};
-use common_meta::rpc::store::RangeRequest;
-use futures::TryStreamExt;
-
-use crate::error::InvalidArgumentsSnafu;
-use crate::metadata::common::StoreConfig;
-use crate::metadata::control::utils::{decode_key_value, get_table_id_by_name, json_fromatter};
-use crate::Tool;
-
-/// Getting metadata from metadata store.
-#[derive(Subcommand)]
-pub enum GetCommand {
-    Key(GetKeyCommand),
-    Table(GetTableCommand),
-}
-
-impl GetCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        match self {
-            GetCommand::Key(cmd) => cmd.build().await,
-            GetCommand::Table(cmd) => cmd.build().await,
-        }
-    }
-}
-
-/// Get key-value pairs from the metadata store.
-#[derive(Debug, Default, Parser)]
-pub struct GetKeyCommand {
-    /// The key to get from the metadata store.
-    #[clap(default_value = "")]
-    key: String,
-
-    /// Whether to perform a prefix query. If true, returns all key-value pairs where the key starts with the given prefix.
-    #[clap(long, default_value = "false")]
-    prefix: bool,
-
-    /// The maximum number of key-value pairs to return. If 0, returns all key-value pairs.
-    #[clap(long, default_value = "0")]
-    limit: u64,
-
-    #[clap(flatten)]
-    store: StoreConfig,
-}
-
-impl GetKeyCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        let kvbackend = self.store.build().await?;
-        Ok(Box::new(GetKeyTool {
-            kvbackend,
-            key: self.key.clone(),
-            prefix: self.prefix,
-            limit: self.limit,
-        }))
-    }
-}
-
-struct GetKeyTool {
-    kvbackend: KvBackendRef,
-    key: String,
-    prefix: bool,
-    limit: u64,
-}
-
-#[async_trait]
-impl Tool for GetKeyTool {
-    async fn do_work(&self) -> Result<(), BoxedError> {
-        let mut req = RangeRequest::default();
-        if self.prefix {
-            req = req.with_prefix(self.key.as_bytes());
-        } else {
-            req = req.with_key(self.key.as_bytes());
-        }
-        let page_size = if self.limit > 0 {
-            min(self.limit as usize, DEFAULT_PAGE_SIZE)
-        } else {
-            DEFAULT_PAGE_SIZE
-        };
-        let pagination_stream =
-            PaginationStream::new(self.kvbackend.clone(), req, page_size, decode_key_value);
-        let mut stream = Box::pin(pagination_stream.into_stream());
-        let mut counter = 0;
-
-        while let Some((key, value)) = stream.try_next().await.map_err(BoxedError::new)? {
-            print!("{}\n{}\n", key, value);
-            counter += 1;
-            if self.limit > 0 && counter >= self.limit {
-                break;
-            }
-        }
-
-        Ok(())
-    }
-}
-
-/// Get table metadata from the metadata store via table id.
-#[derive(Debug, Default, Parser)]
-pub struct GetTableCommand {
-    /// Get table metadata by table id.
-    #[clap(long)]
-    table_id: Option<u32>,
-
-    /// Get table metadata by table name.
-    #[clap(long)]
-    table_name: Option<String>,
-
-    /// The schema name of the table.
-    #[clap(long, default_value = DEFAULT_SCHEMA_NAME)]
-    schema_name: String,
-
-    /// The catalog name of the table.
-    #[clap(long, default_value = DEFAULT_CATALOG_NAME)]
-    catalog_name: String,
-
-    /// Pretty print the output.
-    #[clap(long, default_value = "false")]
-    pretty: bool,
-
-    #[clap(flatten)]
-    store: StoreConfig,
-}
-
-impl GetTableCommand {
-    pub fn validate(&self) -> Result<(), BoxedError> {
-        if matches!(
-            (&self.table_id, &self.table_name),
-            (Some(_), Some(_)) | (None, None)
-        ) {
-            return Err(BoxedError::new(
-                InvalidArgumentsSnafu {
-                    msg: "You must specify either --table-id or --table-name.",
-                }
-                .build(),
-            ));
-        }
-        Ok(())
-    }
-}
-
-struct GetTableTool {
-    kvbackend: KvBackendRef,
-    table_id: Option<u32>,
-    table_name: Option<String>,
-    schema_name: String,
-    catalog_name: String,
-    pretty: bool,
-}
-
-#[async_trait]
-impl Tool for GetTableTool {
-    async fn do_work(&self) -> Result<(), BoxedError> {
-        let table_metadata_manager = TableMetadataManager::new(self.kvbackend.clone());
-        let table_name_manager = table_metadata_manager.table_name_manager();
-        let table_info_manager = table_metadata_manager.table_info_manager();
-        let table_route_manager = table_metadata_manager.table_route_manager();
-
-        let table_id = if let Some(table_name) = &self.table_name {
-            let catalog_name = &self.catalog_name;
-            let schema_name = &self.schema_name;
-
-            let Some(table_id) =
-                get_table_id_by_name(table_name_manager, catalog_name, schema_name, table_name)
-                    .await?
-            else {
-                println!(
-                    "Table({}) not found",
-                    format_full_table_name(catalog_name, schema_name, table_name)
-                );
-                return Ok(());
-            };
-            table_id
-        } else {
-            // Safety: we have validated that table_id or table_name is not None
-            self.table_id.unwrap()
-        };
-
-        let table_info = table_info_manager
-            .get(table_id)
-            .await
-            .map_err(BoxedError::new)?;
-        if let Some(table_info) = table_info {
-            println!(
-                "{}\n{}",
-                TableInfoKey::new(table_id),
-                json_fromatter(self.pretty, &*table_info)
-            );
-        } else {
-            println!("Table info not found");
-        }
-
-        let table_route = table_route_manager
-            .table_route_storage()
-            .get(table_id)
-            .await
-            .map_err(BoxedError::new)?;
-        if let Some(table_route) = table_route {
-            println!(
-                "{}\n{}",
-                TableRouteKey::new(table_id),
-                json_fromatter(self.pretty, &table_route)
-            );
-        } else {
-            println!("Table route not found");
-        }
-
-        Ok(())
-    }
-}
-
-impl GetTableCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        self.validate()?;
-        let kvbackend = self.store.build().await?;
-        Ok(Box::new(GetTableTool {
-            kvbackend,
-            table_id: self.table_id,
-            table_name: self.table_name.clone(),
-            schema_name: self.schema_name.clone(),
-            catalog_name: self.catalog_name.clone(),
-            pretty: self.pretty,
-        }))
-    }
-}

src/cli/src/metadata/control/test_utils.rs

@@ -1,51 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_meta::ddl::test_util::test_create_physical_table_task;
-use common_meta::key::table_route::PhysicalTableRouteValue;
-use common_meta::kv_backend::KvBackendRef;
-use common_meta::peer::Peer;
-use common_meta::rpc::router::{Region, RegionRoute};
-use common_meta::rpc::store::PutRequest;
-use store_api::storage::{RegionId, TableId};
-use table::metadata::RawTableInfo;
-
-/// Puts a key-value pair into the kv backend.
-pub async fn put_key(kv_backend: &KvBackendRef, key: &str, value: &str) {
-    let put_req = PutRequest::new()
-        .with_key(key.as_bytes())
-        .with_value(value.as_bytes());
-    kv_backend.put(put_req).await.unwrap();
-}
-
-/// Prepares the physical table metadata for testing.
-///
-/// Returns the table info and the table route.
-pub async fn prepare_physical_table_metadata(
-    table_name: &str,
-    table_id: TableId,
-) -> (RawTableInfo, PhysicalTableRouteValue) {
-    let mut create_physical_table_task = test_create_physical_table_task(table_name);
-    let table_route = PhysicalTableRouteValue::new(vec![RegionRoute {
-        region: Region {
-            id: RegionId::new(table_id, 1),
-            ..Default::default()
-        },
-        leader_peer: Some(Peer::empty(1)),
-        ..Default::default()
-    }]);
-    create_physical_table_task.set_table_id(table_id);
-
-    (create_physical_table_task.table_info, table_route)
-}

src/cli/src/metadata/control/utils.rs

@@ -1,57 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_error::ext::BoxedError;
-use common_meta::error::Result as CommonMetaResult;
-use common_meta::key::table_name::{TableNameKey, TableNameManager};
-use common_meta::rpc::KeyValue;
-use serde::Serialize;
-use store_api::storage::TableId;
-
-/// Decodes a key-value pair into a string.
-pub fn decode_key_value(kv: KeyValue) -> CommonMetaResult<(String, String)> {
-    let key = String::from_utf8_lossy(&kv.key).to_string();
-    let value = String::from_utf8_lossy(&kv.value).to_string();
-    Ok((key, value))
-}
-
-/// Formats a value as a JSON string.
-pub fn json_fromatter<T>(pretty: bool, value: &T) -> String
-where
-    T: Serialize,
-{
-    if pretty {
-        serde_json::to_string_pretty(value).unwrap()
-    } else {
-        serde_json::to_string(value).unwrap()
-    }
-}
-
-/// Gets the table id by table name.
-pub async fn get_table_id_by_name(
-    table_name_manager: &TableNameManager,
-    catalog_name: &str,
-    schema_name: &str,
-    table_name: &str,
-) -> Result<Option<TableId>, BoxedError> {
-    let table_name_key = TableNameKey::new(catalog_name, schema_name, table_name);
-    let Some(table_name_value) = table_name_manager
-        .get(table_name_key)
-        .await
-        .map_err(BoxedError::new)?
-    else {
-        return Ok(None);
-    };
-    Ok(Some(table_name_value.table_id()))
-}

src/cli/src/metadata/repair.rs

@@ -1,368 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-mod alter_table;
-mod create_table;
-
-use std::sync::Arc;
-use std::time::Duration;
-
-use async_trait::async_trait;
-use clap::Parser;
-use client::api::v1::CreateTableExpr;
-use client::client_manager::NodeClients;
-use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use common_error::ext::{BoxedError, ErrorExt};
-use common_error::status_code::StatusCode;
-use common_grpc::channel_manager::ChannelConfig;
-use common_meta::error::Error as CommonMetaError;
-use common_meta::key::TableMetadataManager;
-use common_meta::kv_backend::KvBackendRef;
-use common_meta::node_manager::NodeManagerRef;
-use common_meta::peer::Peer;
-use common_meta::rpc::router::{find_leaders, RegionRoute};
-use common_telemetry::{error, info, warn};
-use futures::TryStreamExt;
-use snafu::{ensure, ResultExt};
-use store_api::storage::TableId;
-
-use crate::error::{
-    InvalidArgumentsSnafu, Result, SendRequestToDatanodeSnafu, TableMetadataSnafu, UnexpectedSnafu,
-};
-use crate::metadata::common::StoreConfig;
-use crate::metadata::utils::{FullTableMetadata, IteratorInput, TableMetadataIterator};
-use crate::Tool;
-
-/// Repair metadata of logical tables.
-#[derive(Debug, Default, Parser)]
-pub struct RepairLogicalTablesCommand {
-    /// The names of the tables to repair.
-    #[clap(long, value_delimiter = ',', alias = "table-name")]
-    table_names: Vec<String>,
-
-    /// The id of the table to repair.
-    #[clap(long, value_delimiter = ',', alias = "table-id")]
-    table_ids: Vec<TableId>,
-
-    /// The schema of the tables to repair.
-    #[clap(long, default_value = DEFAULT_SCHEMA_NAME)]
-    schema_name: String,
-
-    /// The catalog of the tables to repair.
-    #[clap(long, default_value = DEFAULT_CATALOG_NAME)]
-    catalog_name: String,
-
-    /// Whether to fail fast if any repair operation fails.
-    #[clap(long)]
-    fail_fast: bool,
-
-    #[clap(flatten)]
-    store: StoreConfig,
-
-    /// The timeout for the client to operate the datanode.
-    #[clap(long, default_value_t = 30)]
-    client_timeout_secs: u64,
-
-    /// The timeout for the client to connect to the datanode.
-    #[clap(long, default_value_t = 3)]
-    client_connect_timeout_secs: u64,
-}
-
-impl RepairLogicalTablesCommand {
-    fn validate(&self) -> Result<()> {
-        ensure!(
-            !self.table_names.is_empty() || !self.table_ids.is_empty(),
-            InvalidArgumentsSnafu {
-                msg: "You must specify --table-names or --table-ids.",
-            }
-        );
-        Ok(())
-    }
-}
-
-impl RepairLogicalTablesCommand {
-    pub async fn build(&self) -> std::result::Result<Box<dyn Tool>, BoxedError> {
-        self.validate().map_err(BoxedError::new)?;
-        let kv_backend = self.store.build().await?;
-        let node_client_channel_config = ChannelConfig::new()
-            .timeout(Duration::from_secs(self.client_timeout_secs))
-            .connect_timeout(Duration::from_secs(self.client_connect_timeout_secs));
-        let node_manager = Arc::new(NodeClients::new(node_client_channel_config));
-
-        Ok(Box::new(RepairTool {
-            table_names: self.table_names.clone(),
-            table_ids: self.table_ids.clone(),
-            schema_name: self.schema_name.clone(),
-            catalog_name: self.catalog_name.clone(),
-            fail_fast: self.fail_fast,
-            kv_backend,
-            node_manager,
-        }))
-    }
-}
-
-struct RepairTool {
-    table_names: Vec<String>,
-    table_ids: Vec<TableId>,
-    schema_name: String,
-    catalog_name: String,
-    fail_fast: bool,
-    kv_backend: KvBackendRef,
-    node_manager: NodeManagerRef,
-}
-
-#[async_trait]
-impl Tool for RepairTool {
-    async fn do_work(&self) -> std::result::Result<(), BoxedError> {
-        self.repair_tables().await.map_err(BoxedError::new)
-    }
-}
-
-impl RepairTool {
-    fn generate_iterator_input(&self) -> Result<IteratorInput> {
-        if !self.table_names.is_empty() {
-            let table_names = &self.table_names;
-            let catalog = &self.catalog_name;
-            let schema_name = &self.schema_name;
-
-            let table_names = table_names
-                .iter()
-                .map(|table_name| {
-                    (
-                        catalog.to_string(),
-                        schema_name.to_string(),
-                        table_name.to_string(),
-                    )
-                })
-                .collect::<Vec<_>>();
-            return Ok(IteratorInput::new_table_names(table_names));
-        } else if !self.table_ids.is_empty() {
-            return Ok(IteratorInput::new_table_ids(self.table_ids.clone()));
-        };
-
-        InvalidArgumentsSnafu {
-            msg: "You must specify --table-names or --table-id.",
-        }
-        .fail()
-    }
-
-    async fn repair_tables(&self) -> Result<()> {
-        let input = self.generate_iterator_input()?;
-        let mut table_metadata_iterator =
-            Box::pin(TableMetadataIterator::new(self.kv_backend.clone(), input).into_stream());
-        let table_metadata_manager = TableMetadataManager::new(self.kv_backend.clone());
-
-        let mut skipped_table = 0;
-        let mut success_table = 0;
-        while let Some(full_table_metadata) = table_metadata_iterator.try_next().await? {
-            let full_table_name = full_table_metadata.full_table_name();
-            if !full_table_metadata.is_metric_engine() {
-                warn!(
-                    "Skipping repair for non-metric engine table: {}",
-                    full_table_name
-                );
-                skipped_table += 1;
-                continue;
-            }
-
-            if full_table_metadata.is_physical_table() {
-                warn!("Skipping repair for physical table: {}", full_table_name);
-                skipped_table += 1;
-                continue;
-            }
-
-            let (physical_table_id, physical_table_route) = table_metadata_manager
-                .table_route_manager()
-                .get_physical_table_route(full_table_metadata.table_id)
-                .await
-                .context(TableMetadataSnafu)?;
-
-            if let Err(err) = self
-                .repair_table(
-                    &full_table_metadata,
-                    physical_table_id,
-                    &physical_table_route.region_routes,
-                )
-                .await
-            {
-                error!(
-                    err;
-                    "Failed to repair table: {}, skipped table: {}",
-                    full_table_name,
-                    skipped_table,
-                );
-
-                if self.fail_fast {
-                    return Err(err);
-                }
-            } else {
-                success_table += 1;
-            }
-        }
-
-        info!(
-            "Repair logical tables result: {} tables repaired, {} tables skipped",
-            success_table, skipped_table
-        );
-
-        Ok(())
-    }
-
-    async fn alter_table_on_datanodes(
-        &self,
-        full_table_metadata: &FullTableMetadata,
-        physical_region_routes: &[RegionRoute],
-    ) -> Result<Vec<(Peer, CommonMetaError)>> {
-        let logical_table_id = full_table_metadata.table_id;
-        let alter_table_expr = alter_table::generate_alter_table_expr_for_all_columns(
-            &full_table_metadata.table_info,
-        )?;
-        let node_manager = self.node_manager.clone();
-
-        let mut failed_peers = Vec::new();
-        info!(
-            "Sending alter table requests to all datanodes for table: {}, number of regions:{}.",
-            full_table_metadata.full_table_name(),
-            physical_region_routes.len()
-        );
-        let leaders = find_leaders(physical_region_routes);
-        for peer in &leaders {
-            let alter_table_request = alter_table::make_alter_region_request_for_peer(
-                logical_table_id,
-                &alter_table_expr,
-                peer,
-                physical_region_routes,
-            )?;
-            let datanode = node_manager.datanode(peer).await;
-            if let Err(err) = datanode.handle(alter_table_request).await {
-                failed_peers.push((peer.clone(), err));
-            }
-        }
-
-        Ok(failed_peers)
-    }
-
-    async fn create_table_on_datanode(
-        &self,
-        create_table_expr: &CreateTableExpr,
-        logical_table_id: TableId,
-        physical_table_id: TableId,
-        peer: &Peer,
-        physical_region_routes: &[RegionRoute],
-    ) -> Result<()> {
-        let node_manager = self.node_manager.clone();
-        let datanode = node_manager.datanode(peer).await;
-        let create_table_request = create_table::make_create_region_request_for_peer(
-            logical_table_id,
-            physical_table_id,
-            create_table_expr,
-            peer,
-            physical_region_routes,
-        )?;
-
-        datanode
-            .handle(create_table_request)
-            .await
-            .with_context(|_| SendRequestToDatanodeSnafu { peer: peer.clone() })?;
-
-        Ok(())
-    }
-
-    async fn repair_table(
-        &self,
-        full_table_metadata: &FullTableMetadata,
-        physical_table_id: TableId,
-        physical_region_routes: &[RegionRoute],
-    ) -> Result<()> {
-        let full_table_name = full_table_metadata.full_table_name();
-        // First we sends alter table requests to all datanodes with all columns.
-        let failed_peers = self
-            .alter_table_on_datanodes(full_table_metadata, physical_region_routes)
-            .await?;
-
-        if failed_peers.is_empty() {
-            info!(
-                "All alter table requests sent successfully for table: {}",
-                full_table_name
-            );
-            return Ok(());
-        }
-        warn!(
-            "Sending alter table requests to datanodes for table: {} failed for the datanodes: {:?}",
-            full_table_name,
-            failed_peers.iter().map(|(peer, _)| peer.id).collect::<Vec<_>>()
-        );
-
-        let create_table_expr =
-            create_table::generate_create_table_expr(&full_table_metadata.table_info)?;
-
-        let mut errors = Vec::new();
-        for (peer, err) in failed_peers {
-            if err.status_code() != StatusCode::RegionNotFound {
-                error!(
-                    err;
-                    "Sending alter table requests to datanode: {} for table: {} failed",
-                    peer.id,
-                    full_table_name,
-                );
-                continue;
-            }
-            info!(
-                "Region not found for table: {}, datanode: {}, trying to create the logical table on that datanode",
-                full_table_name,
-                peer.id
-            );
-
-            // If the alter table request fails for any datanode, we attempt to create the table on that datanode
-            // as a fallback mechanism to ensure table consistency across the cluster.
-            if let Err(err) = self
-                .create_table_on_datanode(
-                    &create_table_expr,
-                    full_table_metadata.table_id,
-                    physical_table_id,
-                    &peer,
-                    physical_region_routes,
-                )
-                .await
-            {
-                error!(
-                    err;
-                    "Failed to create table on datanode: {} for table: {}",
-                    peer.id, full_table_name
-                );
-                errors.push(err);
-                if self.fail_fast {
-                    break;
-                }
-            } else {
-                info!(
-                    "Created table on datanode: {} for table: {}",
-                    peer.id, full_table_name
-                );
-            }
-        }
-
-        if !errors.is_empty() {
-            return UnexpectedSnafu {
-                msg: format!(
-                    "Failed to create table on datanodes for table: {}",
-                    full_table_name,
-                ),
-            }
-            .fail();
-        }
-
-        Ok(())
-    }
-}

src/cli/src/metadata/repair/alter_table.rs

@@ -1,84 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use client::api::v1::alter_table_expr::Kind;
-use client::api::v1::region::{region_request, AlterRequests, RegionRequest, RegionRequestHeader};
-use client::api::v1::{AddColumn, AddColumns, AlterTableExpr};
-use common_meta::ddl::alter_logical_tables::make_alter_region_request;
-use common_meta::peer::Peer;
-use common_meta::rpc::router::{find_leader_regions, RegionRoute};
-use operator::expr_helper::column_schemas_to_defs;
-use snafu::ResultExt;
-use store_api::storage::{RegionId, TableId};
-use table::metadata::RawTableInfo;
-
-use crate::error::{CovertColumnSchemasToDefsSnafu, Result};
-
-/// Generates alter table expression for all columns.
-pub fn generate_alter_table_expr_for_all_columns(
-    table_info: &RawTableInfo,
-) -> Result<AlterTableExpr> {
-    let schema = &table_info.meta.schema;
-
-    let mut alter_table_expr = AlterTableExpr {
-        catalog_name: table_info.catalog_name.to_string(),
-        schema_name: table_info.schema_name.to_string(),
-        table_name: table_info.name.to_string(),
-        ..Default::default()
-    };
-
-    let primary_keys = table_info
-        .meta
-        .primary_key_indices
-        .iter()
-        .map(|i| schema.column_schemas[*i].name.clone())
-        .collect::<Vec<_>>();
-
-    let add_columns = column_schemas_to_defs(schema.column_schemas.clone(), &primary_keys)
-        .context(CovertColumnSchemasToDefsSnafu)?;
-
-    alter_table_expr.kind = Some(Kind::AddColumns(AddColumns {
-        add_columns: add_columns
-            .into_iter()
-            .map(|col| AddColumn {
-                column_def: Some(col),
-                location: None,
-                add_if_not_exists: true,
-            })
-            .collect(),
-    }));
-
-    Ok(alter_table_expr)
-}
-
-/// Makes an alter region request for a peer.
-pub fn make_alter_region_request_for_peer(
-    logical_table_id: TableId,
-    alter_table_expr: &AlterTableExpr,
-    peer: &Peer,
-    region_routes: &[RegionRoute],
-) -> Result<RegionRequest> {
-    let regions_on_this_peer = find_leader_regions(region_routes, peer);
-    let mut requests = Vec::with_capacity(regions_on_this_peer.len());
-    for region_number in &regions_on_this_peer {
-        let region_id = RegionId::new(logical_table_id, *region_number);
-        let request = make_alter_region_request(region_id, alter_table_expr);
-        requests.push(request);
-    }
-
-    Ok(RegionRequest {
-        header: Some(RegionRequestHeader::default()),
-        body: Some(region_request::Body::Alters(AlterRequests { requests })),
-    })
-}

src/cli/src/metadata/repair/create_table.rs

@@ -1,89 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-
-use client::api::v1::region::{region_request, CreateRequests, RegionRequest, RegionRequestHeader};
-use client::api::v1::CreateTableExpr;
-use common_meta::ddl::create_logical_tables::create_region_request_builder;
-use common_meta::ddl::utils::region_storage_path;
-use common_meta::peer::Peer;
-use common_meta::rpc::router::{find_leader_regions, RegionRoute};
-use operator::expr_helper::column_schemas_to_defs;
-use snafu::ResultExt;
-use store_api::storage::{RegionId, TableId};
-use table::metadata::RawTableInfo;
-
-use crate::error::{CovertColumnSchemasToDefsSnafu, Result};
-
-/// Generates a `CreateTableExpr` from a `RawTableInfo`.
-pub fn generate_create_table_expr(table_info: &RawTableInfo) -> Result<CreateTableExpr> {
-    let schema = &table_info.meta.schema;
-    let primary_keys = table_info
-        .meta
-        .primary_key_indices
-        .iter()
-        .map(|i| schema.column_schemas[*i].name.clone())
-        .collect::<Vec<_>>();
-
-    let timestamp_index = schema.timestamp_index.as_ref().unwrap();
-    let time_index = schema.column_schemas[*timestamp_index].name.clone();
-    let column_defs = column_schemas_to_defs(schema.column_schemas.clone(), &primary_keys)
-        .context(CovertColumnSchemasToDefsSnafu)?;
-    let table_options = HashMap::from(&table_info.meta.options);
-
-    Ok(CreateTableExpr {
-        catalog_name: table_info.catalog_name.to_string(),
-        schema_name: table_info.schema_name.to_string(),
-        table_name: table_info.name.to_string(),
-        desc: String::default(),
-        column_defs,
-        time_index,
-        primary_keys,
-        create_if_not_exists: true,
-        table_options,
-        table_id: None,
-        engine: table_info.meta.engine.to_string(),
-    })
-}
-
-/// Makes a create region request for a peer.
-pub fn make_create_region_request_for_peer(
-    logical_table_id: TableId,
-    physical_table_id: TableId,
-    create_table_expr: &CreateTableExpr,
-    peer: &Peer,
-    region_routes: &[RegionRoute],
-) -> Result<RegionRequest> {
-    let regions_on_this_peer = find_leader_regions(region_routes, peer);
-    let mut requests = Vec::with_capacity(regions_on_this_peer.len());
-    let request_builder =
-        create_region_request_builder(create_table_expr, physical_table_id).unwrap();
-
-    let catalog = &create_table_expr.catalog_name;
-    let schema = &create_table_expr.schema_name;
-    let storage_path = region_storage_path(catalog, schema);
-
-    for region_number in &regions_on_this_peer {
-        let region_id = RegionId::new(logical_table_id, *region_number);
-        let region_request =
-            request_builder.build_one(region_id, storage_path.clone(), &HashMap::new());
-        requests.push(region_request);
-    }
-
-    Ok(RegionRequest {
-        header: Some(RegionRequestHeader::default()),
-        body: Some(region_request::Body::Creates(CreateRequests { requests })),
-    })
-}

src/cli/src/metadata/snapshot.rs

@@ -1,362 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::path::Path;
-
-use async_trait::async_trait;
-use clap::{Parser, Subcommand};
-use common_base::secrets::{ExposeSecret, SecretString};
-use common_error::ext::BoxedError;
-use common_meta::snapshot::MetadataSnapshotManager;
-use object_store::services::{Fs, S3};
-use object_store::ObjectStore;
-use snafu::{OptionExt, ResultExt};
-
-use crate::error::{InvalidFilePathSnafu, OpenDalSnafu, S3ConfigNotSetSnafu};
-use crate::metadata::common::StoreConfig;
-use crate::Tool;
-
-/// Subcommand for metadata snapshot operations, including saving snapshots, restoring from snapshots, and viewing snapshot information.
-#[derive(Subcommand)]
-pub enum SnapshotCommand {
-    /// Save a snapshot of the current metadata state to a specified location.
-    Save(SaveCommand),
-    /// Restore metadata from a snapshot.
-    Restore(RestoreCommand),
-    /// Explore metadata from a snapshot.
-    Info(InfoCommand),
-}
-
-impl SnapshotCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        match self {
-            SnapshotCommand::Save(cmd) => cmd.build().await,
-            SnapshotCommand::Restore(cmd) => cmd.build().await,
-            SnapshotCommand::Info(cmd) => cmd.build().await,
-        }
-    }
-}
-
-// TODO(qtang): Abstract a generic s3 config for export import meta snapshot restore
-#[derive(Debug, Default, Parser)]
-struct S3Config {
-    /// whether to use s3 as the output directory. default is false.
-    #[clap(long, default_value = "false")]
-    s3: bool,
-    /// The s3 bucket name.
-    #[clap(long)]
-    s3_bucket: Option<String>,
-    /// The s3 region.
-    #[clap(long)]
-    s3_region: Option<String>,
-    /// The s3 access key.
-    #[clap(long)]
-    s3_access_key: Option<SecretString>,
-    /// The s3 secret key.
-    #[clap(long)]
-    s3_secret_key: Option<SecretString>,
-    /// The s3 endpoint. we will automatically use the default s3 decided by the region if not set.
-    #[clap(long)]
-    s3_endpoint: Option<String>,
-}
-
-impl S3Config {
-    pub fn build(&self, root: &str) -> Result<Option<ObjectStore>, BoxedError> {
-        if !self.s3 {
-            Ok(None)
-        } else {
-            if self.s3_region.is_none()
-                || self.s3_access_key.is_none()
-                || self.s3_secret_key.is_none()
-                || self.s3_bucket.is_none()
-            {
-                return S3ConfigNotSetSnafu.fail().map_err(BoxedError::new);
-            }
-            // Safety, unwrap is safe because we have checked the options above.
-            let mut config = S3::default()
-                .bucket(self.s3_bucket.as_ref().unwrap())
-                .region(self.s3_region.as_ref().unwrap())
-                .access_key_id(self.s3_access_key.as_ref().unwrap().expose_secret())
-                .secret_access_key(self.s3_secret_key.as_ref().unwrap().expose_secret());
-
-            if !root.is_empty() && root != "." {
-                config = config.root(root);
-            }
-
-            if let Some(endpoint) = &self.s3_endpoint {
-                config = config.endpoint(endpoint);
-            }
-            Ok(Some(
-                ObjectStore::new(config)
-                    .context(OpenDalSnafu)
-                    .map_err(BoxedError::new)?
-                    .finish(),
-            ))
-        }
-    }
-}
-
-/// Export metadata snapshot tool.
-/// This tool is used to export metadata snapshot from etcd, pg or mysql.
-/// It will dump the metadata snapshot to local file or s3 bucket.
-/// The snapshot file will be in binary format.
-#[derive(Debug, Default, Parser)]
-pub struct SaveCommand {
-    /// The store configuration.
-    #[clap(flatten)]
-    store: StoreConfig,
-    /// The s3 config.
-    #[clap(flatten)]
-    s3_config: S3Config,
-    /// The name of the target snapshot file. we will add the file extension automatically.
-    #[clap(long, default_value = "metadata_snapshot")]
-    file_name: String,
-    /// The directory to store the snapshot file.
-    /// if target output is s3 bucket, this is the root directory in the bucket.
-    /// if target output is local file, this is the local directory.
-    #[clap(long, default_value = "")]
-    output_dir: String,
-}
-
-fn create_local_file_object_store(root: &str) -> Result<ObjectStore, BoxedError> {
-    let root = if root.is_empty() { "." } else { root };
-    let object_store = ObjectStore::new(Fs::default().root(root))
-        .context(OpenDalSnafu)
-        .map_err(BoxedError::new)?
-        .finish();
-    Ok(object_store)
-}
-
-impl SaveCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        let kvbackend = self.store.build().await?;
-        let output_dir = &self.output_dir;
-        let object_store = self.s3_config.build(output_dir).map_err(BoxedError::new)?;
-        if let Some(store) = object_store {
-            let tool = MetaSnapshotTool {
-                inner: MetadataSnapshotManager::new(kvbackend, store),
-                target_file: self.file_name.clone(),
-            };
-            Ok(Box::new(tool))
-        } else {
-            let object_store = create_local_file_object_store(output_dir)?;
-            let tool = MetaSnapshotTool {
-                inner: MetadataSnapshotManager::new(kvbackend, object_store),
-                target_file: self.file_name.clone(),
-            };
-            Ok(Box::new(tool))
-        }
-    }
-}
-
-struct MetaSnapshotTool {
-    inner: MetadataSnapshotManager,
-    target_file: String,
-}
-
-#[async_trait]
-impl Tool for MetaSnapshotTool {
-    async fn do_work(&self) -> std::result::Result<(), BoxedError> {
-        self.inner
-            .dump("", &self.target_file)
-            .await
-            .map_err(BoxedError::new)?;
-        Ok(())
-    }
-}
-
-/// Restore metadata from a snapshot file.
-///
-/// This command restores the metadata state from a previously saved snapshot.
-/// The snapshot can be loaded from either a local file system or an S3 bucket,
-/// depending on the provided configuration.
-#[derive(Debug, Default, Parser)]
-pub struct RestoreCommand {
-    /// The store configuration.
-    #[clap(flatten)]
-    store: StoreConfig,
-    /// The s3 config.
-    #[clap(flatten)]
-    s3_config: S3Config,
-    /// The name of the target snapshot file.
-    #[clap(long, default_value = "metadata_snapshot.metadata.fb")]
-    file_name: String,
-    /// The directory to store the snapshot file.
-    #[clap(long, default_value = ".")]
-    input_dir: String,
-    #[clap(long, default_value = "false")]
-    force: bool,
-}
-
-impl RestoreCommand {
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        let kvbackend = self.store.build().await?;
-        let input_dir = &self.input_dir;
-        let object_store = self.s3_config.build(input_dir).map_err(BoxedError::new)?;
-        if let Some(store) = object_store {
-            let tool = MetaRestoreTool::new(
-                MetadataSnapshotManager::new(kvbackend, store),
-                self.file_name.clone(),
-                self.force,
-            );
-            Ok(Box::new(tool))
-        } else {
-            let object_store = create_local_file_object_store(input_dir)?;
-            let tool = MetaRestoreTool::new(
-                MetadataSnapshotManager::new(kvbackend, object_store),
-                self.file_name.clone(),
-                self.force,
-            );
-            Ok(Box::new(tool))
-        }
-    }
-}
-
-struct MetaRestoreTool {
-    inner: MetadataSnapshotManager,
-    source_file: String,
-    force: bool,
-}
-
-impl MetaRestoreTool {
-    pub fn new(inner: MetadataSnapshotManager, source_file: String, force: bool) -> Self {
-        Self {
-            inner,
-            source_file,
-            force,
-        }
-    }
-}
-
-#[async_trait]
-impl Tool for MetaRestoreTool {
-    async fn do_work(&self) -> std::result::Result<(), BoxedError> {
-        let clean = self
-            .inner
-            .check_target_source_clean()
-            .await
-            .map_err(BoxedError::new)?;
-        if clean {
-            common_telemetry::info!(
-                "The target source is clean, we will restore the metadata snapshot."
-            );
-            self.inner
-                .restore(&self.source_file)
-                .await
-                .map_err(BoxedError::new)?;
-            Ok(())
-        } else if !self.force {
-            common_telemetry::warn!(
-                 "The target source is not clean, if you want to restore the metadata snapshot forcefully, please use --force option."
-             );
-            Ok(())
-        } else {
-            common_telemetry::info!("The target source is not clean, We will restore the metadata snapshot with --force.");
-            self.inner
-                .restore(&self.source_file)
-                .await
-                .map_err(BoxedError::new)?;
-            Ok(())
-        }
-    }
-}
-
-/// Explore metadata from a snapshot file.
-///
-/// This command allows filtering the metadata by a specific key and limiting the number of results.
-/// It prints the filtered metadata to the console.
-#[derive(Debug, Default, Parser)]
-pub struct InfoCommand {
-    /// The s3 config.
-    #[clap(flatten)]
-    s3_config: S3Config,
-    /// The name of the target snapshot file. we will add the file extension automatically.
-    #[clap(long, default_value = "metadata_snapshot")]
-    file_name: String,
-    /// The query string to filter the metadata.
-    #[clap(long, default_value = "*")]
-    inspect_key: String,
-    /// The limit of the metadata to query.
-    #[clap(long)]
-    limit: Option<usize>,
-}
-
-struct MetaInfoTool {
-    inner: ObjectStore,
-    source_file: String,
-    inspect_key: String,
-    limit: Option<usize>,
-}
-
-#[async_trait]
-impl Tool for MetaInfoTool {
-    async fn do_work(&self) -> std::result::Result<(), BoxedError> {
-        let result = MetadataSnapshotManager::info(
-            &self.inner,
-            &self.source_file,
-            &self.inspect_key,
-            self.limit,
-        )
-        .await
-        .map_err(BoxedError::new)?;
-        for item in result {
-            println!("{}", item);
-        }
-        Ok(())
-    }
-}
-
-impl InfoCommand {
-    fn decide_object_store_root_for_local_store(
-        file_path: &str,
-    ) -> Result<(&str, &str), BoxedError> {
-        let path = Path::new(file_path);
-        let parent = path
-            .parent()
-            .and_then(|p| p.to_str())
-            .context(InvalidFilePathSnafu { msg: file_path })
-            .map_err(BoxedError::new)?;
-        let file_name = path
-            .file_name()
-            .and_then(|f| f.to_str())
-            .context(InvalidFilePathSnafu { msg: file_path })
-            .map_err(BoxedError::new)?;
-        let root = if parent.is_empty() { "." } else { parent };
-        Ok((root, file_name))
-    }
-
-    pub async fn build(&self) -> Result<Box<dyn Tool>, BoxedError> {
-        let object_store = self.s3_config.build("").map_err(BoxedError::new)?;
-        if let Some(store) = object_store {
-            let tool = MetaInfoTool {
-                inner: store,
-                source_file: self.file_name.clone(),
-                inspect_key: self.inspect_key.clone(),
-                limit: self.limit,
-            };
-            Ok(Box::new(tool))
-        } else {
-            let (root, file_name) =
-                Self::decide_object_store_root_for_local_store(&self.file_name)?;
-            let object_store = create_local_file_object_store(root)?;
-            let tool = MetaInfoTool {
-                inner: object_store,
-                source_file: file_name.to_string(),
-                inspect_key: self.inspect_key.clone(),
-                limit: self.limit,
-            };
-            Ok(Box::new(tool))
-        }
-    }
-}

src/cli/src/metadata/utils.rs

@@ -1,178 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::VecDeque;
-
-use async_stream::try_stream;
-use common_catalog::consts::METRIC_ENGINE;
-use common_catalog::format_full_table_name;
-use common_meta::key::table_name::TableNameKey;
-use common_meta::key::table_route::TableRouteValue;
-use common_meta::key::TableMetadataManager;
-use common_meta::kv_backend::KvBackendRef;
-use futures::Stream;
-use snafu::{OptionExt, ResultExt};
-use store_api::storage::TableId;
-use table::metadata::RawTableInfo;
-
-use crate::error::{Result, TableMetadataSnafu, UnexpectedSnafu};
-
-/// The input for the iterator.
-pub enum IteratorInput {
-    TableIds(VecDeque<TableId>),
-    TableNames(VecDeque<(String, String, String)>),
-}
-
-impl IteratorInput {
-    /// Creates a new iterator input from a list of table ids.
-    pub fn new_table_ids(table_ids: Vec<TableId>) -> Self {
-        Self::TableIds(table_ids.into())
-    }
-
-    /// Creates a new iterator input from a list of table names.
-    pub fn new_table_names(table_names: Vec<(String, String, String)>) -> Self {
-        Self::TableNames(table_names.into())
-    }
-}
-
-/// An iterator for retrieving table metadata from the metadata store.
-///
-/// This struct provides functionality to iterate over table metadata based on
-/// either [`TableId`] and their associated regions or fully qualified table names.
-pub struct TableMetadataIterator {
-    input: IteratorInput,
-    table_metadata_manager: TableMetadataManager,
-}
-
-/// The full table metadata.
-pub struct FullTableMetadata {
-    pub table_id: TableId,
-    pub table_info: RawTableInfo,
-    pub table_route: TableRouteValue,
-}
-
-impl FullTableMetadata {
-    /// Returns true if it's [TableRouteValue::Physical].
-    pub fn is_physical_table(&self) -> bool {
-        self.table_route.is_physical()
-    }
-
-    /// Returns true if it's a metric engine table.
-    pub fn is_metric_engine(&self) -> bool {
-        self.table_info.meta.engine == METRIC_ENGINE
-    }
-
-    /// Returns the full table name.
-    pub fn full_table_name(&self) -> String {
-        format_full_table_name(
-            &self.table_info.catalog_name,
-            &self.table_info.schema_name,
-            &self.table_info.name,
-        )
-    }
-}
-
-impl TableMetadataIterator {
-    pub fn new(kvbackend: KvBackendRef, input: IteratorInput) -> Self {
-        let table_metadata_manager = TableMetadataManager::new(kvbackend);
-        Self {
-            input,
-            table_metadata_manager,
-        }
-    }
-
-    /// Returns the next table metadata.
-    ///
-    /// This method handles two types of inputs:
-    /// - TableIds: Returns metadata for a specific [`TableId`].
-    /// - TableNames: Returns metadata for a table identified by its full name (catalog.schema.table).
-    ///
-    /// Returns `None` when there are no more tables to process.
-    pub async fn next(&mut self) -> Result<Option<FullTableMetadata>> {
-        match &mut self.input {
-            IteratorInput::TableIds(table_ids) => {
-                if let Some(table_id) = table_ids.pop_front() {
-                    let full_table_metadata = self.get_table_metadata(table_id).await?;
-                    return Ok(Some(full_table_metadata));
-                }
-            }
-
-            IteratorInput::TableNames(table_names) => {
-                if let Some(full_table_name) = table_names.pop_front() {
-                    let table_id = self.get_table_id_by_name(full_table_name).await?;
-                    let full_table_metadata = self.get_table_metadata(table_id).await?;
-                    return Ok(Some(full_table_metadata));
-                }
-            }
-        }
-
-        Ok(None)
-    }
-
-    /// Converts the iterator into a stream of table metadata.
-    pub fn into_stream(mut self) -> impl Stream<Item = Result<FullTableMetadata>> {
-        try_stream!({
-            while let Some(full_table_metadata) = self.next().await? {
-                yield full_table_metadata;
-            }
-        })
-    }
-
-    async fn get_table_id_by_name(
-        &mut self,
-        (catalog_name, schema_name, table_name): (String, String, String),
-    ) -> Result<TableId> {
-        let key = TableNameKey::new(&catalog_name, &schema_name, &table_name);
-        let table_id = self
-            .table_metadata_manager
-            .table_name_manager()
-            .get(key)
-            .await
-            .context(TableMetadataSnafu)?
-            .with_context(|| UnexpectedSnafu {
-                msg: format!(
-                    "Table not found: {}",
-                    format_full_table_name(&catalog_name, &schema_name, &table_name)
-                ),
-            })?
-            .table_id();
-        Ok(table_id)
-    }
-
-    async fn get_table_metadata(&mut self, table_id: TableId) -> Result<FullTableMetadata> {
-        let (table_info, table_route) = self
-            .table_metadata_manager
-            .get_full_table_info(table_id)
-            .await
-            .context(TableMetadataSnafu)?;
-
-        let table_info = table_info
-            .with_context(|| UnexpectedSnafu {
-                msg: format!("Table info not found for table id: {table_id}"),
-            })?
-            .into_inner()
-            .table_info;
-        let table_route = table_route
-            .with_context(|| UnexpectedSnafu {
-                msg: format!("Table route not found for table id: {table_id}"),
-            })?
-            .into_inner();
-
-        Ok(FullTableMetadata {
-            table_id,
-            table_info,
-            table_route,
-        })
-    }
-}

src/client/Cargo.toml

@@ -25,7 +25,6 @@ common-meta.workspace = true
 common-query.workspace = true
 common-recordbatch.workspace = true
 common-telemetry.workspace = true
-datatypes.workspace = true
 enum_dispatch = "0.3"
 futures.workspace = true
 futures-util.workspace = true

src/client/src/client.rs

@@ -162,23 +162,14 @@ impl Client {
             .as_bytes() as usize
     }
 
-    pub fn make_flight_client(
-        &self,
-        send_compression: bool,
-        accept_compression: bool,
-    ) -> Result<FlightClient> {
+    pub fn make_flight_client(&self) -> Result<FlightClient> {
         let (addr, channel) = self.find_channel()?;
 
-        let mut client = FlightServiceClient::new(channel)
+        let client = FlightServiceClient::new(channel)
             .max_decoding_message_size(self.max_grpc_recv_message_size())
-            .max_encoding_message_size(self.max_grpc_send_message_size());
-        // todo(hl): support compression methods.
-        if send_compression {
-            client = client.send_compressed(CompressionEncoding::Zstd);
-        }
-        if accept_compression {
-            client = client.accept_compressed(CompressionEncoding::Zstd);
-        }
+            .max_encoding_message_size(self.max_grpc_send_message_size())
+            .accept_compressed(CompressionEncoding::Zstd)
+            .send_compressed(CompressionEncoding::Zstd);
 
         Ok(FlightClient { addr, client })
     }
@@ -187,7 +178,9 @@ impl Client {
         let (addr, channel) = self.find_channel()?;
         let client = PbRegionClient::new(channel)
             .max_decoding_message_size(self.max_grpc_recv_message_size())
-            .max_encoding_message_size(self.max_grpc_send_message_size());
+            .max_encoding_message_size(self.max_grpc_send_message_size())
+            .accept_compressed(CompressionEncoding::Zstd)
+            .send_compressed(CompressionEncoding::Zstd);
         Ok((addr, client))
     }
 

src/client/src/client_manager.rs

@@ -49,16 +49,7 @@ impl NodeManager for NodeClients {
     async fn datanode(&self, datanode: &Peer) -> DatanodeRef {
         let client = self.get_client(datanode).await;
 
-        let ChannelConfig {
-            send_compression,
-            accept_compression,
-            ..
-        } = self.channel_manager.config();
-        Arc::new(RegionRequester::new(
-            client,
-            *send_compression,
-            *accept_compression,
-        ))
+        Arc::new(RegionRequester::new(client))
     }
 
     async fn flownode(&self, flownode: &Peer) -> FlownodeRef {

src/client/src/database.rs

@@ -14,7 +14,6 @@
 
 use std::pin::Pin;
 use std::str::FromStr;
-use std::sync::Arc;
 
 use api::v1::auth_header::AuthScheme;
 use api::v1::ddl_request::Expr as DdlExpr;
@@ -23,7 +22,7 @@ use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
     AlterTableExpr, AuthHeader, Basic, CreateTableExpr, DdlRequest, GreptimeRequest,
-    InsertRequests, QueryRequest, RequestHeader, RowInsertRequests,
+    InsertRequests, QueryRequest, RequestHeader,
 };
 use arrow_flight::{FlightData, Ticket};
 use async_stream::stream;
@@ -31,26 +30,26 @@ use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
 use common_catalog::build_db_string;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use common_error::ext::BoxedError;
+use common_error::ext::{BoxedError, ErrorExt};
 use common_grpc::flight::do_put::DoPutResponse;
 use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_query::Output;
 use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper};
+use common_recordbatch::RecordBatchStreamWrapper;
 use common_telemetry::tracing_context::W3cTrace;
 use common_telemetry::{error, warn};
 use futures::future;
 use futures_util::{Stream, StreamExt, TryStreamExt};
 use prost::Message;
-use snafu::{ensure, OptionExt, ResultExt};
+use snafu::{ensure, ResultExt};
 use tonic::metadata::{AsciiMetadataKey, AsciiMetadataValue, MetadataMap, MetadataValue};
 use tonic::transport::Channel;
 
 use crate::error::{
     ConvertFlightDataSnafu, Error, FlightGetSnafu, IllegalFlightMessagesSnafu,
-    InvalidTonicMetadataValueSnafu,
+    InvalidTonicMetadataValueSnafu, ServerSnafu,
 };
-use crate::{error, from_grpc_response, Client, Result};
+use crate::{from_grpc_response, Client, Result};
 
 type FlightDataStream = Pin<Box<dyn Stream<Item = FlightData> + Send>>;
 
@@ -118,7 +117,6 @@ impl Database {
         }
     }
 
-    /// Set the catalog for the database client.
     pub fn set_catalog(&mut self, catalog: impl Into<String>) {
         self.catalog = catalog.into();
     }
@@ -131,7 +129,6 @@ impl Database {
         }
     }
 
-    /// Set the schema for the database client.
     pub fn set_schema(&mut self, schema: impl Into<String>) {
         self.schema = schema.into();
     }
@@ -144,24 +141,20 @@ impl Database {
         }
     }
 
-    /// Set the timezone for the database client.
     pub fn set_timezone(&mut self, timezone: impl Into<String>) {
         self.timezone = timezone.into();
     }
 
-    /// Set the auth scheme for the database client.
     pub fn set_auth(&mut self, auth: AuthScheme) {
         self.ctx.auth_header = Some(AuthHeader {
             auth_scheme: Some(auth),
         });
     }
 
-    /// Make an InsertRequests request to the database.
     pub async fn insert(&self, requests: InsertRequests) -> Result<u32> {
         self.handle(Request::Inserts(requests)).await
     }
 
-    /// Make an InsertRequests request to the database with hints.
     pub async fn insert_with_hints(
         &self,
         requests: InsertRequests,
@@ -178,28 +171,6 @@ impl Database {
         from_grpc_response(response)
     }
 
-    /// Make a RowInsertRequests request to the database.
-    pub async fn row_inserts(&self, requests: RowInsertRequests) -> Result<u32> {
-        self.handle(Request::RowInserts(requests)).await
-    }
-
-    /// Make a RowInsertRequests request to the database with hints.
-    pub async fn row_inserts_with_hints(
-        &self,
-        requests: RowInsertRequests,
-        hints: &[(&str, &str)],
-    ) -> Result<u32> {
-        let mut client = make_database_client(&self.client)?.inner;
-        let request = self.to_rpc_request(Request::RowInserts(requests));
-
-        let mut request = tonic::Request::new(request);
-        let metadata = request.metadata_mut();
-        Self::put_hints(metadata, hints)?;
-
-        let response = client.handle(request).await?.into_inner();
-        from_grpc_response(response)
-    }
-
     fn put_hints(metadata: &mut MetadataMap, hints: &[(&str, &str)]) -> Result<()> {
         let Some(value) = hints
             .iter()
@@ -215,7 +186,6 @@ impl Database {
         Ok(())
     }
 
-    /// Make a request to the database.
     pub async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = make_database_client(&self.client)?.inner;
         let request = self.to_rpc_request(request);
@@ -225,22 +195,12 @@ impl Database {
 
     /// Retry if connection fails, max_retries is the max number of retries, so the total wait time
     /// is `max_retries * GRPC_CONN_TIMEOUT`
-    pub async fn handle_with_retry(
-        &self,
-        request: Request,
-        max_retries: u32,
-        hints: &[(&str, &str)],
-    ) -> Result<u32> {
+    pub async fn handle_with_retry(&self, request: Request, max_retries: u32) -> Result<u32> {
         let mut client = make_database_client(&self.client)?.inner;
         let mut retries = 0;
-
         let request = self.to_rpc_request(request);
-
         loop {
-            let mut tonic_request = tonic::Request::new(request.clone());
-            let metadata = tonic_request.metadata_mut();
-            Self::put_hints(metadata, hints)?;
-            let raw_response = client.handle(tonic_request).await;
+            let raw_response = client.handle(request.clone()).await;
             match (raw_response, retries < max_retries) {
                 (Ok(resp), _) => return from_grpc_response(resp.into_inner()),
                 (Err(err), true) => {
@@ -250,18 +210,12 @@ impl Database {
                         retries += 1;
                         warn!("Retrying {} times with error = {:?}", retries, err);
                         continue;
-                    } else {
-                        error!(
-                            err; "Failed to send request to grpc handle, retries = {}, not retryable error, aborting",
-                            retries
-                        );
-                        return Err(err.into());
                     }
                 }
                 (Err(err), false) => {
                     error!(
-                        err; "Failed to send request to grpc handle after {} retries",
-                        retries,
+                        "Failed to send request to grpc handle after {} retries, error = {:?}",
+                        retries, err
                     );
                     return Err(err.into());
                 }
@@ -285,7 +239,6 @@ impl Database {
         }
     }
 
-    /// Executes a SQL query without any hints.
     pub async fn sql<S>(&self, sql: S) -> Result<Output>
     where
         S: AsRef<str>,
@@ -293,7 +246,6 @@ impl Database {
         self.sql_with_hint(sql, &[]).await
     }
 
-    /// Executes a SQL query with optional hints for query optimization.
     pub async fn sql_with_hint<S>(&self, sql: S, hints: &[(&str, &str)]) -> Result<Output>
     where
         S: AsRef<str>,
@@ -304,7 +256,6 @@ impl Database {
         self.do_get(request, hints).await
     }
 
-    /// Executes a logical plan directly without SQL parsing.
     pub async fn logical_plan(&self, logical_plan: Vec<u8>) -> Result<Output> {
         let request = Request::Query(QueryRequest {
             query: Some(Query::LogicalPlan(logical_plan)),
@@ -312,7 +263,6 @@ impl Database {
         self.do_get(request, &[]).await
     }
 
-    /// Creates a new table using the provided table expression.
     pub async fn create(&self, expr: CreateTableExpr) -> Result<Output> {
         let request = Request::Ddl(DdlRequest {
             expr: Some(DdlExpr::CreateTable(expr)),
@@ -320,7 +270,6 @@ impl Database {
         self.do_get(request, &[]).await
     }
 
-    /// Alters an existing table using the provided alter expression.
     pub async fn alter(&self, expr: AlterTableExpr) -> Result<Output> {
         let request = Request::Ddl(DdlRequest {
             expr: Some(DdlExpr::AlterTable(expr)),
@@ -337,21 +286,26 @@ impl Database {
         let mut request = tonic::Request::new(request);
         Self::put_hints(request.metadata_mut(), hints)?;
 
-        let mut client = self.client.make_flight_client(false, false)?;
+        let mut client = self.client.make_flight_client()?;
 
         let response = client.mut_inner().do_get(request).await.or_else(|e| {
             let tonic_code = e.code();
             let e: Error = e.into();
+            let code = e.status_code();
+            let msg = e.to_string();
+            let error =
+                Err(BoxedError::new(ServerSnafu { code, msg }.build())).with_context(|_| {
+                    FlightGetSnafu {
+                        addr: client.addr().to_string(),
+                        tonic_code,
+                    }
+                });
             error!(
                 "Failed to do Flight get, addr: {}, code: {}, source: {:?}",
                 client.addr(),
                 tonic_code,
-                e
+                error
             );
-            let error = Err(BoxedError::new(e)).with_context(|_| FlightGetSnafu {
-                addr: client.addr().to_string(),
-                tonic_code,
-            });
             error
         })?;
 
@@ -361,10 +315,7 @@ impl Database {
         let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
             flight_data
                 .map_err(Error::from)
-                .and_then(|data| decoder.try_decode(&data).context(ConvertFlightDataSnafu))?
-                .context(IllegalFlightMessagesSnafu {
-                    reason: "none message",
-                })
+                .and_then(|data| decoder.try_decode(&data).context(ConvertFlightDataSnafu))
         });
 
         let Some(first_flight_message) = flight_message_stream.next().await else {
@@ -386,30 +337,20 @@ impl Database {
                 );
                 Ok(Output::new_with_affected_rows(rows))
             }
-            FlightMessage::RecordBatch(_) | FlightMessage::Metrics(_) => {
+            FlightMessage::Recordbatch(_) | FlightMessage::Metrics(_) => {
                 IllegalFlightMessagesSnafu {
                     reason: "The first flight message cannot be a RecordBatch or Metrics message",
                 }
                 .fail()
             }
             FlightMessage::Schema(schema) => {
-                let schema = Arc::new(
-                    datatypes::schema::Schema::try_from(schema)
-                        .context(error::ConvertSchemaSnafu)?,
-                );
-                let schema_cloned = schema.clone();
                 let stream = Box::pin(stream!({
                     while let Some(flight_message) = flight_message_stream.next().await {
                         let flight_message = flight_message
                             .map_err(BoxedError::new)
                             .context(ExternalSnafu)?;
                         match flight_message {
-                            FlightMessage::RecordBatch(arrow_batch) => {
-                                yield RecordBatch::try_from_df_record_batch(
-                                    schema_cloned.clone(),
-                                    arrow_batch,
-                                )
-                            }
+                            FlightMessage::Recordbatch(record_batch) => yield Ok(record_batch),
                             FlightMessage::Metrics(_) => {}
                             FlightMessage::AffectedRows(_) | FlightMessage::Schema(_) => {
                                 yield IllegalFlightMessagesSnafu {reason: format!("A Schema message must be succeeded exclusively by a set of RecordBatch messages, flight_message: {:?}", flight_message)}
@@ -457,7 +398,7 @@ impl Database {
             MetadataValue::from_str(db_to_put).context(InvalidTonicMetadataValueSnafu)?,
         );
 
-        let mut client = self.client.make_flight_client(false, false)?;
+        let mut client = self.client.make_flight_client()?;
         let response = client.mut_inner().do_put(request).await?;
         let response = response
             .into_inner()
@@ -484,11 +425,8 @@ mod tests {
 
     use api::v1::auth_header::AuthScheme;
     use api::v1::{AuthHeader, Basic};
-    use common_error::status_code::StatusCode;
-    use tonic::{Code, Status};
 
     use super::*;
-    use crate::error::TonicSnafu;
 
     #[test]
     fn test_flight_ctx() {
@@ -511,19 +449,4 @@ mod tests {
             })
         )
     }
-
-    #[test]
-    fn test_from_tonic_status() {
-        let expected = TonicSnafu {
-            code: StatusCode::Internal,
-            msg: "blabla".to_string(),
-            tonic_code: Code::Internal,
-        }
-        .build();
-
-        let status = Status::new(Code::Internal, "blabla");
-        let actual: Error = status.into();
-
-        assert_eq!(expected.to_string(), actual.to_string());
-    }
 }

src/client/src/error.rs

@@ -14,13 +14,13 @@
 
 use std::any::Any;
 
-use common_error::define_from_tonic_status;
 use common_error::ext::{BoxedError, ErrorExt};
-use common_error::status_code::StatusCode;
+use common_error::status_code::{convert_tonic_code_to_status_code, StatusCode};
+use common_error::{GREPTIME_DB_HEADER_ERROR_CODE, GREPTIME_DB_HEADER_ERROR_MSG};
 use common_macro::stack_trace_debug;
 use snafu::{location, Location, Snafu};
 use tonic::metadata::errors::InvalidMetadataValue;
-use tonic::Code;
+use tonic::{Code, Status};
 
 #[derive(Snafu)]
 #[snafu(visibility(pub))]
@@ -117,22 +117,6 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-
-    #[snafu(display("Failed to convert Schema"))]
-    ConvertSchema {
-        #[snafu(implicit)]
-        location: Location,
-        source: datatypes::error::Error,
-    },
-
-    #[snafu(display("{}", msg))]
-    Tonic {
-        code: StatusCode,
-        msg: String,
-        tonic_code: Code,
-        #[snafu(implicit)]
-        location: Location,
-    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -144,7 +128,7 @@ impl ErrorExt for Error {
             | Error::MissingField { .. }
             | Error::IllegalDatabaseResponse { .. } => StatusCode::Internal,
 
-            Error::Server { code, .. } | Error::Tonic { code, .. } => *code,
+            Error::Server { code, .. } => *code,
             Error::FlightGet { source, .. }
             | Error::RegionServer { source, .. }
             | Error::FlowServer { source, .. } => source.status_code(),
@@ -153,7 +137,6 @@ impl ErrorExt for Error {
             | Error::CreateTlsChannel { source, .. } => source.status_code(),
             Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected,
             Error::InvalidTonicMetadataValue { .. } => StatusCode::InvalidArguments,
-            Error::ConvertSchema { source, .. } => source.status_code(),
         }
     }
 
@@ -162,7 +145,34 @@ impl ErrorExt for Error {
     }
 }
 
-define_from_tonic_status!(Error, Tonic);
+impl From<Status> for Error {
+    fn from(e: Status) -> Self {
+        fn get_metadata_value(e: &Status, key: &str) -> Option<String> {
+            e.metadata()
+                .get(key)
+                .and_then(|v| String::from_utf8(v.as_bytes().to_vec()).ok())
+        }
+
+        let code = get_metadata_value(&e, GREPTIME_DB_HEADER_ERROR_CODE).and_then(|s| {
+            if let Ok(code) = s.parse::<u32>() {
+                StatusCode::from_u32(code)
+            } else {
+                None
+            }
+        });
+        let tonic_code = e.code();
+        let code = code.unwrap_or_else(|| convert_tonic_code_to_status_code(tonic_code));
+
+        let msg = get_metadata_value(&e, GREPTIME_DB_HEADER_ERROR_MSG)
+            .unwrap_or_else(|| e.message().to_string());
+
+        Self::Server {
+            code,
+            msg,
+            location: location!(),
+        }
+    }
+}
 
 impl Error {
     pub fn should_retry(&self) -> bool {

src/client/src/flow.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::flow::{DirtyWindowRequest, DirtyWindowRequests, FlowRequest, FlowResponse};
+use api::v1::flow::{FlowRequest, FlowResponse};
 use api::v1::region::InsertRequests;
 use common_error::ext::BoxedError;
 use common_meta::node_manager::Flownode;
@@ -44,16 +44,6 @@ impl Flownode for FlowRequester {
             .map_err(BoxedError::new)
             .context(common_meta::error::ExternalSnafu)
     }
-
-    async fn handle_mark_window_dirty(
-        &self,
-        req: DirtyWindowRequest,
-    ) -> common_meta::error::Result<FlowResponse> {
-        self.handle_mark_window_dirty(req)
-            .await
-            .map_err(BoxedError::new)
-            .context(common_meta::error::ExternalSnafu)
-    }
 }
 
 impl FlowRequester {
@@ -101,20 +91,4 @@ impl FlowRequester {
             .into_inner();
         Ok(response)
     }
-
-    async fn handle_mark_window_dirty(&self, req: DirtyWindowRequest) -> Result<FlowResponse> {
-        let (addr, mut client) = self.client.raw_flow_client()?;
-        let response = client
-            .handle_mark_dirty_time_window(DirtyWindowRequests {
-                requests: vec![req],
-            })
-            .await
-            .or_else(|e| {
-                let code = e.code();
-                let err: crate::error::Error = e.into();
-                Err(BoxedError::new(err)).context(FlowServerSnafu { addr, code })
-            })?
-            .into_inner();
-        Ok(response)
-    }
 }

src/client/src/region.rs

@@ -21,14 +21,14 @@ use arc_swap::ArcSwapOption;
 use arrow_flight::Ticket;
 use async_stream::stream;
 use async_trait::async_trait;
-use common_error::ext::BoxedError;
+use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_meta::error::{self as meta_error, Result as MetaResult};
 use common_meta::node_manager::Datanode;
 use common_query::request::QueryRequest;
 use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper, SendableRecordBatchStream};
+use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
 use common_telemetry::error;
 use common_telemetry::tracing_context::TracingContext;
 use prost::Message;
@@ -46,8 +46,6 @@ use crate::{metrics, Client, Error};
 #[derive(Debug)]
 pub struct RegionRequester {
     client: Client,
-    send_compression: bool,
-    accept_compression: bool,
 }
 
 #[async_trait]
@@ -57,7 +55,6 @@ impl Datanode for RegionRequester {
             if err.should_retry() {
                 meta_error::Error::RetryLater {
                     source: BoxedError::new(err),
-                    clean_poisons: false,
                 }
             } else {
                 meta_error::Error::External {
@@ -91,34 +88,34 @@ impl Datanode for RegionRequester {
 }
 
 impl RegionRequester {
-    pub fn new(client: Client, send_compression: bool, accept_compression: bool) -> Self {
-        Self {
-            client,
-            send_compression,
-            accept_compression,
-        }
+    pub fn new(client: Client) -> Self {
+        Self { client }
     }
 
     pub async fn do_get_inner(&self, ticket: Ticket) -> Result<SendableRecordBatchStream> {
-        let mut flight_client = self
-            .client
-            .make_flight_client(self.send_compression, self.accept_compression)?;
+        let mut flight_client = self.client.make_flight_client()?;
         let response = flight_client
             .mut_inner()
             .do_get(ticket)
             .await
-            .or_else(|e| {
+            .map_err(|e| {
                 let tonic_code = e.code();
                 let e: error::Error = e.into();
+                let code = e.status_code();
+                let msg = e.to_string();
+                let error = ServerSnafu { code, msg }
+                    .fail::<()>()
+                    .map_err(BoxedError::new)
+                    .with_context(|_| FlightGetSnafu {
+                        tonic_code,
+                        addr: flight_client.addr().to_string(),
+                    })
+                    .unwrap_err();
                 error!(
                     e; "Failed to do Flight get, addr: {}, code: {}",
                     flight_client.addr(),
                     tonic_code
                 );
-                let error = Err(BoxedError::new(e)).with_context(|_| FlightGetSnafu {
-                    addr: flight_client.addr().to_string(),
-                    tonic_code,
-                });
                 error
             })?;
 
@@ -128,10 +125,7 @@ impl RegionRequester {
         let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
             flight_data
                 .map_err(Error::from)
-                .and_then(|data| decoder.try_decode(&data).context(ConvertFlightDataSnafu))?
-                .context(IllegalFlightMessagesSnafu {
-                    reason: "none message",
-                })
+                .and_then(|data| decoder.try_decode(&data).context(ConvertFlightDataSnafu))
         });
 
         let Some(first_flight_message) = flight_message_stream.next().await else {
@@ -152,78 +146,18 @@ impl RegionRequester {
 
         let tracing_context = TracingContext::from_current_span();
 
-        let schema = Arc::new(
-            datatypes::schema::Schema::try_from(schema).context(error::ConvertSchemaSnafu)?,
-        );
-        let schema_cloned = schema.clone();
         let stream = Box::pin(stream!({
             let _span = tracing_context.attach(common_telemetry::tracing::info_span!(
                 "poll_flight_data_stream"
             ));
-
-            let mut buffered_message: Option<FlightMessage> = None;
-            let mut stream_ended = false;
-
-            while !stream_ended {
-                // get the next message from the buffered message or read from the flight message stream
-                let flight_message_item = if let Some(msg) = buffered_message.take() {
-                    Some(Ok(msg))
-                } else {
-                    flight_message_stream.next().await
-                };
-
-                let flight_message = match flight_message_item {
-                    Some(Ok(message)) => message,
-                    Some(Err(e)) => {
-                        yield Err(BoxedError::new(e)).context(ExternalSnafu);
-                        break;
-                    }
-                    None => break,
-                };
+            while let Some(flight_message) = flight_message_stream.next().await {
+                let flight_message = flight_message
+                    .map_err(BoxedError::new)
+                    .context(ExternalSnafu)?;
 
                 match flight_message {
-                    FlightMessage::RecordBatch(record_batch) => {
-                        let result_to_yield = RecordBatch::try_from_df_record_batch(
-                            schema_cloned.clone(),
-                            record_batch,
-                        );
-
-                        // get the next message from the stream. normally it should be a metrics message.
-                        if let Some(next_flight_message_result) = flight_message_stream.next().await
-                        {
-                            match next_flight_message_result {
-                                Ok(FlightMessage::Metrics(s)) => {
-                                    let m = serde_json::from_str(&s).ok().map(Arc::new);
-                                    metrics_ref.swap(m);
-                                }
-                                Ok(FlightMessage::RecordBatch(rb)) => {
-                                    // for some reason it's not a metrics message, so we need to buffer this record batch
-                                    // and yield it in the next iteration.
-                                    buffered_message = Some(FlightMessage::RecordBatch(rb));
-                                }
-                                Ok(_) => {
-                                    yield IllegalFlightMessagesSnafu {
-                                        reason: "A RecordBatch message can only be succeeded by a Metrics message or another RecordBatch message"
-                                    }
-                                    .fail()
-                                    .map_err(BoxedError::new)
-                                    .context(ExternalSnafu);
-                                    break;
-                                }
-                                Err(e) => {
-                                    yield Err(BoxedError::new(e)).context(ExternalSnafu);
-                                    break;
-                                }
-                            }
-                        } else {
-                            // the stream has ended
-                            stream_ended = true;
-                        }
-
-                        yield result_to_yield;
-                    }
+                    FlightMessage::Recordbatch(record_batch) => yield Ok(record_batch),
                     FlightMessage::Metrics(s) => {
-                        // just a branch in case of some metrics message comes after other things.
                         let m = serde_json::from_str(&s).ok().map(Arc::new);
                         metrics_ref.swap(m);
                         break;

src/cmd/Cargo.toml

@@ -10,13 +10,7 @@ name = "greptime"
 path = "src/bin/greptime.rs"
 
 [features]
-default = [
-    "servers/pprof",
-    "servers/mem-prof",
-    "meta-srv/pg_kvbackend",
-    "meta-srv/mysql_kvbackend",
-]
-enterprise = ["common-meta/enterprise", "frontend/enterprise", "meta-srv/enterprise", "catalog/enterprise"]
+default = ["servers/pprof", "servers/mem-prof"]
 tokio-console = ["common-telemetry/tokio-console"]
 
 [lints]
@@ -52,6 +46,7 @@ common-version.workspace = true
 common-wal.workspace = true
 datanode.workspace = true
 datatypes.workspace = true
+either = "1.8"
 etcd-client.workspace = true
 file-engine.workspace = true
 flow.workspace = true
@@ -66,7 +61,6 @@ metric-engine.workspace = true
 mito2.workspace = true
 moka.workspace = true
 nu-ansi-term = "0.46"
-object-store.workspace = true
 plugins.workspace = true
 prometheus.workspace = true
 prost.workspace = true
@@ -80,7 +74,6 @@ servers.workspace = true
 session.workspace = true
 similar-asserts.workspace = true
 snafu.workspace = true
-stat.workspace = true
 store-api.workspace = true
 substrait.workspace = true
 table.workspace = true