### Added Deduplication and Merge Functionality

- Introduced `DedupReader` and `MergeReader` in `src/mito2/src/read/sync/dedup.rs` and `src/mito2/src/read/sync/merge.rs` to handle deduplication and merging of sorted batches.
 ### Enhanced `BulkMemtable` Iteration
 - Updated `BulkMemtable` in `src/mito2/src/memtable/bulk.rs` to support deduplication and merge modes during iteration.
 - Added `BulkIterContext` to manage iteration context.
 ### Testing Enhancements
 - Added comprehensive tests for `BulkMemtable` and `BulkPart` in `src/mito2/src/memtable/bulk.rs` and `src/mito2/src/memtable/bulk/part.rs`.
 ### Code Refactoring
 - Made `BulkPart` and `BulkPartMeta` cloneable in `src/mito2/src/memtable/bulk/part.rs`.
 - Exposed internal test modules for better test coverage in `src/mito2/src/memtable/time_series.rs` and `src/mito2/src/read/merge.rs`.
 ### New Modules
 - Created `sync` module in `src/mito2/src/read.rs` to organize synchronous read operations.

.github/CODEOWNERS

@@ -4,7 +4,7 @@
 
 * @GreptimeTeam/db-approver
 
-## [Module] Database Engine
+## [Module] Databse Engine
 /src/index @zhongzc
 /src/mito2 @evenyag @v0y4g3r @waynexia
 /src/query @evenyag

.github/actions/build-dev-builder-images/action.yml

@@ -41,14 +41,7 @@ runs:
         username: ${{ inputs.dockerhub-image-registry-username }}
         password: ${{ inputs.dockerhub-image-registry-token }}
 
-    - name: Set up qemu for multi-platform builds
-      uses: docker/setup-qemu-action@v3
-      with:
-        platforms: linux/amd64,linux/arm64
-        # The latest version will lead to segmentation fault.
-        image: tonistiigi/binfmt:qemu-v7.0.0-28
-
-    - name: Build and push dev-builder-ubuntu image # Build image for amd64 and arm64 platform.
+    - name: Build and push dev-builder-ubuntu image
       shell: bash
       if: ${{ inputs.build-dev-builder-ubuntu == 'true' }}
       run: |
@@ -59,7 +52,7 @@ runs:
           IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
           DEV_BUILDER_IMAGE_TAG=${{ inputs.version }}
 
-    - name: Build and push dev-builder-centos image # Only build image for amd64 platform.
+    - name: Build and push dev-builder-centos image
       shell: bash
       if: ${{ inputs.build-dev-builder-centos == 'true' }}
       run: |
@@ -76,7 +69,8 @@ runs:
       run: |
         make dev-builder \
           BASE_IMAGE=android \
-          BUILDX_MULTI_PLATFORM_BUILD=amd64 \
           IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
           IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-          DEV_BUILDER_IMAGE_TAG=${{ inputs.version }}
+          DEV_BUILDER_IMAGE_TAG=${{ inputs.version }} && \
+
+        docker push ${{ inputs.dockerhub-image-registry }}/${{ inputs.dockerhub-image-namespace }}/dev-builder-android:${{ inputs.version }}

.github/actions/build-greptime-images/action.yml

@@ -34,8 +34,8 @@ inputs:
     required: true
   push-latest-tag:
     description: Whether to push the latest tag
-    required: true
-    default: 'false'
+    required: false
+    default: 'true'
 runs:
   using: composite
   steps:
@@ -47,11 +47,7 @@ runs:
         password: ${{ inputs.image-registry-password }}
 
     - name: Set up qemu for multi-platform builds
-      uses: docker/setup-qemu-action@v3
-      with:
-        platforms: linux/amd64,linux/arm64
-        # The latest version will lead to segmentation fault.
-        image: tonistiigi/binfmt:qemu-v7.0.0-28
+      uses: docker/setup-qemu-action@v2
 
     - name: Set up buildx
       uses: docker/setup-buildx-action@v2

.github/actions/build-images/action.yml

@@ -22,8 +22,8 @@ inputs:
     required: true
   push-latest-tag:
     description: Whether to push the latest tag
-    required: true
-    default: 'false'
+    required: false
+    default: 'true'
   dev-mode:
     description: Enable dev mode, only build standard greptime
     required: false

.github/actions/build-linux-artifacts/action.yml

@@ -52,7 +52,7 @@ runs:
       uses: ./.github/actions/build-greptime-binary
       with:
         base-image: ubuntu
-        features: servers/dashboard
+        features: servers/dashboard,pg_kvbackend
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
         version: ${{ inputs.version }}
@@ -70,7 +70,7 @@ runs:
       if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Builds greptime for centos if the host machine is amd64.
       with:
         base-image: centos
-        features: servers/dashboard
+        features: servers/dashboard,pg_kvbackend
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
         version: ${{ inputs.version }}

.github/actions/build-windows-artifacts/action.yml

@@ -47,6 +47,7 @@ runs:
       shell: pwsh
       run: make test sqlness-test
       env:
+        RUSTUP_WINDOWS_PATH_ADD_BIN: 1 # Workaround for https://github.com/nextest-rs/nextest/issues/1493
         RUST_BACKTRACE: 1
         SQLNESS_OPTS: "--preserve-state"
 

.github/actions/release-cn-artifacts/action.yaml

@@ -51,8 +51,8 @@ inputs:
     required: true
   upload-to-s3:
     description: Upload to S3
-    required: true
-    default: 'false'
+    required: false
+    default: 'true'
   artifacts-dir:
     description: Directory to store artifacts
     required: false
@@ -64,11 +64,11 @@ inputs:
   upload-max-retry-times:
     description: Max retry times for uploading artifacts to S3
     required: false
-    default: "30"
+    default: "20"
   upload-retry-timeout:
     description: Timeout for uploading artifacts to S3
     required: false
-    default: "120" # minutes
+    default: "30" # minutes
 runs:
   using: composite
   steps:
@@ -77,21 +77,13 @@ runs:
       with:
         path: ${{ inputs.artifacts-dir }}
 
-    - name: Install s5cmd
-      shell: bash
-      run: |
-        wget https://github.com/peak/s5cmd/releases/download/v2.3.0/s5cmd_2.3.0_Linux-64bit.tar.gz
-        tar -xzf s5cmd_2.3.0_Linux-64bit.tar.gz
-        sudo mv s5cmd /usr/local/bin/
-        sudo chmod +x /usr/local/bin/s5cmd
-
     - name: Release artifacts to cn region
       uses: nick-invision/retry@v2
       if: ${{ inputs.upload-to-s3 == 'true' }}
       env:
         AWS_ACCESS_KEY_ID: ${{ inputs.aws-cn-access-key-id }}
         AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-cn-secret-access-key }}
-        AWS_REGION: ${{ inputs.aws-cn-region }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-cn-region }}
         UPDATE_VERSION_INFO: ${{ inputs.update-version-info }}
       with:
         max_attempts: ${{ inputs.upload-max-retry-times }}

.github/actions/setup-greptimedb-cluster/action.yml

@@ -8,15 +8,15 @@ inputs:
     default: 2
     description: "Number of Datanode replicas"
   meta-replicas:
-    default: 2
+    default: 1
     description: "Number of Metasrv replicas"
-  image-registry:
+  image-registry: 
     default: "docker.io"
     description: "Image registry"
-  image-repository:
+  image-repository: 
     default: "greptime/greptimedb"
     description: "Image repository"
-  image-tag:
+  image-tag: 
     default: "latest"
     description: 'Image tag'
   etcd-endpoints:
@@ -32,12 +32,12 @@ runs:
   steps:
   - name: Install GreptimeDB operator
     uses: nick-fields/retry@v3
-    with:
+    with: 
       timeout_minutes: 3
       max_attempts: 3
       shell: bash
       command: |
-        helm repo add greptime https://greptimeteam.github.io/helm-charts/
+        helm repo add greptime https://greptimeteam.github.io/helm-charts/ 
         helm repo update
         helm upgrade \
           --install \
@@ -48,10 +48,10 @@ runs:
           --wait-for-jobs
   - name: Install GreptimeDB cluster
     shell: bash
-    run: |
+    run: | 
       helm upgrade \
         --install my-greptimedb \
-        --set meta.backendStorage.etcd.endpoints=${{ inputs.etcd-endpoints }} \
+        --set meta.etcdEndpoints=${{ inputs.etcd-endpoints }} \
         --set meta.enableRegionFailover=${{ inputs.enable-region-failover }} \
         --set image.registry=${{ inputs.image-registry }} \
         --set image.repository=${{ inputs.image-repository }}  \
@@ -59,7 +59,7 @@ runs:
         --set base.podTemplate.main.resources.requests.cpu=50m \
         --set base.podTemplate.main.resources.requests.memory=256Mi \
         --set base.podTemplate.main.resources.limits.cpu=2000m \
-        --set base.podTemplate.main.resources.limits.memory=3Gi \
+        --set base.podTemplate.main.resources.limits.memory=2Gi \
         --set frontend.replicas=${{ inputs.frontend-replicas }} \
         --set datanode.replicas=${{ inputs.datanode-replicas }} \
         --set meta.replicas=${{ inputs.meta-replicas }} \
@@ -72,7 +72,7 @@ runs:
   - name: Wait for GreptimeDB
     shell: bash
     run: |
-      while true; do
+      while true; do 
         PHASE=$(kubectl -n my-greptimedb get gtc my-greptimedb -o jsonpath='{.status.clusterPhase}')
         if [ "$PHASE" == "Running" ]; then
           echo "Cluster is ready"
@@ -86,10 +86,10 @@ runs:
   - name: Print GreptimeDB info
     if: always()
     shell: bash
-    run: |
+    run: | 
       kubectl get all --show-labels -n my-greptimedb
   - name: Describe Nodes
     if: always()
     shell: bash
-    run: |
+    run: | 
       kubectl describe nodes

.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml

@@ -2,14 +2,13 @@ meta:
   configData: |-
     [runtime]
     global_rt_size = 4
-
+    
     [wal]
     provider = "kafka"
     broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"]
     num_topics = 3
-    auto_prune_interval = "30s"
-    trigger_flush_threshold = 100
 
+        
     [datanode]
     [datanode.client]
     timeout = "120s"
@@ -22,7 +21,7 @@ datanode:
     [wal]
     provider = "kafka"
     broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"]
-    overwrite_entry_start_id = true
+    linger = "2ms"
 frontend:
   configData: |-
     [runtime]

.github/actions/start-runner/action.yml

@@ -56,7 +56,7 @@ runs:
 
     - name: Start EC2 runner
       if: startsWith(inputs.runner, 'ec2')
-      uses: machulav/ec2-github-runner@v2.3.8
+      uses: machulav/ec2-github-runner@v2
       id: start-linux-arm64-ec2-runner
       with:
         mode: start

.github/actions/stop-runner/action.yml

@@ -33,7 +33,7 @@ runs:
 
     - name: Stop EC2 runner
       if: ${{ inputs.label && inputs.ec2-instance-id }}
-      uses: machulav/ec2-github-runner@v2.3.8
+      uses: machulav/ec2-github-runner@v2
       with:
         mode: stop
         label: ${{ inputs.label }}

.github/labeler.yaml

@@ -1,15 +0,0 @@
-ci:
-  - changed-files:
-      - any-glob-to-any-file: .github/**
-
-docker:
-  - changed-files:
-      - any-glob-to-any-file: docker/**
-
-documentation:
-  - changed-files:
-      - any-glob-to-any-file: docs/**
-
-dashboard:
-  - changed-files:
-      - any-glob-to-any-file: grafana/**

.github/scripts/check-version.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Get current version
-CURRENT_VERSION=$1
-if [ -z "$CURRENT_VERSION" ]; then
-  echo "Error: Failed to get current version"
-  exit 1
-fi
-
-# Get the latest version from GitHub Releases
-API_RESPONSE=$(curl -s "https://api.github.com/repos/GreptimeTeam/greptimedb/releases/latest")
-
-if [ -z "$API_RESPONSE" ] || [ "$(echo "$API_RESPONSE" | jq -r '.message')" = "Not Found" ]; then
-  echo "Error: Failed to fetch latest version from GitHub"
-  exit 1
-fi
-
-# Get the latest version
-LATEST_VERSION=$(echo "$API_RESPONSE" | jq -r '.tag_name')
-
-if [ -z "$LATEST_VERSION" ] || [ "$LATEST_VERSION" = "null" ]; then
-  echo "Error: No valid version found in GitHub releases"
-  exit 1
-fi
-
-# Cleaned up version number format (removed possible 'v' prefix and -nightly suffix)
-CLEAN_CURRENT=$(echo "$CURRENT_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//')
-CLEAN_LATEST=$(echo "$LATEST_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//')
-
-echo "Current version: $CLEAN_CURRENT"
-echo "Latest release version: $CLEAN_LATEST"
-
-# Use sort -V to compare versions
-HIGHER_VERSION=$(printf "%s\n%s" "$CLEAN_CURRENT" "$CLEAN_LATEST" | sort -V | tail -n1)
-
-if [ "$HIGHER_VERSION" = "$CLEAN_CURRENT" ]; then
-  echo "Current version ($CLEAN_CURRENT) is NEWER than or EQUAL to latest ($CLEAN_LATEST)"
-  echo "should-push-latest-tag=true" >> $GITHUB_OUTPUT
-else
-  echo "Current version ($CLEAN_CURRENT) is OLDER than latest ($CLEAN_LATEST)"
-  echo "should-push-latest-tag=false" >> $GITHUB_OUTPUT
-fi

.github/scripts/create-version.sh

@@ -8,25 +8,24 @@ set -e
 # - If it's a nightly build, the version is 'nightly-YYYYMMDD-$(git rev-parse --short HEAD)', like 'nightly-20230712-e5b243c'.
 # create_version ${GIHUB_EVENT_NAME} ${NEXT_RELEASE_VERSION} ${NIGHTLY_RELEASE_PREFIX}
 function create_version() {
-  # Read from environment variables.
+  # Read from envrionment variables.
   if [ -z "$GITHUB_EVENT_NAME" ]; then
-      echo "GITHUB_EVENT_NAME is empty" >&2
+      echo "GITHUB_EVENT_NAME is empty"
       exit 1
   fi
 
   if [ -z "$NEXT_RELEASE_VERSION" ]; then
-      echo "NEXT_RELEASE_VERSION is empty, use version from Cargo.toml" >&2
-      # NOTE: Need a `v` prefix for the version string.
-      export NEXT_RELEASE_VERSION=v$(grep '^version = ' Cargo.toml | cut -d '"' -f 2 | head -n 1)
+      echo "NEXT_RELEASE_VERSION is empty"
+      exit 1
   fi
 
   if [ -z "$NIGHTLY_RELEASE_PREFIX" ]; then
-      echo "NIGHTLY_RELEASE_PREFIX is empty" >&2
+      echo "NIGHTLY_RELEASE_PREFIX is empty"
       exit 1
   fi
 
   # Reuse $NEXT_RELEASE_VERSION to identify whether it's a nightly build.
-  # It will be like 'nightly-20230808-7d0d8dc6'.
+  # It will be like 'nigtly-20230808-7d0d8dc6'.
   if [ "$NEXT_RELEASE_VERSION" = nightly ]; then
     echo "$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")-$(git rev-parse --short HEAD)"
     exit 0
@@ -36,7 +35,7 @@ function create_version() {
   # It will be like 'dev-2023080819-f0e7216c'.
   if [ "$NEXT_RELEASE_VERSION" = dev ]; then
     if [ -z "$COMMIT_SHA" ]; then
-      echo "COMMIT_SHA is empty in dev build" >&2
+      echo "COMMIT_SHA is empty in dev build"
       exit 1
     fi
     echo "dev-$(date "+%Y%m%d-%s")-$(echo "$COMMIT_SHA" | cut -c1-8)"
@@ -46,7 +45,7 @@ function create_version() {
   # Note: Only output 'version=xxx' to stdout when everything is ok, so that it can be used in GitHub Actions Outputs.
   if [ "$GITHUB_EVENT_NAME" = push ]; then
     if [ -z "$GITHUB_REF_NAME" ]; then
-      echo "GITHUB_REF_NAME is empty in push event" >&2
+      echo "GITHUB_REF_NAME is empty in push event"
       exit 1
     fi
     echo "$GITHUB_REF_NAME"
@@ -55,15 +54,15 @@ function create_version() {
   elif [ "$GITHUB_EVENT_NAME" = schedule ]; then
     echo "$NEXT_RELEASE_VERSION-$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")"
   else
-    echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME" >&2
+    echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME"
     exit 1
   fi
 }
 
 # You can run as following examples:
-#  GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nightly GITHUB_REF_NAME=v0.3.0 ./create-version.sh
-#  GITHUB_EVENT_NAME=workflow_dispatch NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nightly ./create-version.sh
-#  GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nightly ./create-version.sh
-#  GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=nightly NIGHTLY_RELEASE_PREFIX=nightly ./create-version.sh
-#  GITHUB_EVENT_NAME=workflow_dispatch COMMIT_SHA=f0e7216c4bb6acce9b29a21ec2d683be2e3f984a NEXT_RELEASE_VERSION=dev NIGHTLY_RELEASE_PREFIX=nightly ./create-version.sh
+#  GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly GITHUB_REF_NAME=v0.3.0 ./create-version.sh
+#  GITHUB_EVENT_NAME=workflow_dispatch NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=nightly NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=workflow_dispatch COMMIT_SHA=f0e7216c4bb6acce9b29a21ec2d683be2e3f984a NEXT_RELEASE_VERSION=dev NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
 create_version

.github/scripts/deploy-greptimedb.sh

@@ -10,7 +10,7 @@ GREPTIMEDB_IMAGE_TAG=${GREPTIMEDB_IMAGE_TAG:-latest}
 ETCD_CHART="oci://registry-1.docker.io/bitnamicharts/etcd"
 GREPTIME_CHART="https://greptimeteam.github.io/helm-charts/"
 
-# Create a cluster with 1 control-plane node and 5 workers.
+# Ceate a cluster with 1 control-plane node and 5 workers.
 function create_kind_cluster() {
   cat <<EOF | kind create cluster --name "${CLUSTER}" --image kindest/node:"$KUBERNETES_VERSION" --config=-
 kind: Cluster
@@ -68,7 +68,7 @@ function deploy_greptimedb_cluster() {
 
   helm install "$cluster_name" greptime/greptimedb-cluster \
     --set image.tag="$GREPTIMEDB_IMAGE_TAG" \
-    --set meta.backendStorage.etcd.endpoints="etcd.$install_namespace:2379" \
+    --set meta.etcdEndpoints="etcd.$install_namespace:2379" \
     -n "$install_namespace"
 
   # Wait for greptimedb cluster to be ready.
@@ -103,7 +103,7 @@ function deploy_greptimedb_cluster_with_s3_storage() {
 
   helm install "$cluster_name" greptime/greptimedb-cluster -n "$install_namespace" \
     --set image.tag="$GREPTIMEDB_IMAGE_TAG" \
-    --set meta.backendStorage.etcd.endpoints="etcd.$install_namespace:2379" \
+    --set meta.etcdEndpoints="etcd.$install_namespace:2379" \
     --set storage.s3.bucket="$AWS_CI_TEST_BUCKET" \
     --set storage.s3.region="$AWS_REGION" \
     --set storage.s3.root="$DATA_ROOT" \

.github/scripts/update-dev-builder-version.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-
-DEV_BUILDER_IMAGE_TAG=$1
-
-update_dev_builder_version() {
-  if [ -z "$DEV_BUILDER_IMAGE_TAG" ]; then
-    echo "Error: Should specify the dev-builder image tag"
-    exit 1
-  fi
-
-  # Configure Git configs.
-  git config --global user.email greptimedb-ci@greptime.com
-  git config --global user.name greptimedb-ci
-
-  # Checkout a new branch.
-  BRANCH_NAME="ci/update-dev-builder-$(date +%Y%m%d%H%M%S)"
-  git checkout -b $BRANCH_NAME
-
-  # Update the dev-builder image tag in the Makefile.
-  sed -i "s/DEV_BUILDER_IMAGE_TAG ?=.*/DEV_BUILDER_IMAGE_TAG ?= ${DEV_BUILDER_IMAGE_TAG}/g" Makefile
-
-  # Commit the changes.
-  git add Makefile
-  git commit -m "ci: update dev-builder image tag"
-  git push origin $BRANCH_NAME
-
-  # Create a Pull Request.
-  gh pr create \
-      --title "ci: update dev-builder image tag" \
-      --body "This PR updates the dev-builder image tag" \
-      --base main \
-      --head $BRANCH_NAME \
-      --reviewer zyy17 \
-      --reviewer daviderli614
-}
-
-update_dev_builder_version

.github/scripts/update-helm-charts-version.sh

@@ -1,46 +0,0 @@
-#!/bin/bash
-
-set -e
-
-VERSION=${VERSION}
-GITHUB_TOKEN=${GITHUB_TOKEN}
-
-update_helm_charts_version() {
-  # Configure Git configs.
-  git config --global user.email update-helm-charts-version@greptime.com
-  git config --global user.name update-helm-charts-version
-
-  # Clone helm-charts repository.
-  git clone "https://x-access-token:${GITHUB_TOKEN}@github.com/GreptimeTeam/helm-charts.git"
-  cd helm-charts
-
-  # Set default remote for gh CLI
-  gh repo set-default GreptimeTeam/helm-charts
-
-  # Checkout a new branch.
-  BRANCH_NAME="chore/greptimedb-${VERSION}"
-  git checkout -b $BRANCH_NAME
-
-  # Update version.
-  make update-version CHART=greptimedb-cluster VERSION=${VERSION}
-  make update-version CHART=greptimedb-standalone VERSION=${VERSION}
-
-  # Update docs.
-  make docs
-
-  # Commit the changes.
-  git add .
-  git commit -s -m "chore: Update GreptimeDB version to ${VERSION}"
-  git push origin $BRANCH_NAME
-
-  # Create a Pull Request.
-  gh pr create \
-    --title "chore: Update GreptimeDB version to ${VERSION}" \
-    --body "This PR updates the GreptimeDB version." \
-    --base main \
-    --head $BRANCH_NAME \
-    --reviewer zyy17 \
-    --reviewer daviderli614
-}
-
-update_helm_charts_version

.github/scripts/update-homebrew-greptme-version.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-set -e
-
-VERSION=${VERSION}
-GITHUB_TOKEN=${GITHUB_TOKEN}
-
-update_homebrew_greptime_version() {
-  # Configure Git configs.
-  git config --global user.email update-greptime-version@greptime.com
-  git config --global user.name update-greptime-version
-
-  # Clone helm-charts repository.
-  git clone "https://x-access-token:${GITHUB_TOKEN}@github.com/GreptimeTeam/homebrew-greptime.git"
-  cd homebrew-greptime
-
-  # Set default remote for gh CLI
-  gh repo set-default GreptimeTeam/homebrew-greptime
-
-  # Checkout a new branch.
-  BRANCH_NAME="chore/greptimedb-${VERSION}"
-  git checkout -b $BRANCH_NAME
-
-  # Update version.
-  make update-greptime-version VERSION=${VERSION}
-
-  # Commit the changes.
-  git add .
-  git commit -s -m "chore: Update GreptimeDB version to ${VERSION}"
-  git push origin $BRANCH_NAME
-
-  # Create a Pull Request.
-  gh pr create \
-    --title "chore: Update GreptimeDB version to ${VERSION}" \
-    --body "This PR updates the GreptimeDB version." \
-    --base main \
-    --head $BRANCH_NAME \
-    --reviewer zyy17 \
-    --reviewer daviderli614
-}
-
-update_homebrew_greptime_version

.github/scripts/upload-artifacts-to-s3.sh

@@ -33,7 +33,7 @@ function upload_artifacts() {
   #    ├── greptime-darwin-amd64-v0.2.0.sha256sum
   #    └── greptime-darwin-amd64-v0.2.0.tar.gz
   find "$ARTIFACTS_DIR" -type f \( -name "*.tar.gz" -o -name "*.sha256sum" \) | while IFS= read -r file; do
-    s5cmd cp \
+    aws s3 cp \
       "$file" "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/$VERSION/$(basename "$file")"
   done
 }
@@ -41,11 +41,11 @@ function upload_artifacts() {
 # Updates the latest version information in AWS S3 if UPDATE_VERSION_INFO is true.
 function update_version_info() {
   if [ "$UPDATE_VERSION_INFO" == "true" ]; then
-    # If it's the official release(like v1.0.0, v1.0.1, v1.0.2, etc.), update latest-version.txt.
+    # If it's the officail release(like v1.0.0, v1.0.1, v1.0.2, etc.), update latest-version.txt.
     if [[ "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
       echo "Updating latest-version.txt"
       echo "$VERSION" > latest-version.txt
-      s5cmd cp \
+      aws s3 cp \
         latest-version.txt "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/latest-version.txt"
     fi
 
@@ -53,7 +53,7 @@ function update_version_info() {
     if [[ "$VERSION" == *"nightly"* ]]; then
       echo "Updating latest-nightly-version.txt"
       echo "$VERSION" > latest-nightly-version.txt
-      s5cmd cp \
+      aws s3 cp \
         latest-nightly-version.txt "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/latest-nightly-version.txt"
     fi
   fi

.github/workflows/apidoc.yml

@@ -14,11 +14,9 @@ name: Build API docs
 
 jobs:
   apidoc:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v4
-      with:
-        persist-credentials: false
     - uses: arduino/setup-protoc@v3
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/dependency-check.yml

@@ -12,8 +12,6 @@ jobs:
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
-      with:
-        persist-credentials: false
 
     - name: Set up Rust
       uses: actions-rust-lang/setup-rust-toolchain@v1

.github/workflows/dev-build.yml

@@ -16,11 +16,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-22.04
-          - ubuntu-22.04-8-cores
-          - ubuntu-22.04-16-cores
-          - ubuntu-22.04-32-cores
-          - ubuntu-22.04-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -55,11 +55,6 @@ on:
         description: Build and push images to DockerHub and ACR
         required: false
         default: true
-      upload_artifacts_to_s3:
-        type: boolean
-        description: Whether upload artifacts to s3
-        required: false
-        default: false
       cargo_profile:
         type: choice
         description: The cargo profile to use in building GreptimeDB.
@@ -81,14 +76,20 @@ env:
 
   NIGHTLY_RELEASE_PREFIX: nightly
 
+  # Use the different image name to avoid conflict with the release images.
+  IMAGE_NAME: greptimedb-dev
+
   # The source code will check out in the following path: '${WORKING_DIR}/dev/greptime'.
   CHECKOUT_GREPTIMEDB_PATH: dev/greptimedb
 
+permissions:
+  issues: write
+
 jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -106,7 +107,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Create version
         id: create-version
@@ -161,7 +161,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Checkout greptimedb
         uses: actions/checkout@v4
@@ -169,7 +168,6 @@ jobs:
           repository: ${{ inputs.repository }}
           ref: ${{ inputs.commit }}
           path: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
-          persist-credentials: true
 
       - uses: ./.github/actions/build-linux-artifacts
         with:
@@ -194,7 +192,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Checkout greptimedb
         uses: actions/checkout@v4
@@ -202,7 +199,6 @@ jobs:
           repository: ${{ inputs.repository }}
           ref: ${{ inputs.commit }}
           path: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
-          persist-credentials: true
 
       - uses: ./.github/actions/build-linux-artifacts
         with:
@@ -223,34 +219,26 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       build-result: ${{ steps.set-build-result.outputs.build-result }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Build and push images to dockerhub
         uses: ./.github/actions/build-images
         with:
           image-registry: docker.io
           image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-name: ${{ vars.DEV_BUILD_IMAGE_NAME }}
+          image-name: ${{ env.IMAGE_NAME }}
           image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
           push-latest-tag: false # Don't push the latest tag to registry.
           dev-mode: true # Only build the standard images.
 
-      - name: Echo Docker image tag to step summary
-        run: |
-          echo "## Docker Image Tag" >> $GITHUB_STEP_SUMMARY
-          echo "Image Tag: \`${{ needs.allocate-runners.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "Full Image Name: \`docker.io/${{ vars.IMAGE_NAMESPACE }}/${{ vars.DEV_BUILD_IMAGE_NAME }}:${{ needs.allocate-runners.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "Pull Command: \`docker pull docker.io/${{ vars.IMAGE_NAMESPACE }}/${{ vars.DEV_BUILD_IMAGE_NAME }}:${{ needs.allocate-runners.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY
-
       - name: Set build result
         id: set-build-result
         run: |
@@ -263,20 +251,19 @@ jobs:
       allocate-runners,
       release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Release artifacts to CN region
         uses: ./.github/actions/release-cn-artifacts
         with:
           src-image-registry: docker.io
           src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          src-image-name: ${{ vars.DEV_BUILD_IMAGE_NAME }}
+          src-image-name: ${{ env.IMAGE_NAME }}
           dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
           dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
           dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
@@ -286,7 +273,6 @@ jobs:
           aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: ${{ inputs.upload_artifacts_to_s3 }}
           dev-mode: true                     # Only build the standard images(exclude centos images).
           push-latest-tag: false             # Don't push the latest tag to registry.
           update-version-info: false         # Don't update the version info in S3.
@@ -295,7 +281,7 @@ jobs:
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -305,7 +291,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Stop EC2 runner
         uses: ./.github/actions/stop-runner
@@ -321,7 +306,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -331,7 +316,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Stop EC2 runner
         uses: ./.github/actions/stop-runner
@@ -349,17 +333,11 @@ jobs:
     needs: [
       release-images-to-dockerhub
     ]
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-
+    runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - name: Report CI status
         id: report-ci-status

.github/workflows/develop.yml

@@ -22,13 +22,10 @@ concurrency:
 
 jobs:
   check-typos-and-docs:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Check typos and docs
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: crate-ci/typos@master
       - name: Check the config docs
         run: |
@@ -37,27 +34,21 @@ jobs:
           || (echo "'config/config.md' is not up-to-date, please run 'make config-docs'." && exit 1)
 
   license-header-check:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     name: Check License Header
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: korandoru/hawkeye@v5
 
   check:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Check
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-20.04 ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -74,14 +65,11 @@ jobs:
         run: cargo check --locked --workspace --all-targets
 
   toml:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Toml Check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: actions-rust-lang/setup-rust-toolchain@v1
       - name: Install taplo
         run: cargo +stable install taplo-cli --version ^0.9 --locked --force
@@ -89,17 +77,14 @@ jobs:
         run: taplo format --check
 
   build:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Build GreptimeDB binaries
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-20.04 ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -132,7 +117,6 @@ jobs:
           version: current
 
   fuzztest:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Fuzz Test
     needs: build
     runs-on: ubuntu-latest
@@ -155,8 +139,6 @@ jobs:
           echo "Disk space after:"
           df -h
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -189,13 +171,11 @@ jobs:
           max-total-time: 120
 
   unstable-fuzztest:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Unstable Fuzz Test
     needs: build-greptime-ci
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
-      fail-fast: false
       matrix:
         target: [ "unstable_fuzz_create_table_standalone" ]
     steps:
@@ -212,8 +192,6 @@ jobs:
           echo "Disk space after:"
           df -h
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -223,12 +201,12 @@ jobs:
         run: |
           sudo apt update && sudo apt install -y libfuzzer-14-dev
           cargo install cargo-fuzz cargo-gc-bin --force
-      - name: Download pre-built binary
+      - name: Download pre-built binariy
         uses: actions/download-artifact@v4
         with:
           name: bin
           path: .
-      - name: Unzip binary
+      - name: Unzip bianry
         run: |
           tar -xvf ./bin.tar.gz
           rm ./bin.tar.gz
@@ -250,24 +228,16 @@ jobs:
           name: unstable-fuzz-logs
           path: /tmp/unstable-greptime/
           retention-days: 3
-      - name: Describe pods
-        if: failure()
-        shell: bash
-        run: |
-          kubectl describe pod -n my-greptimedb
 
   build-greptime-ci:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Build GreptimeDB binary (profile-CI)
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-20.04 ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -281,7 +251,7 @@ jobs:
       - name: Install cargo-gc-bin
         shell: bash
         run: cargo install cargo-gc-bin --force
-      - name: Build greptime binary
+      - name: Build greptime bianry
         shell: bash
         # `cargo gc` will invoke `cargo build` with specified args
         run: cargo gc --profile ci -- --bin greptime --features pg_kvbackend
@@ -299,13 +269,11 @@ jobs:
           version: current
 
   distributed-fuzztest:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Fuzz Test (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
     runs-on: ubuntu-latest
     needs:  build-greptime-ci
     timeout-minutes: 60
     strategy:
-      fail-fast: false
       matrix:
         target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
         mode:
@@ -327,17 +295,15 @@ jobs:
           echo "Disk space after:"
           df -h
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - name: Setup Kind
         uses: ./.github/actions/setup-kind
       - if: matrix.mode.minio
         name: Setup Minio
         uses: ./.github/actions/setup-minio
       - if: matrix.mode.kafka
-        name: Setup Kafka cluster
+        name: Setup Kafka cluser
         uses: ./.github/actions/setup-kafka-cluster
-      - name: Setup Etcd cluster
+      - name: Setup Etcd cluser
         uses: ./.github/actions/setup-etcd-cluster
       # Prepares for fuzz tests
       - uses: arduino/setup-protoc@v3
@@ -410,11 +376,6 @@ jobs:
         shell: bash
         run: |
           kubectl describe nodes
-      - name: Describe pod
-        if: failure()
-        shell: bash
-        run: |
-          kubectl describe pod -n my-greptimedb
       - name: Export kind logs
         if: failure()
         shell: bash
@@ -437,13 +398,11 @@ jobs:
           docker system prune -f
 
   distributed-fuzztest-with-chaos:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Fuzz Test with Chaos (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
     runs-on: ubuntu-latest
     needs:  build-greptime-ci
     timeout-minutes: 60
     strategy:
-      fail-fast: false
       matrix:
         target: ["fuzz_migrate_mito_regions", "fuzz_migrate_metric_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
         mode:
@@ -478,8 +437,6 @@ jobs:
           echo "Disk space after:"
           df -h
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - name: Setup Kind
         uses: ./.github/actions/setup-kind
       - name: Setup Chaos Mesh
@@ -488,9 +445,9 @@ jobs:
         name: Setup Minio
         uses: ./.github/actions/setup-minio
       - if: matrix.mode.kafka
-        name: Setup Kafka cluster
+        name: Setup Kafka cluser
         uses: ./.github/actions/setup-kafka-cluster
-      - name: Setup Etcd cluster
+      - name: Setup Etcd cluser
         uses: ./.github/actions/setup-etcd-cluster
       # Prepares for fuzz tests
       - uses: arduino/setup-protoc@v3
@@ -564,11 +521,6 @@ jobs:
         shell: bash
         run: |
           kubectl describe nodes
-      - name: Describe pods
-        if: failure()
-        shell: bash
-        run: |
-          kubectl describe pod -n my-greptimedb
       - name: Export kind logs
         if: failure()
         shell: bash
@@ -591,14 +543,12 @@ jobs:
           docker system prune -f
 
   sqlness:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Sqlness Test (${{ matrix.mode.name }})
     needs: build
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-20.04 ]
         mode:
           - name: "Basic"
             opts: ""
@@ -606,14 +556,12 @@ jobs:
           - name: "Remote WAL"
             opts: "-w kafka -k 127.0.0.1:9092"
             kafka: true
-          - name: "PostgreSQL KvBackend"
+          - name: "Pg Kvbackend"
             opts: "--setup-pg"
             kafka: false
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - if: matrix.mode.kafka
         name: Setup kafka server
         working-directory: tests-integration/fixtures
@@ -636,14 +584,11 @@ jobs:
           retention-days: 3
 
   fmt:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Rustfmt
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -654,14 +599,11 @@ jobs:
         run: make fmt-check
 
   clippy:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Clippy
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -680,25 +622,20 @@ jobs:
         run: make clippy
 
   conflict-check:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     name: Check for conflict
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - name: Merge Conflict Finder
         uses: olivernybroe/action-conflict-finder@v4.0
 
   test:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'merge_group' }}
+    if: github.event_name != 'merge_group'
     runs-on: ubuntu-22.04-arm
     timeout-minutes: 60
     needs:  [conflict-check, clippy, fmt]
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -706,7 +643,7 @@ jobs:
       - name: Install toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          cache: false
+            cache: false
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
         with:
@@ -737,19 +674,16 @@ jobs:
           GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
           GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
           GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
-          GT_MYSQL_ENDPOINTS: mysql://greptimedb:admin@127.0.0.1:3306/mysql
           GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
           GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
           UNITTEST_LOG_DIR: "__unittest_logs"
 
   coverage:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && github.event_name == 'merge_group' }}
-    runs-on: ubuntu-22.04-8-cores
+    if: github.event_name == 'merge_group'
+    runs-on: ubuntu-20.04-8-cores
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -789,7 +723,6 @@ jobs:
           GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
           GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
           GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
-          GT_MYSQL_ENDPOINTS: mysql://greptimedb:admin@127.0.0.1:3306/mysql
           GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
           GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
           UNITTEST_LOG_DIR: "__unittest_logs"
@@ -803,10 +736,9 @@ jobs:
           verbose: true
 
   # compat:
-  #   if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
   #   name: Compatibility Test
   #   needs: build
-  #   runs-on: ubuntu-22.04
+  #   runs-on: ubuntu-20.04
   #   timeout-minutes: 60
   #   steps:
   #     - uses: actions/checkout@v4

.github/workflows/docbot.yml

@@ -3,21 +3,16 @@ on:
   pull_request_target:
     types: [opened, edited]
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
+permissions:
+  pull-requests: write
+  contents: read
 
 jobs:
   docbot:
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
-      contents: read
+    runs-on: ubuntu-20.04
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - name: Maybe Follow Up Docs Issue
         working-directory: cyborg

.github/workflows/docs.yml

@@ -31,47 +31,43 @@ name: CI
 jobs:
   typos:
     name: Spell Check with Typos
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: crate-ci/typos@master
 
   license-header-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     name: Check License Header
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: korandoru/hawkeye@v5
 
   check:
     name: Check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   fmt:
     name: Rustfmt
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   clippy:
     name: Clippy
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   coverage:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
@@ -80,7 +76,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-20.04 ]
         mode:
           - name: "Basic"
           - name: "Remote WAL"

.github/workflows/nightly-build.yml

@@ -14,11 +14,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-22.04
-          - ubuntu-22.04-8-cores
-          - ubuntu-22.04-16-cores
-          - ubuntu-22.04-32-cores
-          - ubuntu-22.04-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -66,11 +66,18 @@ env:
 
   NIGHTLY_RELEASE_PREFIX: nightly
 
+  # Use the different image name to avoid conflict with the release images.
+  # The DockerHub image will be greptime/greptimedb-nightly.
+  IMAGE_NAME: greptimedb-nightly
+
+permissions:
+  issues: write
+
 jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -88,7 +95,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Create version
         id: create-version
@@ -141,7 +147,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - uses: ./.github/actions/build-linux-artifacts
         with:
@@ -163,7 +168,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - uses: ./.github/actions/build-linux-artifacts
         with:
@@ -182,25 +186,24 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       nightly-build-result: ${{ steps.set-nightly-build-result.outputs.nightly-build-result }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Build and push images to dockerhub
         uses: ./.github/actions/build-images
         with:
           image-registry: docker.io
           image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-name: ${{ vars.NIGHTLY_BUILD_IMAGE_NAME }}
+          image-name: ${{ env.IMAGE_NAME }}
           image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          push-latest-tag: false
+          push-latest-tag: true
 
       - name: Set nightly build result
         id: set-nightly-build-result
@@ -214,7 +217,7 @@ jobs:
       allocate-runners,
       release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -223,14 +226,13 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Release artifacts to CN region
         uses: ./.github/actions/release-cn-artifacts
         with:
           src-image-registry: docker.io
           src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          src-image-name: ${{ vars.NIGHTLY_BUILD_IMAGE_NAME }}
+          src-image-name: ${{ env.IMAGE_NAME }}
           dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
           dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
           dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
@@ -240,16 +242,15 @@ jobs:
           aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: false
           dev-mode: false
           update-version-info: false  # Don't update version info in S3.
-          push-latest-tag: false
+          push-latest-tag: true
 
   stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -259,7 +260,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Stop EC2 runner
         uses: ./.github/actions/stop-runner
@@ -275,7 +275,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -285,7 +285,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Stop EC2 runner
         uses: ./.github/actions/stop-runner
@@ -303,15 +302,11 @@ jobs:
     needs: [
       release-images-to-dockerhub
     ]
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
+    runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - name: Report CI status
         id: report-ci-status

.github/workflows/nightly-ci.yml

@@ -9,17 +9,19 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+permissions:
+  issues: write
+
 jobs:
   sqlness-test:
     name: Run sqlness test
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Check install.sh
         run: ./.github/scripts/check-install-script.sh
@@ -44,14 +46,9 @@ jobs:
     name: Sqlness tests on Windows
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: windows-2022-8-cores
-    permissions:
-      issues: write
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - uses: arduino/setup-protoc@v3
         with:
@@ -79,9 +76,6 @@ jobs:
     steps:
       - run: git config --global core.autocrlf false
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - uses: arduino/setup-protoc@v3
         with:
@@ -107,6 +101,7 @@ jobs:
           CARGO_BUILD_RUSTFLAGS: "-C linker=lld-link"
           RUST_BACKTRACE: 1
           CARGO_INCREMENTAL: 0
+          RUSTUP_WINDOWS_PATH_ADD_BIN: 1 # Workaround for https://github.com/nextest-rs/nextest/issues/1493
           GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
           GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
           GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
@@ -116,23 +111,19 @@ jobs:
   cleanbuild-linux-nix:
     name: Run clean build on Linux
     runs-on: ubuntu-latest
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    timeout-minutes: 45
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
+      - uses: cachix/install-nix-action@v27
         with:
-          fetch-depth: 0
-          persist-credentials: false
-      - uses: cachix/install-nix-action@v31
-      - run: nix develop --command cargo check --bin greptime
-        env:
-          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
+          nix_path: nixpkgs=channel:nixos-24.11
+      - run: nix develop --command cargo build
 
   check-status:
     name: Check status
     needs: [sqlness-test, sqlness-windows, test-on-windows]
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       check-result: ${{ steps.set-check-result.outputs.check-result }}
     steps:
@@ -145,14 +136,11 @@ jobs:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} # Not requiring successful dependent jobs, always run.
     name: Send notification to Greptime team
     needs: [check-status]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - name: Report CI status
         id: report-ci-status

.github/workflows/pr-labeling.yaml

@@ -1,42 +0,0 @@
-name: 'PR Labeling'
-
-on:
-  pull_request_target:
-    types:
-      - opened
-      - synchronize
-      - reopened
-
-permissions:
-  contents: read
-  pull-requests: write
-  issues: write
-
-jobs:
-  labeler:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v4
-
-      - uses: actions/labeler@v5
-        with:
-          configuration-path: ".github/labeler.yaml"
-          repo-token: "${{ secrets.GITHUB_TOKEN }}"
-
-  size-label:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: pascalgn/size-label-action@v0.5.5
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        with:
-          sizes: >
-            {
-              "0": "XS",
-              "100": "S",
-              "300": "M",
-              "1000": "L",
-              "1500": "XL",
-              "2000": "XXL"
-            }

.github/workflows/release-dev-builder-images.yaml

@@ -24,20 +24,12 @@ on:
         description: Release dev-builder-android image
         required: false
         default: false
-      update_dev_builder_image_tag:
-        type: boolean
-        description: Update the DEV_BUILDER_IMAGE_TAG in Makefile and create a PR
-        required: false
-        default: false
 
 jobs:
   release-dev-builder-images:
     name: Release dev builder images
-    # The jobs are triggered by the following events:
-    # 1. Manually triggered workflow_dispatch event
-    # 2. Push event when the PR that modifies the `rust-toolchain.toml` or `docker/dev-builder/**` is merged to main
-    if: ${{ github.event_name == 'push' || inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }}
-    runs-on: ubuntu-latest
+    if: ${{ inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }} # Only manually trigger this job.
+    runs-on: ubuntu-20.04-16-cores
     outputs:
       version: ${{ steps.set-version.outputs.version }}
     steps:
@@ -45,7 +37,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Configure build image version
         id: set-version
@@ -65,13 +56,13 @@ jobs:
           version: ${{ env.VERSION }}
           dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
-          build-dev-builder-ubuntu: ${{ inputs.release_dev_builder_ubuntu_image || github.event_name == 'push' }}
-          build-dev-builder-centos: ${{ inputs.release_dev_builder_centos_image || github.event_name == 'push' }}
-          build-dev-builder-android: ${{ inputs.release_dev_builder_android_image || github.event_name == 'push' }}
+          build-dev-builder-ubuntu: ${{ inputs.release_dev_builder_ubuntu_image }}
+          build-dev-builder-centos: ${{ inputs.release_dev_builder_centos_image }}
+          build-dev-builder-android: ${{ inputs.release_dev_builder_android_image }}
 
   release-dev-builder-images-ecr:
     name: Release dev builder images to AWS ECR
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       release-dev-builder-images
     ]
@@ -93,70 +84,52 @@ jobs:
 
       - name: Push dev-builder-ubuntu image
         shell: bash
-        if: ${{ inputs.release_dev_builder_ubuntu_image || github.event_name == 'push' }}
-        env:
-          IMAGE_VERSION: ${{ needs.release-dev-builder-images.outputs.version }}
-          IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }}
-          ECR_IMAGE_REGISTRY: ${{ vars.ECR_IMAGE_REGISTRY }}
-          ECR_IMAGE_NAMESPACE: ${{ vars.ECR_IMAGE_NAMESPACE }}
+        if: ${{ inputs.release_dev_builder_ubuntu_image }}
         run: |
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-ubuntu:$IMAGE_VERSION \
-            docker://$ECR_IMAGE_REGISTRY/$ECR_IMAGE_NAMESPACE/dev-builder-ubuntu:$IMAGE_VERSION
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }}
 
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-ubuntu:latest \
-            docker://$ECR_IMAGE_REGISTRY/$ECR_IMAGE_NAMESPACE/dev-builder-ubuntu:latest
-
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-ubuntu:latest
       - name: Push dev-builder-centos image
         shell: bash
-        if: ${{ inputs.release_dev_builder_centos_image || github.event_name == 'push' }}
-        env:
-          IMAGE_VERSION: ${{ needs.release-dev-builder-images.outputs.version }}
-          IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }}
-          ECR_IMAGE_REGISTRY: ${{ vars.ECR_IMAGE_REGISTRY }}
-          ECR_IMAGE_NAMESPACE: ${{ vars.ECR_IMAGE_NAMESPACE }}
+        if: ${{ inputs.release_dev_builder_centos_image }}
         run: |
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-centos:$IMAGE_VERSION \
-            docker://$ECR_IMAGE_REGISTRY/$ECR_IMAGE_NAMESPACE/dev-builder-centos:$IMAGE_VERSION
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }}
 
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-centos:latest \
-            docker://$ECR_IMAGE_REGISTRY/$ECR_IMAGE_NAMESPACE/dev-builder-centos:latest
-
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-centos:latest
       - name: Push dev-builder-android image
         shell: bash
-        if: ${{ inputs.release_dev_builder_android_image || github.event_name == 'push' }}
-        env:
-          IMAGE_VERSION: ${{ needs.release-dev-builder-images.outputs.version }}
-          IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }}
-          ECR_IMAGE_REGISTRY: ${{ vars.ECR_IMAGE_REGISTRY }}
-          ECR_IMAGE_NAMESPACE: ${{ vars.ECR_IMAGE_NAMESPACE }}
+        if: ${{ inputs.release_dev_builder_android_image }}
         run: |
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-android:$IMAGE_VERSION \
-            docker://$ECR_IMAGE_REGISTRY/$ECR_IMAGE_NAMESPACE/dev-builder-android:$IMAGE_VERSION
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }}
 
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-android:latest \
-            docker://$ECR_IMAGE_REGISTRY/$ECR_IMAGE_NAMESPACE/dev-builder-android:latest
-
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-android:latest
   release-dev-builder-images-cn: # Note: Be careful issue: https://github.com/containers/skopeo/issues/1874 and we decide to use the latest stable skopeo container.
     name: Release dev builder images to CN region
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       release-dev-builder-images
     ]
@@ -170,63 +143,30 @@ jobs:
 
       - name: Push dev-builder-ubuntu image
         shell: bash
-        if: ${{ inputs.release_dev_builder_ubuntu_image || github.event_name == 'push' }}
-        env:
-          IMAGE_VERSION: ${{ needs.release-dev-builder-images.outputs.version }}
-          IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }}
-          ACR_IMAGE_REGISTRY: ${{ vars.ACR_IMAGE_REGISTRY }}
+        if: ${{ inputs.release_dev_builder_ubuntu_image }}
         run: |
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-ubuntu:$IMAGE_VERSION \
-            docker://$ACR_IMAGE_REGISTRY/$IMAGE_NAMESPACE/dev-builder-ubuntu:$IMAGE_VERSION
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }}
 
       - name: Push dev-builder-centos image
         shell: bash
-        if: ${{ inputs.release_dev_builder_centos_image || github.event_name == 'push' }}
-        env:
-          IMAGE_VERSION: ${{ needs.release-dev-builder-images.outputs.version }}
-          IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }}
-          ACR_IMAGE_REGISTRY: ${{ vars.ACR_IMAGE_REGISTRY }}
+        if: ${{ inputs.release_dev_builder_centos_image }}
         run: |
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-centos:$IMAGE_VERSION \
-            docker://$ACR_IMAGE_REGISTRY/$IMAGE_NAMESPACE/dev-builder-centos:$IMAGE_VERSION
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }}
 
       - name: Push dev-builder-android image
         shell: bash
-        if: ${{ inputs.release_dev_builder_android_image || github.event_name == 'push' }}
-        env:
-          IMAGE_VERSION: ${{ needs.release-dev-builder-images.outputs.version }}
-          IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }}
-          ACR_IMAGE_REGISTRY: ${{ vars.ACR_IMAGE_REGISTRY }}
+        if: ${{ inputs.release_dev_builder_android_image }}
         run: |
           docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
             -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
             quay.io/skopeo/stable:latest \
-            copy -a docker://docker.io/$IMAGE_NAMESPACE/dev-builder-android:$IMAGE_VERSION \
-            docker://$ACR_IMAGE_REGISTRY/$IMAGE_NAMESPACE/dev-builder-android:$IMAGE_VERSION
-  
-  update-dev-builder-image-tag:
-    name: Update dev-builder image tag
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    if: ${{ github.event_name == 'push' || inputs.update_dev_builder_image_tag }}
-    needs: [
-      release-dev-builder-images
-    ]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Update dev-builder image tag
-        shell: bash
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          ./.github/scripts/update-dev-builder-version.sh ${{ needs.release-dev-builder-images.outputs.version }}
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }}

.github/workflows/release.yml

@@ -18,11 +18,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-22.04
-          - ubuntu-22.04-8-cores
-          - ubuntu-22.04-16-cores
-          - ubuntu-22.04-32-cores
-          - ubuntu-22.04-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -88,14 +88,21 @@ env:
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
   DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}
 
-  # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nightly-20230313;
+  # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
+  # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
+  NEXT_RELEASE_VERSION: v0.12.0
+
+# Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+permissions:
+  issues: write # Allows the action to create issues for cyborg.
+  contents: write # Allows the action to create a release.
 
 jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -110,14 +117,11 @@ jobs:
 
       # The 'version' use as the global tag name of the release workflow.
       version: ${{ steps.create-version.outputs.version }}
-
-      should-push-latest-tag: ${{ steps.check-version.outputs.should-push-latest-tag }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Check Rust toolchain version
         shell: bash
@@ -126,7 +130,7 @@ jobs:
 
       # The create-version will create a global variable named 'version' in the global workflows.
       # - If it's a tag push release, the version is the tag name(${{ github.ref_name }});
-      # - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nightly-20230313;
+      # - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nigthly-20230313;
       # - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-<short-git-sha>-YYYYMMDDSS', like v0.2.0-e5b243c-2023071245;
       - name: Create version
         id: create-version
@@ -135,13 +139,9 @@ jobs:
         env:
           GITHUB_EVENT_NAME: ${{ github.event_name }}
           GITHUB_REF_NAME: ${{ github.ref_name }}
+          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
           NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
 
-      - name: Check version
-        id: check-version
-        run: |
-          ./.github/scripts/check-version.sh "${{ steps.create-version.outputs.version }}"
-
       - name: Allocate linux-amd64 runner
         if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
         uses: ./.github/actions/start-runner
@@ -181,7 +181,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - uses: ./.github/actions/build-linux-artifacts
         with:
@@ -203,7 +202,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - uses: ./.github/actions/build-linux-artifacts
         with:
@@ -239,7 +237,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - uses: ./.github/actions/build-macos-artifacts
         with:
@@ -279,7 +276,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - uses: ./.github/actions/build-windows-artifacts
         with:
@@ -303,25 +299,22 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-2004-16-cores
     outputs:
       build-image-result: ${{ steps.set-build-image-result.outputs.build-image-result }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Build and push images to dockerhub
         uses: ./.github/actions/build-images
         with:
           image-registry: docker.io
           image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-name: ${{ vars.GREPTIMEDB_IMAGE_NAME }}
           image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          push-latest-tag: ${{ needs.allocate-runners.outputs.should-push-latest-tag == 'true' && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
 
       - name: Set build image result
         id: set-build-image-result
@@ -339,7 +332,7 @@ jobs:
       build-windows-artifacts,
       release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-20.04
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -348,14 +341,13 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Release artifacts to CN region
         uses: ./.github/actions/release-cn-artifacts
         with:
           src-image-registry: docker.io
           src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          src-image-name: ${{ vars.GREPTIMEDB_IMAGE_NAME }}
+          src-image-name: greptimedb
           dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
           dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
           dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
@@ -366,9 +358,8 @@ jobs:
           aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
           dev-mode: false
-          upload-to-s3: true
           update-version-info: true
-          push-latest-tag: ${{ needs.allocate-runners.outputs.should-push-latest-tag == 'true' && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
+          push-latest-tag: true
 
   publish-github-release:
     name: Create GitHub release and upload artifacts
@@ -381,12 +372,11 @@ jobs:
       build-windows-artifacts,
       release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Publish GitHub release
         uses: ./.github/actions/publish-github-release
@@ -395,12 +385,12 @@ jobs:
 
   ### Stop runners ###
   # It's very necessary to split the job of releasing runners into 'stop-linux-amd64-runner' and 'stop-linux-arm64-runner'.
-  # Because we can terminate the specified EC2 instance immediately after the job is finished without unnecessary waiting.
+  # Because we can terminate the specified EC2 instance immediately after the job is finished without uncessary waiting.
   stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -410,7 +400,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Stop EC2 runner
         uses: ./.github/actions/stop-runner
@@ -426,7 +415,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -436,7 +425,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Stop EC2 runner
         uses: ./.github/actions/stop-runner
@@ -448,73 +436,21 @@ jobs:
           aws-region: ${{ vars.EC2_RUNNER_REGION }}
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
 
-  bump-downstream-repo-versions:
-    name: Bump downstream repo versions
+  bump-doc-version:
+    name: Bump doc version
     if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }}
-    needs: [allocate-runners, publish-github-release]
-    runs-on: ubuntu-latest
-    # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
-    permissions:
-      issues: write # Allows the action to create issues for cyborg.
-      contents: write # Allows the action to create a release.
+    needs: [allocate-runners]
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
-      - name: Bump downstream repo versions
+      - name: Bump doc version
         working-directory: cyborg
-        run: pnpm tsx bin/bump-versions.ts
+        run: pnpm tsx bin/bump-doc-version.ts
         env:
-          TARGET_REPOS: website,docs,demo
           VERSION: ${{ needs.allocate-runners.outputs.version }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          WEBSITE_REPO_TOKEN: ${{ secrets.WEBSITE_REPO_TOKEN }}
           DOCS_REPO_TOKEN: ${{ secrets.DOCS_REPO_TOKEN }}
-          DEMO_REPO_TOKEN: ${{ secrets.DEMO_REPO_TOKEN }}
-
-  bump-helm-charts-version:
-    name: Bump helm charts version
-    if: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
-    needs: [allocate-runners, publish-github-release]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Bump helm charts version
-        env:
-          GITHUB_TOKEN: ${{ secrets.HELM_CHARTS_REPO_TOKEN }}
-          VERSION: ${{ needs.allocate-runners.outputs.version }}
-        run: |
-          ./.github/scripts/update-helm-charts-version.sh
-
-  bump-homebrew-greptime-version:
-    name: Bump homebrew greptime version
-    if: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }}
-    needs: [allocate-runners, publish-github-release]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Bump homebrew greptime version
-        env:
-          GITHUB_TOKEN: ${{ secrets.HOMEBREW_GREPTIME_REPO_TOKEN }}
-          VERSION: ${{ needs.allocate-runners.outputs.version }}
-        run: |
-          ./.github/scripts/update-homebrew-greptme-version.sh
 
   notification:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || github.event_name == 'schedule') && always() }}
@@ -524,18 +460,11 @@ jobs:
       build-macos-artifacts,
       build-windows-artifacts,
     ]
-    runs-on: ubuntu-latest
-    # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
-    permissions:
-      issues: write # Allows the action to create issues for cyborg.
-      contents: write # Allows the action to create a release.
+    runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - name: Report CI status
         id: report-ci-status

.github/workflows/schedule.yml

@@ -4,20 +4,18 @@ on:
     - cron: '4 2 * * *'
   workflow_dispatch:
 
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
 
 jobs:
   maintenance:
     name: Periodic Maintenance
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      issues: write
-      pull-requests: write
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
       - uses: ./.github/actions/setup-cyborg
       - name: Do Maintenance
         working-directory: cyborg

.github/workflows/semantic-pull-request.yml

@@ -1,24 +1,15 @@
 name: "Semantic Pull Request"
 
 on:
-  pull_request:
+  pull_request_target:
     types:
       - opened
       - reopened
       - edited
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-permissions:
-  issues: write
-  contents: write
-  pull-requests: write
-
 jobs:
   check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4

.gitignore

@@ -54,10 +54,3 @@ tests-fuzz/corpus/
 # Nix
 .direnv
 .envrc
-
-## default data home
-greptimedb_data
-
-# github
-!/.github
-

AUTHOR.md

@@ -3,28 +3,30 @@
 ## Individual Committers (in alphabetical order)
 
 * [CookiePieWw](https://github.com/CookiePieWw)
+* [KKould](https://github.com/KKould)
+* [NiwakaDev](https://github.com/NiwakaDev)
 * [etolbakov](https://github.com/etolbakov)
 * [irenjj](https://github.com/irenjj)
-* [KKould](https://github.com/KKould)
-* [Lanqing Yang](https://github.com/lyang24)
-* [NiwakaDev](https://github.com/NiwakaDev)
 * [tisonkun](https://github.com/tisonkun)
-
+* [Lanqing Yang](https://github.com/lyang24)
 
 ## Team Members (in alphabetical order)
 
+* [Breeze-P](https://github.com/Breeze-P)
+* [GrepTime](https://github.com/GrepTime)
+* [MichaelScofield](https://github.com/MichaelScofield)
+* [Wenjie0329](https://github.com/Wenjie0329)
+* [WenyXu](https://github.com/WenyXu)
+* [ZonaHex](https://github.com/ZonaHex)
 * [apdong2022](https://github.com/apdong2022)
 * [beryl678](https://github.com/beryl678)
-* [Breeze-P](https://github.com/Breeze-P)
 * [daviderli614](https://github.com/daviderli614)
 * [discord9](https://github.com/discord9)
 * [evenyag](https://github.com/evenyag)
 * [fengjiachun](https://github.com/fengjiachun)
 * [fengys1996](https://github.com/fengys1996)
-* [GrepTime](https://github.com/GrepTime)
 * [holalengyu](https://github.com/holalengyu)
 * [killme2008](https://github.com/killme2008)
-* [MichaelScofield](https://github.com/MichaelScofield)
 * [nicecui](https://github.com/nicecui)
 * [paomian](https://github.com/paomian)
 * [shuiyisong](https://github.com/shuiyisong)
@@ -32,14 +34,11 @@
 * [sunng87](https://github.com/sunng87)
 * [v0y4g3r](https://github.com/v0y4g3r)
 * [waynexia](https://github.com/waynexia)
-* [Wenjie0329](https://github.com/Wenjie0329)
-* [WenyXu](https://github.com/WenyXu)
 * [xtang](https://github.com/xtang)
 * [zhaoyingnan01](https://github.com/zhaoyingnan01)
 * [zhongzc](https://github.com/zhongzc)
-* [ZonaHex](https://github.com/ZonaHex)
 * [zyy17](https://github.com/zyy17)
 
 ## All Contributors
 
-To see the full list of contributors, please visit our [Contributors page](https://github.com/GreptimeTeam/greptimedb/graphs/contributors)
+[![All Contributors](https://contrib.rocks/image?repo=GreptimeTeam/greptimedb)](https://github.com/GreptimeTeam/greptimedb/graphs/contributors)

Cargo.toml

@@ -67,7 +67,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.12.2"
+version = "0.12.0"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -81,7 +81,6 @@ rust.unknown_lints = "deny"
 rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
 
 [workspace.dependencies]
-# DO_NOT_REMOVE_THIS: BEGIN_OF_EXTERNAL_DEPENDENCIES
 # We turn off default-features for some dependencies here so the workspaces which inherit them can
 # selectively turn them on if needed, since we can override default-features = true (from false)
 # for the inherited dependency but cannot do the reverse (override from true to false).
@@ -107,7 +106,6 @@ bitflags = "2.4.1"
 bytemuck = "1.12"
 bytes = { version = "1.7", features = ["serde"] }
 chrono = { version = "0.4", features = ["serde"] }
-chrono-tz = "0.10.1"
 clap = { version = "4.4", features = ["derive"] }
 config = "0.13.0"
 crossbeam-utils = "0.8"
@@ -129,7 +127,8 @@ etcd-client = "0.14"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "072ce580502e015df1a6b03a185b60309a7c2a7a" }
+# branch: poc-write-path
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "1915576b113a494f5352fd61f211d899b7f87aab" }
 hex = "0.4"
 http = "1"
 humantime = "2.1"
@@ -140,8 +139,8 @@ itertools = "0.10"
 jsonb = { git = "https://github.com/databendlabs/jsonb.git", rev = "8c8d2fc294a39f3ff08909d60f718639cfba3875", default-features = false }
 lazy_static = "1.4"
 local-ip-address = "0.6"
-loki-proto = { git = "https://github.com/GreptimeTeam/loki-proto.git", rev = "1434ecf23a2654025d86188fb5205e7a74b225d3" }
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" }
+loki-api = { git = "https://github.com/shuiyisong/tracing-loki", branch = "chore/prost_version" }
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "a10facb353b41460eeb98578868ebf19c2084fac" }
 mockall = "0.11.4"
 moka = "0.12"
 nalgebra = "0.33"
@@ -160,9 +159,7 @@ parquet = { version = "53.0.0", default-features = false, features = ["arrow", "
 paste = "1.0"
 pin-project = "1.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-promql-parser = { git = "https://github.com/GreptimeTeam/promql-parser.git", features = [
-    "ser",
-], rev = "27abb8e16003a50c720f00d6c85f41f5fa2a2a8e" }
+promql-parser = { version = "0.4.3", features = ["ser"] }
 prost = "0.13"
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.8"
@@ -211,7 +208,6 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"]
 typetag = "0.2"
 uuid = { version = "1.7", features = ["serde", "v4", "fast-rng"] }
 zstd = "0.13"
-# DO_NOT_REMOVE_THIS: END_OF_EXTERNAL_DEPENDENCIES
 
 ## workspaces members
 api = { path = "src/api" }
@@ -283,10 +279,12 @@ tokio-rustls = { git = "https://github.com/GreptimeTeam/tokio-rustls", rev = "46
 # This is commented, since we are not using aws-lc-sys, if we need to use it, we need to uncomment this line or use a release after this commit, or it wouldn't compile with gcc < 8.1
 # see https://github.com/aws/aws-lc-rs/pull/526
 # aws-lc-sys = { git ="https://github.com/aws/aws-lc-rs", rev = "556558441e3494af4b156ae95ebc07ebc2fd38aa" }
+# Apply a fix for pprof for unaligned pointer access
+pprof = { git = "https://github.com/GreptimeTeam/pprof-rs", rev = "1bd1e21" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
-rev = "5618e779cf2bb4755b499c630fba4c35e91898cb"
+rev = "a10facb353b41460eeb98578868ebf19c2084fac"
 
 [profile.release]
 debug = 1

Cross.toml

@@ -1,6 +1,3 @@
-[target.aarch64-unknown-linux-gnu]
-image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:0.2.5"
-
 [build]
 pre-build = [
     "dpkg --add-architecture $CROSS_DEB_ARCH",
@@ -8,8 +5,3 @@ pre-build = [
     "curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip && unzip protoc-3.15.8-linux-x86_64.zip -d /usr/",
     "chmod a+x /usr/bin/protoc && chmod -R a+rx /usr/include/google",
 ]
-
-[build.env]
-passthrough = [
-    "JEMALLOC_SYS_WITH_LG_PAGE",
-]

README.md

@@ -116,7 +116,7 @@ docker run -p 127.0.0.1:4000-4003:4000-4003 \
   --name greptime --rm \
   greptime/greptimedb:latest standalone start \
   --http-addr 0.0.0.0:4000 \
-  --rpc-bind-addr 0.0.0.0:4001 \
+  --rpc-addr 0.0.0.0:4001 \
   --mysql-addr 0.0.0.0:4002 \
   --postgres-addr 0.0.0.0:4003
 ```

config/config.md

@@ -29,7 +29,7 @@
 | `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
 | `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
 | `grpc` | -- | -- | The gRPC server options. |
-| `grpc.bind_addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
+| `grpc.addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
@@ -40,7 +40,6 @@
 | `mysql.enable` | Bool | `true` | Whether to enable. |
 | `mysql.addr` | String | `127.0.0.1:4002` | The addr to bind the MySQL server. |
 | `mysql.runtime_size` | Integer | `2` | The number of server worker threads. |
-| `mysql.keep_alive` | String | `0s` | Server-side keep-alive time.<br/>Set to 0 (default) to disable. |
 | `mysql.tls` | -- | -- | -- |
 | `mysql.tls.mode` | String | `disable` | TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html<br/>- `disable` (default value)<br/>- `prefer`<br/>- `require`<br/>- `verify-ca`<br/>- `verify-full` |
 | `mysql.tls.cert_path` | String | Unset | Certificate file path. |
@@ -50,7 +49,6 @@
 | `postgres.enable` | Bool | `true` | Whether to enable |
 | `postgres.addr` | String | `127.0.0.1:4003` | The addr to bind the PostgresSQL server. |
 | `postgres.runtime_size` | Integer | `2` | The number of server worker threads. |
-| `postgres.keep_alive` | String | `0s` | Server-side keep-alive time.<br/>Set to 0 (default) to disable. |
 | `postgres.tls` | -- | -- | PostgresSQL server TLS options, see `mysql.tls` section. |
 | `postgres.tls.mode` | String | `disable` | TLS mode. |
 | `postgres.tls.cert_path` | String | Unset | Certificate file path. |
@@ -60,8 +58,6 @@
 | `opentsdb.enable` | Bool | `true` | Whether to enable OpenTSDB put in HTTP API. |
 | `influxdb` | -- | -- | InfluxDB protocol options. |
 | `influxdb.enable` | Bool | `true` | Whether to enable InfluxDB protocol in HTTP API. |
-| `jaeger` | -- | -- | Jaeger protocol options. |
-| `jaeger.enable` | Bool | `true` | Whether to enable Jaeger protocol in HTTP API. |
 | `prom_store` | -- | -- | Prometheus remote storage options |
 | `prom_store.enable` | Bool | `true` | Whether to enable Prometheus remote write and read in HTTP API. |
 | `prom_store.with_metric_engine` | Bool | `true` | Whether to store the data from Prometheus remote write in metric engine. |
@@ -69,8 +65,8 @@
 | `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka. |
 | `wal.dir` | String | Unset | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.file_size` | String | `128MB` | The size of the WAL segment file.<br/>**It's only used when the provider is `raft_engine`**. |
-| `wal.purge_threshold` | String | `1GB` | The threshold of the WAL size to trigger a purge.<br/>**It's only used when the provider is `raft_engine`**. |
-| `wal.purge_interval` | String | `1m` | The interval to trigger a purge.<br/>**It's only used when the provider is `raft_engine`**. |
+| `wal.purge_threshold` | String | `1GB` | The threshold of the WAL size to trigger a flush.<br/>**It's only used when the provider is `raft_engine`**. |
+| `wal.purge_interval` | String | `1m` | The interval to trigger a flush.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.read_batch_size` | Integer | `128` | The read batch size.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.sync_write` | Bool | `false` | Whether to use sync write.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.enable_log_recycle` | Bool | `true` | Whether to reuse logically truncated log files.<br/>**It's only used when the provider is `raft_engine`**. |
@@ -92,9 +88,8 @@
 | `wal.backoff_deadline` | String | `5mins` | The deadline of retries.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system<br/>can still successfully replay memtable data without throwing an<br/>out-of-range error.<br/>However, enabling this option might lead to unexpected data loss,<br/>as the system will skip over missing entries instead of treating<br/>them as critical errors. |
 | `metadata_store` | -- | -- | Metadata storage options. |
-| `metadata_store.file_size` | String | `64MB` | The size of the metadata store log file. |
-| `metadata_store.purge_threshold` | String | `256MB` | The threshold of the metadata store size to trigger a purge. |
-| `metadata_store.purge_interval` | String | `1m` | The interval of the metadata store to trigger a purge. |
+| `metadata_store.file_size` | String | `256MB` | Kv file size in bytes. |
+| `metadata_store.purge_threshold` | String | `4GB` | Kv purge threshold. |
 | `procedure` | -- | -- | Procedure storage options. |
 | `procedure.max_retry_times` | Integer | `3` | Procedure max retry time. |
 | `procedure.retry_delay` | String | `500ms` | Initial retry delay of procedures, increases exponentially |
@@ -152,7 +147,6 @@
 | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. |
 | `region_engine.mito.index.aux_path` | String | `""` | Auxiliary directory path for the index in filesystem, used to store intermediate files for<br/>creating the index and staging files for searching the index, defaults to `{data_home}/index_intermediate`.<br/>The default name for this directory is `index_intermediate` for backward compatibility.<br/><br/>This path contains two subdirectories:<br/>- `__intm`: for storing intermediate files used during creating index.<br/>- `staging`: for storing staging files used during searching index. |
 | `region_engine.mito.index.staging_size` | String | `2GB` | The max capacity of the staging directory. |
-| `region_engine.mito.index.staging_ttl` | String | `7d` | The TTL of the staging directory.<br/>Defaults to 7 days.<br/>Setting it to "0s" to disable TTL. |
 | `region_engine.mito.index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. |
 | `region_engine.mito.index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. |
 | `region_engine.mito.index.content_cache_page_size` | String | `64KiB` | Page size for inverted index content cache. |
@@ -227,8 +221,8 @@
 | `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
 | `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
 | `grpc` | -- | -- | The gRPC server options. |
-| `grpc.bind_addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:4001` | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
+| `grpc.addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
+| `grpc.hostname` | String | `127.0.0.1:4001` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
@@ -239,7 +233,6 @@
 | `mysql.enable` | Bool | `true` | Whether to enable. |
 | `mysql.addr` | String | `127.0.0.1:4002` | The addr to bind the MySQL server. |
 | `mysql.runtime_size` | Integer | `2` | The number of server worker threads. |
-| `mysql.keep_alive` | String | `0s` | Server-side keep-alive time.<br/>Set to 0 (default) to disable. |
 | `mysql.tls` | -- | -- | -- |
 | `mysql.tls.mode` | String | `disable` | TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html<br/>- `disable` (default value)<br/>- `prefer`<br/>- `require`<br/>- `verify-ca`<br/>- `verify-full` |
 | `mysql.tls.cert_path` | String | Unset | Certificate file path. |
@@ -249,7 +242,6 @@
 | `postgres.enable` | Bool | `true` | Whether to enable |
 | `postgres.addr` | String | `127.0.0.1:4003` | The addr to bind the PostgresSQL server. |
 | `postgres.runtime_size` | Integer | `2` | The number of server worker threads. |
-| `postgres.keep_alive` | String | `0s` | Server-side keep-alive time.<br/>Set to 0 (default) to disable. |
 | `postgres.tls` | -- | -- | PostgresSQL server TLS options, see `mysql.tls` section. |
 | `postgres.tls.mode` | String | `disable` | TLS mode. |
 | `postgres.tls.cert_path` | String | Unset | Certificate file path. |
@@ -259,8 +251,6 @@
 | `opentsdb.enable` | Bool | `true` | Whether to enable OpenTSDB put in HTTP API. |
 | `influxdb` | -- | -- | InfluxDB protocol options. |
 | `influxdb.enable` | Bool | `true` | Whether to enable InfluxDB protocol in HTTP API. |
-| `jaeger` | -- | -- | Jaeger protocol options. |
-| `jaeger.enable` | Bool | `true` | Whether to enable Jaeger protocol in HTTP API. |
 | `prom_store` | -- | -- | Prometheus remote storage options |
 | `prom_store.enable` | Bool | `true` | Whether to enable Prometheus remote write and read in HTTP API. |
 | `prom_store.with_metric_engine` | Bool | `true` | Whether to store the data from Prometheus remote write in metric engine. |
@@ -310,7 +300,7 @@
 | --- | -----| ------- | ----------- |
 | `data_home` | String | `/tmp/metasrv/` | The working home directory. |
 | `bind_addr` | String | `127.0.0.1:3002` | The bind address of metasrv. |
-| `server_addr` | String | `127.0.0.1:3002` | The communication server address for the frontend and datanode to connect to metasrv.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `bind_addr`. |
+| `server_addr` | String | `127.0.0.1:3002` | The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost. |
 | `store_addrs` | Array | -- | Store server address default to etcd store.<br/>For postgres store, the format is:<br/>"password=password dbname=postgres user=postgres host=localhost port=5432"<br/>For etcd store, the format is:<br/>"127.0.0.1:2379" |
 | `store_key_prefix` | String | `""` | If it's not empty, the metasrv will store all data with this key prefix. |
 | `backend` | String | `etcd_store` | The datastore for meta server.<br/>Available values:<br/>- `etcd_store` (default value)<br/>- `memory_store`<br/>- `postgres_store` |
@@ -319,7 +309,6 @@
 | `selector` | String | `round_robin` | Datanode selector type.<br/>- `round_robin` (default value)<br/>- `lease_based`<br/>- `load_based`<br/>For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector". |
 | `use_memory_store` | Bool | `false` | Store data in memory. |
 | `enable_region_failover` | Bool | `false` | Whether to enable region failover.<br/>This feature is only available on GreptimeDB running on cluster mode and<br/>- Using Remote WAL<br/>- Using shared storage (e.g., s3). |
-| `node_max_idle_time` | String | `24hours` | Max allowed idle time before removing node info from metasrv memory. |
 | `enable_telemetry` | Bool | `true` | Whether to enable greptimedb telemetry. Enabled by default. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
@@ -387,14 +376,19 @@
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
 | `max_concurrent_queries` | Integer | `0` | The maximum current queries allowed to be executed. Zero means unlimited. |
+| `rpc_addr` | String | Unset | Deprecated, use `grpc.addr` instead. |
+| `rpc_hostname` | String | Unset | Deprecated, use `grpc.hostname` instead. |
+| `rpc_runtime_size` | Integer | Unset | Deprecated, use `grpc.runtime_size` instead. |
+| `rpc_max_recv_message_size` | String | Unset | Deprecated, use `grpc.rpc_max_recv_message_size` instead. |
+| `rpc_max_send_message_size` | String | Unset | Deprecated, use `grpc.rpc_max_send_message_size` instead. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. Enabled by default. |
 | `http` | -- | -- | The HTTP server options. |
 | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
 | `http.timeout` | String | `30s` | HTTP request timeout. Set to 0 to disable timeout. |
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `grpc` | -- | -- | The gRPC server options. |
-| `grpc.bind_addr` | String | `127.0.0.1:3001` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:3001` | The address advertised to the metasrv, and used for connections from outside the host.<br/>If left empty or unset, the server will automatically use the IP address of the first network interface<br/>on the host, with the same port number as the one specified in `grpc.bind_addr`. |
+| `grpc.addr` | String | `127.0.0.1:3001` | The address to bind the gRPC server. |
+| `grpc.hostname` | String | `127.0.0.1:3001` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
@@ -493,7 +487,6 @@
 | `region_engine.mito.index` | -- | -- | The options for index in Mito engine. |
 | `region_engine.mito.index.aux_path` | String | `""` | Auxiliary directory path for the index in filesystem, used to store intermediate files for<br/>creating the index and staging files for searching the index, defaults to `{data_home}/index_intermediate`.<br/>The default name for this directory is `index_intermediate` for backward compatibility.<br/><br/>This path contains two subdirectories:<br/>- `__intm`: for storing intermediate files used during creating index.<br/>- `staging`: for storing staging files used during searching index. |
 | `region_engine.mito.index.staging_size` | String | `2GB` | The max capacity of the staging directory. |
-| `region_engine.mito.index.staging_ttl` | String | `7d` | The TTL of the staging directory.<br/>Defaults to 7 days.<br/>Setting it to "0s" to disable TTL. |
 | `region_engine.mito.index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. |
 | `region_engine.mito.index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. |
 | `region_engine.mito.index.content_cache_page_size` | String | `64KiB` | Page size for inverted index content cache. |
@@ -556,8 +549,8 @@
 | `flow` | -- | -- | flow engine options. |
 | `flow.num_workers` | Integer | `0` | The number of flow worker in flownode.<br/>Not setting(or set to 0) this value will use the number of CPU cores divided by 2. |
 | `grpc` | -- | -- | The gRPC server options. |
-| `grpc.bind_addr` | String | `127.0.0.1:6800` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:6800` | The address advertised to the metasrv,<br/>and used for connections from outside the host |
+| `grpc.addr` | String | `127.0.0.1:6800` | The address to bind the gRPC server. |
+| `grpc.hostname` | String | `127.0.0.1` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
 | `grpc.runtime_size` | Integer | `2` | The number of server worker threads. |
 | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |

config/datanode.example.toml

@@ -19,6 +19,26 @@ init_regions_parallelism = 16
 ## The maximum current queries allowed to be executed. Zero means unlimited.
 max_concurrent_queries = 0
 
+## Deprecated, use `grpc.addr` instead.
+## @toml2docs:none-default
+rpc_addr = "127.0.0.1:3001"
+
+## Deprecated, use `grpc.hostname` instead.
+## @toml2docs:none-default
+rpc_hostname = "127.0.0.1"
+
+## Deprecated, use `grpc.runtime_size` instead.
+## @toml2docs:none-default
+rpc_runtime_size = 8
+
+## Deprecated, use `grpc.rpc_max_recv_message_size` instead.
+## @toml2docs:none-default
+rpc_max_recv_message_size = "512MB"
+
+## Deprecated, use `grpc.rpc_max_send_message_size` instead.
+## @toml2docs:none-default
+rpc_max_send_message_size = "512MB"
+
 ## Enable telemetry to collect anonymous usage data. Enabled by default.
 #+ enable_telemetry = true
 
@@ -36,11 +56,10 @@ body_limit = "64MB"
 ## The gRPC server options.
 [grpc]
 ## The address to bind the gRPC server.
-bind_addr = "127.0.0.1:3001"
-## The address advertised to the metasrv, and used for connections from outside the host.
-## If left empty or unset, the server will automatically use the IP address of the first network interface
-## on the host, with the same port number as the one specified in `grpc.bind_addr`.
-server_addr = "127.0.0.1:3001"
+addr = "127.0.0.1:3001"
+## The hostname advertised to the metasrv,
+## and used for connections from outside the host
+hostname = "127.0.0.1:3001"
 ## The number of server worker threads.
 runtime_size = 8
 ## The maximum receive message size for gRPC server.
@@ -231,7 +250,6 @@ overwrite_entry_start_id = false
 # secret_access_key = "123456"
 # endpoint = "https://s3.amazonaws.com"
 # region = "us-west-2"
-# enable_virtual_host_style = false
 
 # Example of using Oss as the storage.
 # [storage]
@@ -498,11 +516,6 @@ aux_path = ""
 ## The max capacity of the staging directory.
 staging_size = "2GB"
 
-## The TTL of the staging directory.
-## Defaults to 7 days.
-## Setting it to "0s" to disable TTL.
-staging_ttl = "7d"
-
 ## Cache size for inverted index metadata.
 metadata_cache_size = "64MiB"
 

config/flownode.example.toml

@@ -14,10 +14,10 @@ node_id = 14
 ## The gRPC server options.
 [grpc]
 ## The address to bind the gRPC server.
-bind_addr = "127.0.0.1:6800"
-## The address advertised to the metasrv,
+addr = "127.0.0.1:6800"
+## The hostname advertised to the metasrv,
 ## and used for connections from outside the host
-server_addr = "127.0.0.1:6800"
+hostname = "127.0.0.1"
 ## The number of server worker threads.
 runtime_size = 2
 ## The maximum receive message size for gRPC server.

config/frontend.example.toml

@@ -41,11 +41,10 @@ cors_allowed_origins = ["https://example.com"]
 ## The gRPC server options.
 [grpc]
 ## The address to bind the gRPC server.
-bind_addr = "127.0.0.1:4001"
-## The address advertised to the metasrv, and used for connections from outside the host.
-## If left empty or unset, the server will automatically use the IP address of the first network interface
-## on the host, with the same port number as the one specified in `grpc.bind_addr`.
-server_addr = "127.0.0.1:4001"
+addr = "127.0.0.1:4001"
+## The hostname advertised to the metasrv,
+## and used for connections from outside the host
+hostname = "127.0.0.1:4001"
 ## The number of server worker threads.
 runtime_size = 8
 
@@ -74,9 +73,6 @@ enable = true
 addr = "127.0.0.1:4002"
 ## The number of server worker threads.
 runtime_size = 2
-## Server-side keep-alive time.
-## Set to 0 (default) to disable.
-keep_alive = "0s"
 
 # MySQL server TLS options.
 [mysql.tls]
@@ -108,9 +104,6 @@ enable = true
 addr = "127.0.0.1:4003"
 ## The number of server worker threads.
 runtime_size = 2
-## Server-side keep-alive time.
-## Set to 0 (default) to disable.
-keep_alive = "0s"
 
 ## PostgresSQL server TLS options, see `mysql.tls` section.
 [postgres.tls]
@@ -138,11 +131,6 @@ enable = true
 ## Whether to enable InfluxDB protocol in HTTP API.
 enable = true
 
-## Jaeger protocol options.
-[jaeger]
-## Whether to enable Jaeger protocol in HTTP API.
-enable = true
-
 ## Prometheus remote storage options
 [prom_store]
 ## Whether to enable Prometheus remote write and read in HTTP API.

config/metasrv.example.toml

@@ -4,9 +4,7 @@ data_home = "/tmp/metasrv/"
 ## The bind address of metasrv.
 bind_addr = "127.0.0.1:3002"
 
-## The communication server address for the frontend and datanode to connect to metasrv.
-## If left empty or unset, the server will automatically use the IP address of the first network interface
-## on the host, with the same port number as the one specified in `bind_addr`.
+## The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost.
 server_addr = "127.0.0.1:3002"
 
 ## Store server address default to etcd store.
@@ -50,9 +48,6 @@ use_memory_store = false
 ## - Using shared storage (e.g., s3).
 enable_region_failover = false
 
-## Max allowed idle time before removing node info from metasrv memory.
-node_max_idle_time = "24hours"
-
 ## Whether to enable greptimedb telemetry. Enabled by default.
 #+ enable_telemetry = true
 

config/standalone.example.toml

@@ -49,7 +49,7 @@ cors_allowed_origins = ["https://example.com"]
 ## The gRPC server options.
 [grpc]
 ## The address to bind the gRPC server.
-bind_addr = "127.0.0.1:4001"
+addr = "127.0.0.1:4001"
 ## The number of server worker threads.
 runtime_size = 8
 
@@ -78,9 +78,6 @@ enable = true
 addr = "127.0.0.1:4002"
 ## The number of server worker threads.
 runtime_size = 2
-## Server-side keep-alive time.
-## Set to 0 (default) to disable.
-keep_alive = "0s"
 
 # MySQL server TLS options.
 [mysql.tls]
@@ -112,9 +109,6 @@ enable = true
 addr = "127.0.0.1:4003"
 ## The number of server worker threads.
 runtime_size = 2
-## Server-side keep-alive time.
-## Set to 0 (default) to disable.
-keep_alive = "0s"
 
 ## PostgresSQL server TLS options, see `mysql.tls` section.
 [postgres.tls]
@@ -142,11 +136,6 @@ enable = true
 ## Whether to enable InfluxDB protocol in HTTP API.
 enable = true
 
-## Jaeger protocol options.
-[jaeger]
-## Whether to enable Jaeger protocol in HTTP API.
-enable = true
-
 ## Prometheus remote storage options
 [prom_store]
 ## Whether to enable Prometheus remote write and read in HTTP API.
@@ -170,11 +159,11 @@ dir = "/tmp/greptimedb/wal"
 ## **It's only used when the provider is `raft_engine`**.
 file_size = "128MB"
 
-## The threshold of the WAL size to trigger a purge.
+## The threshold of the WAL size to trigger a flush.
 ## **It's only used when the provider is `raft_engine`**.
 purge_threshold = "1GB"
 
-## The interval to trigger a purge.
+## The interval to trigger a flush.
 ## **It's only used when the provider is `raft_engine`**.
 purge_interval = "1m"
 
@@ -289,12 +278,10 @@ overwrite_entry_start_id = false
 
 ## Metadata storage options.
 [metadata_store]
-## The size of the metadata store log file.
-file_size = "64MB"
-## The threshold of the metadata store size to trigger a purge.
-purge_threshold = "256MB"
-## The interval of the metadata store to trigger a purge.
-purge_interval = "1m"
+## Kv file size in bytes.
+file_size = "256MB"
+## Kv purge threshold.
+purge_threshold = "4GB"
 
 ## Procedure storage options.
 [procedure]
@@ -318,7 +305,6 @@ retry_delay = "500ms"
 # secret_access_key = "123456"
 # endpoint = "https://s3.amazonaws.com"
 # region = "us-west-2"
-# enable_virtual_host_style = false
 
 # Example of using Oss as the storage.
 # [storage]
@@ -585,11 +571,6 @@ aux_path = ""
 ## The max capacity of the staging directory.
 staging_size = "2GB"
 
-## The TTL of the staging directory.
-## Defaults to 7 days.
-## Setting it to "0s" to disable TTL.
-staging_ttl = "7d"
-
 ## Cache size for inverted index metadata.
 metadata_cache_size = "64MiB"
 

cyborg/bin/bump-versions.ts

@@ -1,156 +0,0 @@
-/*
- * Copyright 2023 Greptime Team
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import * as core from "@actions/core";
-import {obtainClient} from "@/common";
-
-interface RepoConfig {
-  tokenEnv: string;
-  repo: string;
-  workflowLogic: (version: string) => [string, string] | null;
-}
-
-const REPO_CONFIGS: Record<string, RepoConfig> = {
-  website: {
-    tokenEnv: "WEBSITE_REPO_TOKEN",
-    repo: "website",
-    workflowLogic: (version: string) => {
-      // Skip nightly versions for website
-      if (version.includes('nightly')) {
-        console.log('Nightly version detected for website, skipping workflow trigger.');
-        return null;
-      }
-      return ['bump-patch-version.yml', version];
-    }
-  },
-  demo: {
-    tokenEnv: "DEMO_REPO_TOKEN",
-    repo: "demo-scene",
-    workflowLogic: (version: string) => {
-      // Skip nightly versions for demo
-      if (version.includes('nightly')) {
-        console.log('Nightly version detected for demo, skipping workflow trigger.');
-        return null;
-      }
-      return ['bump-patch-version.yml', version];
-    }
-  },
-  docs: {
-    tokenEnv: "DOCS_REPO_TOKEN",
-    repo: "docs",
-    workflowLogic: (version: string) => {
-      // Check if it's a nightly version
-      if (version.includes('nightly')) {
-        return ['bump-nightly-version.yml', version];
-      }
-
-      const parts = version.split('.');
-      if (parts.length !== 3) {
-        throw new Error('Invalid version format');
-      }
-
-      // If patch version (last number) is 0, it's a major version
-      // Return only major.minor version
-      if (parts[2] === '0') {
-        return ['bump-version.yml', `${parts[0]}.${parts[1]}`];
-      }
-
-      // Otherwise it's a patch version, use full version
-      return ['bump-patch-version.yml', version];
-    }
-  }
-};
-
-async function triggerWorkflow(repoConfig: RepoConfig, workflowId: string, version: string) {
-  const client = obtainClient(repoConfig.tokenEnv);
-  try {
-    await client.rest.actions.createWorkflowDispatch({
-      owner: "GreptimeTeam",
-      repo: repoConfig.repo,
-      workflow_id: workflowId,
-      ref: "main",
-      inputs: {
-        version,
-      },
-    });
-    console.log(`Successfully triggered ${workflowId} workflow for ${repoConfig.repo} with version ${version}`);
-  } catch (error) {
-    core.setFailed(`Failed to trigger workflow for ${repoConfig.repo}: ${error.message}`);
-    throw error;
-  }
-}
-
-async function processRepo(repoName: string, version: string) {
-  const repoConfig = REPO_CONFIGS[repoName];
-  if (!repoConfig) {
-    throw new Error(`Unknown repository: ${repoName}`);
-  }
-
-  try {
-    const workflowResult = repoConfig.workflowLogic(version);
-    if (workflowResult === null) {
-      // Skip this repo (e.g., nightly version for website)
-      return;
-    }
-
-    const [workflowId, apiVersion] = workflowResult;
-    await triggerWorkflow(repoConfig, workflowId, apiVersion);
-  } catch (error) {
-    core.setFailed(`Error processing ${repoName} with version ${version}: ${error.message}`);
-    throw error;
-  }
-}
-
-async function main() {
-  const version = process.env.VERSION;
-  if (!version) {
-    core.setFailed("VERSION environment variable is required");
-    process.exit(1);
-  }
-
-  // Remove 'v' prefix if exists
-  const cleanVersion = version.startsWith('v') ? version.slice(1) : version;
-
-  // Get target repositories from environment variable
-  // Default to both if not specified
-  const targetRepos = process.env.TARGET_REPOS?.split(',').map(repo => repo.trim()) || ['website', 'docs'];
-
-  console.log(`Processing version ${cleanVersion} for repositories: ${targetRepos.join(', ')}`);
-
-  const errors: string[] = [];
-
-  // Process each repository
-  for (const repo of targetRepos) {
-    try {
-      await processRepo(repo, cleanVersion);
-    } catch (error) {
-      errors.push(`${repo}: ${error.message}`);
-    }
-  }
-
-  if (errors.length > 0) {
-    core.setFailed(`Failed to process some repositories: ${errors.join('; ')}`);
-    process.exit(1);
-  }
-
-  console.log('All repositories processed successfully');
-}
-
-// Execute main function
-main().catch((error) => {
-  core.setFailed(`Unexpected error: ${error.message}`);
-  process.exit(1);
-});

cyborg/bin/follow-up-docs-issue.ts

@@ -55,25 +55,12 @@ async function main() {
         await client.rest.issues.addLabels({
             owner, repo, issue_number: number, labels: [labelDocsRequired],
         })
-
-        // Get available assignees for the docs repo
-        const assigneesResponse = await docsClient.rest.issues.listAssignees({
-            owner: 'GreptimeTeam',
-            repo: 'docs',
-        })
-        const validAssignees = assigneesResponse.data.map(assignee => assignee.login)
-        core.info(`Available assignees: ${validAssignees.join(', ')}`)
-
-        // Check if the actor is a valid assignee, otherwise fallback to fengjiachun
-        const assignee = validAssignees.includes(actor) ? actor : 'fengjiachun'
-        core.info(`Assigning issue to: ${assignee}`)
-
         await docsClient.rest.issues.create({
             owner: 'GreptimeTeam',
             repo: 'docs',
             title: `Update docs for ${title}`,
             body: `A document change request is generated from ${html_url}`,
-            assignee: assignee,
+            assignee: actor,
         }).then((res) => {
             core.info(`Created issue ${res.data}`)
         })

docker/docker-compose/cluster-with-etcd.yaml

@@ -43,8 +43,8 @@ services:
     command:
       - metasrv
       - start
-      - --rpc-bind-addr=0.0.0.0:3002
-      - --rpc-server-addr=metasrv:3002
+      - --bind-addr=0.0.0.0:3002
+      - --server-addr=metasrv:3002
       - --store-addrs=etcd0:2379
       - --http-addr=0.0.0.0:3000
     healthcheck:
@@ -68,8 +68,8 @@ services:
       - datanode
       - start
       - --node-id=0
-      - --rpc-bind-addr=0.0.0.0:3001
-      - --rpc-server-addr=datanode0:3001
+      - --rpc-addr=0.0.0.0:3001
+      - --rpc-hostname=datanode0:3001
       - --metasrv-addrs=metasrv:3002
       - --http-addr=0.0.0.0:5000
     volumes:
@@ -98,7 +98,7 @@ services:
       - start
       - --metasrv-addrs=metasrv:3002
       - --http-addr=0.0.0.0:4000
-      - --rpc-bind-addr=0.0.0.0:4001
+      - --rpc-addr=0.0.0.0:4001
       - --mysql-addr=0.0.0.0:4002
       - --postgres-addr=0.0.0.0:4003
     healthcheck:
@@ -123,8 +123,8 @@ services:
       - start
       - --node-id=0
       - --metasrv-addrs=metasrv:3002
-      - --rpc-bind-addr=0.0.0.0:4004
-      - --rpc-server-addr=flownode0:4004
+      - --rpc-addr=0.0.0.0:4004
+      - --rpc-hostname=flownode0:4004
       - --http-addr=0.0.0.0:4005
     depends_on:
       frontend0:

docs/how-to/how-to-profile-memory.md

@@ -4,16 +4,6 @@ This crate provides an easy approach to dump memory profiling info.
 
 ## Prerequisites
 ### jemalloc
-jeprof is already compiled in the target directory of GreptimeDB. You can find the binary and use it.
-```
-# find jeprof binary
-find . -name 'jeprof'
-# add executable permission
-chmod +x <path_to_jeprof>
-```
-The path is usually under `./target/${PROFILE}/build/tikv-jemalloc-sys-${HASH}/out/build/bin/jeprof`.
-The default version of jemalloc installed from the package manager may not have the `--collapsed` option.
-You may need to check the whether the `jeprof` version is >= `5.3.0` if you want to install it from the package manager.
 ```bash
 # for macOS
 brew install jemalloc
@@ -33,11 +23,7 @@ curl https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph
 Start GreptimeDB instance with environment variables:
 
 ```bash
-# for Linux
 MALLOC_CONF=prof:true ./target/debug/greptime standalone start
-
-# for macOS
-_RJEM_MALLOC_CONF=prof:true ./target/debug/greptime standalone start
 ```
 
 Dump memory profiling data through HTTP API:

grafana/greptimedb.json

@@ -384,8 +384,8 @@
         "rowHeight": 0.9,
         "showValue": "auto",
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "none",
+          "sort": "none"
         }
       },
       "targets": [
@@ -483,8 +483,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "pluginVersion": "10.2.3",
@@ -578,8 +578,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "pluginVersion": "10.2.3",
@@ -601,7 +601,7 @@
       "type": "timeseries"
     },
     {
-      "collapsed": false,
+      "collapsed": true,
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -684,8 +684,8 @@
               "showLegend": true
             },
             "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
+              "mode": "single",
+              "sort": "none"
             }
           },
           "targets": [
@@ -878,8 +878,8 @@
               "showLegend": true
             },
             "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
+              "mode": "single",
+              "sort": "none"
             }
           },
           "targets": [
@@ -1124,8 +1124,8 @@
               "showLegend": true
             },
             "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
+              "mode": "single",
+              "sort": "none"
             }
           },
           "targets": [
@@ -1223,8 +1223,8 @@
               "showLegend": true
             },
             "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
+              "mode": "single",
+              "sort": "none"
             }
           },
           "targets": [
@@ -1322,8 +1322,8 @@
               "showLegend": true
             },
             "tooltip": {
-              "mode": "multi",
-              "sort": "desc"
+              "mode": "single",
+              "sort": "none"
             }
           },
           "targets": [
@@ -1456,8 +1456,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -1573,8 +1573,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -1673,8 +1673,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -1773,8 +1773,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -1890,8 +1890,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2002,8 +2002,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2120,8 +2120,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2233,8 +2233,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2334,8 +2334,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2435,8 +2435,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2548,8 +2548,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2661,8 +2661,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2788,8 +2788,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2889,8 +2889,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -2990,8 +2990,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -3091,8 +3091,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -3191,8 +3191,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -3302,8 +3302,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -3432,8 +3432,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -3543,8 +3543,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -3657,8 +3657,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -3808,8 +3808,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -3909,8 +3909,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -4011,8 +4011,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [
@@ -4113,8 +4113,8 @@
           "showLegend": true
         },
         "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
+          "mode": "single",
+          "sort": "none"
         }
       },
       "targets": [

scripts/install.sh

@@ -53,54 +53,6 @@ get_arch_type() {
   esac
 }
 
-# Verify SHA256 checksum
-verify_sha256() {
-  file="$1"
-  expected_sha256="$2"
-
-  if command -v sha256sum >/dev/null 2>&1; then
-    actual_sha256=$(sha256sum "$file" | cut -d' ' -f1)
-  elif command -v shasum >/dev/null 2>&1; then
-    actual_sha256=$(shasum -a 256 "$file" | cut -d' ' -f1)
-  else
-    echo "Warning: No SHA256 verification tool found (sha256sum or shasum). Skipping checksum verification."
-    return 0
-  fi
-
-  if [ "$actual_sha256" = "$expected_sha256" ]; then
-    echo "SHA256 checksum verified successfully."
-    return 0
-  else
-    echo "Error: SHA256 checksum verification failed!"
-    echo "Expected: $expected_sha256"
-    echo "Actual: $actual_sha256"
-    return 1
-  fi
-}
-
-# Prompt for user confirmation (compatible with different shells)
-prompt_confirmation() {
-  message="$1"
-  printf "%s (y/N): " "$message"
-
-  # Try to read user input, fallback if read fails
-  answer=""
-  if read answer </dev/tty 2>/dev/null; then
-    case "$answer" in
-      [Yy]|[Yy][Ee][Ss])
-        return 0
-        ;;
-      *)
-        return 1
-        ;;
-    esac
-  else
-    echo ""
-    echo "Cannot read user input. Defaulting to No."
-    return 1
-  fi
-}
-
 download_artifact() {
   if [ -n "${OS_TYPE}" ] && [ -n "${ARCH_TYPE}" ]; then
     # Use the latest stable released version.
@@ -119,104 +71,17 @@ download_artifact() {
     fi
 
     echo "Downloading ${BIN}, OS: ${OS_TYPE}, Arch: ${ARCH_TYPE}, Version: ${VERSION}"
-    PKG_NAME="${BIN}-${OS_TYPE}-${ARCH_TYPE}-${VERSION}"
-    PACKAGE_NAME="${PKG_NAME}.tar.gz"
-    SHA256_FILE="${PKG_NAME}.sha256sum"
+    PACKAGE_NAME="${BIN}-${OS_TYPE}-${ARCH_TYPE}-${VERSION}.tar.gz"
 
     if [ -n "${PACKAGE_NAME}" ]; then
-      # Check if files already exist and prompt for override
-      if [ -f "${PACKAGE_NAME}" ]; then
-        echo "File ${PACKAGE_NAME} already exists."
-        if prompt_confirmation "Do you want to override it?"; then
-          echo "Overriding existing file..."
-          rm -f "${PACKAGE_NAME}"
-        else
-          echo "Skipping download. Using existing file."
-        fi
-      fi
-
-      if [ -f "${BIN}" ]; then
-        echo "Binary ${BIN} already exists."
-        if prompt_confirmation "Do you want to override it?"; then
-          echo "Will override existing binary..."
-          rm -f "${BIN}"
-        else
-          echo "Installation cancelled."
-          exit 0
-        fi
-      fi
-
-      # Download package if not exists
-      if [ ! -f "${PACKAGE_NAME}" ]; then
-        echo "Downloading ${PACKAGE_NAME}..."
-        # Use curl instead of wget for better compatibility
-        if command -v curl >/dev/null 2>&1; then
-          if ! curl -L -o "${PACKAGE_NAME}" "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${PACKAGE_NAME}"; then
-            echo "Error: Failed to download ${PACKAGE_NAME}"
-            exit 1
-          fi
-        elif command -v wget >/dev/null 2>&1; then
-          if ! wget -O "${PACKAGE_NAME}" "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${PACKAGE_NAME}"; then
-            echo "Error: Failed to download ${PACKAGE_NAME}"
-            exit 1
-          fi
-        else
-          echo "Error: Neither curl nor wget is available for downloading."
-          exit 1
-        fi
-      fi
-
-      # Download and verify SHA256 checksum
-      echo "Downloading SHA256 checksum..."
-      sha256_download_success=0
-      if command -v curl >/dev/null 2>&1; then
-        if curl -L -s -o "${SHA256_FILE}" "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${SHA256_FILE}" 2>/dev/null; then
-          sha256_download_success=1
-        fi
-      elif command -v wget >/dev/null 2>&1; then
-        if wget -q -O "${SHA256_FILE}" "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${SHA256_FILE}" 2>/dev/null; then
-          sha256_download_success=1
-        fi
-      fi
-
-      if [ $sha256_download_success -eq 1 ] && [ -f "${SHA256_FILE}" ]; then
-        expected_sha256=$(cat "${SHA256_FILE}" | cut -d' ' -f1)
-        if [ -n "$expected_sha256" ]; then
-          if ! verify_sha256 "${PACKAGE_NAME}" "${expected_sha256}"; then
-            echo "SHA256 verification failed. Removing downloaded file."
-            rm -f "${PACKAGE_NAME}" "${SHA256_FILE}"
-            exit 1
-          fi
-        else
-          echo "Warning: Could not parse SHA256 checksum from file."
-        fi
-        rm -f "${SHA256_FILE}"
-      else
-        echo "Warning: Could not download SHA256 checksum file. Skipping verification."
-      fi
+      wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${PACKAGE_NAME}"
 
       # Extract the binary and clean the rest.
-      echo "Extracting ${PACKAGE_NAME}..."
-      if ! tar xf "${PACKAGE_NAME}"; then
-        echo "Error: Failed to extract ${PACKAGE_NAME}"
-        exit 1
-      fi
-
-      # Find the binary in the extracted directory
-      extracted_dir="${PACKAGE_NAME%.tar.gz}"
-      if [ -f "${extracted_dir}/${BIN}" ]; then
-        mv "${extracted_dir}/${BIN}" "${PWD}/"
-        rm -f "${PACKAGE_NAME}"
-        rm -rf "${extracted_dir}"
-        chmod +x "${BIN}"
-        echo "Installation completed successfully!"
-        echo "Run './${BIN} --help' to get started"
-      else
-        echo "Error: Binary ${BIN} not found in extracted archive"
-        rm -f "${PACKAGE_NAME}"
-        rm -rf "${extracted_dir}"
-        exit 1
-      fi
+      tar xvf "${PACKAGE_NAME}" && \
+      mv "${PACKAGE_NAME%.tar.gz}/${BIN}" "${PWD}" && \
+      rm -r "${PACKAGE_NAME}" && \
+      rm -r "${PACKAGE_NAME%.tar.gz}" && \
+      echo "Run './${BIN} --help' to get started"
     fi
   fi
 }

src/api/Cargo.toml

@@ -15,10 +15,13 @@ common-macro.workspace = true
 common-time.workspace = true
 datatypes.workspace = true
 greptime-proto.workspace = true
-paste.workspace = true
+paste = "1.0"
 prost.workspace = true
 serde_json.workspace = true
 snafu.workspace = true
 
 [build-dependencies]
 tonic-build = "0.11"
+
+[dev-dependencies]
+paste = "1.0"

src/api/src/v1/column_def.rs

@@ -15,10 +15,10 @@
 use std::collections::HashMap;
 
 use datatypes::schema::{
-    ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextOptions, SkippingIndexType,
-    COMMENT_KEY, FULLTEXT_KEY, INVERTED_INDEX_KEY, SKIPPING_INDEX_KEY,
+    ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextOptions, COMMENT_KEY,
+    FULLTEXT_KEY, INVERTED_INDEX_KEY, SKIPPING_INDEX_KEY,
 };
-use greptime_proto::v1::{Analyzer, SkippingIndexType as PbSkippingIndexType};
+use greptime_proto::v1::Analyzer;
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
@@ -121,13 +121,6 @@ pub fn as_fulltext_option(analyzer: Analyzer) -> FulltextAnalyzer {
     }
 }
 
-/// Tries to construct a `SkippingIndexType` from the given skipping index type.
-pub fn as_skipping_index_type(skipping_index_type: PbSkippingIndexType) -> SkippingIndexType {
-    match skipping_index_type {
-        PbSkippingIndexType::BloomFilter => SkippingIndexType::BloomFilter,
-    }
-}
-
 #[cfg(test)]
 mod tests {
 

src/catalog/Cargo.toml

@@ -15,7 +15,7 @@ api.workspace = true
 arrow.workspace = true
 arrow-schema.workspace = true
 async-stream.workspace = true
-async-trait.workspace = true
+async-trait = "0.1"
 bytes.workspace = true
 common-catalog.workspace = true
 common-error.workspace = true
@@ -31,7 +31,7 @@ common-version.workspace = true
 dashmap.workspace = true
 datafusion.workspace = true
 datatypes.workspace = true
-futures.workspace = true
+futures = "0.3"
 futures-util.workspace = true
 humantime.workspace = true
 itertools.workspace = true
@@ -39,7 +39,7 @@ lazy_static.workspace = true
 meta-client.workspace = true
 moka = { workspace = true, features = ["future", "sync"] }
 partition.workspace = true
-paste.workspace = true
+paste = "1.0"
 prometheus.workspace = true
 rustc-hash.workspace = true
 serde_json.workspace = true
@@ -49,7 +49,7 @@ sql.workspace = true
 store-api.workspace = true
 table.workspace = true
 tokio.workspace = true
-tokio-stream.workspace = true
+tokio-stream = "0.1"
 
 [dev-dependencies]
 cache.workspace = true

src/catalog/src/system_schema/information_schema/key_column_usage.rs

@@ -228,6 +228,12 @@ impl InformationSchemaKeyColumnUsageBuilder {
                 let keys = &table_info.meta.primary_key_indices;
                 let schema = table.schema();
 
+                // For compatibility, use primary key columns as inverted index columns.
+                let pk_as_inverted_index = !schema
+                    .column_schemas()
+                    .iter()
+                    .any(|c| c.has_inverted_index_key());
+
                 for (idx, column) in schema.column_schemas().iter().enumerate() {
                     let mut constraints = vec![];
                     if column.is_time_index() {
@@ -245,6 +251,10 @@ impl InformationSchemaKeyColumnUsageBuilder {
                     // TODO(dimbtp): foreign key constraint not supported yet
                     if keys.contains(&idx) {
                         constraints.push(PRI_CONSTRAINT_NAME);
+
+                        if pk_as_inverted_index {
+                            constraints.push(INVERTED_INDEX_CONSTRAINT_NAME);
+                        }
                     }
                     if column.is_inverted_indexed() {
                         constraints.push(INVERTED_INDEX_CONSTRAINT_NAME);

src/cli/src/bench.rs

@@ -24,11 +24,10 @@ use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::etcd::EtcdStore;
 use common_meta::kv_backend::memory::MemoryKvBackend;
 #[cfg(feature = "pg_kvbackend")]
-use common_meta::kv_backend::rds::PgStore;
+use common_meta::kv_backend::postgres::PgStore;
 use common_meta::peer::Peer;
 use common_meta::rpc::router::{Region, RegionRoute};
 use common_telemetry::info;
-use common_wal::options::WalOptions;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use rand::Rng;
@@ -185,7 +184,7 @@ fn create_region_routes(regions: Vec<RegionNumber>) -> Vec<RegionRoute> {
     region_routes
 }
 
-fn create_region_wal_options(regions: Vec<RegionNumber>) -> HashMap<RegionNumber, WalOptions> {
+fn create_region_wal_options(regions: Vec<RegionNumber>) -> HashMap<RegionNumber, String> {
     // TODO(niebayes): construct region wal options for benchmark.
     let _ = regions;
     HashMap::default()

src/cli/src/bench/metadata.rs

@@ -49,12 +49,7 @@ impl TableMetadataBencher {
 
                 let regions: Vec<_> = (0..64).collect();
                 let region_routes = create_region_routes(regions.clone());
-                let region_wal_options = create_region_wal_options(regions)
-                    .into_iter()
-                    .map(|(region_id, wal_options)| {
-                        (region_id, serde_json::to_string(&wal_options).unwrap())
-                    })
-                    .collect();
+                let region_wal_options = create_region_wal_options(regions);
 
                 let start = Instant::now();
 
@@ -114,17 +109,9 @@ impl TableMetadataBencher {
                 let table_info = table_info.unwrap();
                 let table_route = table_route.unwrap();
                 let table_id = table_info.table_info.ident.table_id;
-
-                let regions: Vec<_> = (0..64).collect();
-                let region_wal_options = create_region_wal_options(regions);
                 let _ = self
                     .table_metadata_manager
-                    .delete_table_metadata(
-                        table_id,
-                        &table_info.table_name(),
-                        &table_route,
-                        &region_wal_options,
-                    )
+                    .delete_table_metadata(table_id, &table_info.table_name(), &table_route)
                     .await;
                 start.elapsed()
             },

src/cmd/src/datanode.rs

@@ -126,14 +126,10 @@ impl SubCommand {
 struct StartCommand {
     #[clap(long)]
     node_id: Option<u64>,
-    /// The address to bind the gRPC server.
-    #[clap(long, alias = "rpc-addr")]
-    rpc_bind_addr: Option<String>,
-    /// The address advertised to the metasrv, and used for connections from outside the host.
-    /// If left empty or unset, the server will automatically use the IP address of the first network interface
-    /// on the host, with the same port number as the one specified in `rpc_bind_addr`.
-    #[clap(long, alias = "rpc-hostname")]
-    rpc_server_addr: Option<String>,
+    #[clap(long)]
+    rpc_addr: Option<String>,
+    #[clap(long)]
+    rpc_hostname: Option<String>,
     #[clap(long, value_delimiter = ',', num_args = 1..)]
     metasrv_addrs: Option<Vec<String>>,
     #[clap(short, long)]
@@ -185,18 +181,18 @@ impl StartCommand {
             tokio_console_addr: global_options.tokio_console_addr.clone(),
         };
 
-        if let Some(addr) = &self.rpc_bind_addr {
-            opts.grpc.bind_addr.clone_from(addr);
+        if let Some(addr) = &self.rpc_addr {
+            opts.grpc.addr.clone_from(addr);
         } else if let Some(addr) = &opts.rpc_addr {
             warn!("Use the deprecated attribute `DatanodeOptions.rpc_addr`, please use `grpc.addr` instead.");
-            opts.grpc.bind_addr.clone_from(addr);
+            opts.grpc.addr.clone_from(addr);
         }
 
-        if let Some(server_addr) = &self.rpc_server_addr {
-            opts.grpc.server_addr.clone_from(server_addr);
-        } else if let Some(server_addr) = &opts.rpc_hostname {
+        if let Some(hostname) = &self.rpc_hostname {
+            opts.grpc.hostname.clone_from(hostname);
+        } else if let Some(hostname) = &opts.rpc_hostname {
             warn!("Use the deprecated attribute `DatanodeOptions.rpc_hostname`, please use `grpc.hostname` instead.");
-            opts.grpc.server_addr.clone_from(server_addr);
+            opts.grpc.hostname.clone_from(hostname);
         }
 
         if let Some(runtime_size) = opts.rpc_runtime_size {
@@ -281,7 +277,7 @@ impl StartCommand {
 
         let plugin_opts = opts.plugins;
         let mut opts = opts.component;
-        opts.grpc.detect_server_addr();
+        opts.grpc.detect_hostname();
         let mut plugins = Plugins::new();
         plugins::setup_datanode_plugins(&mut plugins, &plugin_opts, &opts)
             .await
@@ -361,8 +357,8 @@ mod tests {
             rpc_addr = "127.0.0.1:4001"
             rpc_hostname = "192.168.0.1"
             [grpc]
-            bind_addr = "127.0.0.1:3001"
-            server_addr = "127.0.0.1"
+            addr = "127.0.0.1:3001"
+            hostname = "127.0.0.1"
             runtime_size = 8
         "#;
         write!(file, "{}", toml_str).unwrap();
@@ -373,8 +369,8 @@ mod tests {
         };
 
         let options = cmd.load_options(&Default::default()).unwrap().component;
-        assert_eq!("127.0.0.1:4001".to_string(), options.grpc.bind_addr);
-        assert_eq!("192.168.0.1".to_string(), options.grpc.server_addr);
+        assert_eq!("127.0.0.1:4001".to_string(), options.grpc.addr);
+        assert_eq!("192.168.0.1".to_string(), options.grpc.hostname);
     }
 
     #[test]
@@ -435,7 +431,7 @@ mod tests {
 
         let options = cmd.load_options(&Default::default()).unwrap().component;
 
-        assert_eq!("127.0.0.1:3001".to_string(), options.grpc.bind_addr);
+        assert_eq!("127.0.0.1:3001".to_string(), options.grpc.addr);
         assert_eq!(Some(42), options.node_id);
 
         let DatanodeWalConfig::RaftEngine(raft_engine_config) = options.wal else {
@@ -649,7 +645,7 @@ mod tests {
                     opts.http.addr,
                     DatanodeOptions::default().component.http.addr
                 );
-                assert_eq!(opts.grpc.server_addr, "10.103.174.219");
+                assert_eq!(opts.grpc.hostname, "10.103.174.219");
             },
         );
     }

src/cmd/src/flownode.rs

@@ -129,13 +129,11 @@ struct StartCommand {
     #[clap(long)]
     node_id: Option<u64>,
     /// Bind address for the gRPC server.
-    #[clap(long, alias = "rpc-addr")]
-    rpc_bind_addr: Option<String>,
-    /// The address advertised to the metasrv, and used for connections from outside the host.
-    /// If left empty or unset, the server will automatically use the IP address of the first network interface
-    /// on the host, with the same port number as the one specified in `rpc_bind_addr`.
-    #[clap(long, alias = "rpc-hostname")]
-    rpc_server_addr: Option<String>,
+    #[clap(long)]
+    rpc_addr: Option<String>,
+    /// Hostname for the gRPC server.
+    #[clap(long)]
+    rpc_hostname: Option<String>,
     /// Metasrv address list;
     #[clap(long, value_delimiter = ',', num_args = 1..)]
     metasrv_addrs: Option<Vec<String>>,
@@ -186,12 +184,12 @@ impl StartCommand {
             tokio_console_addr: global_options.tokio_console_addr.clone(),
         };
 
-        if let Some(addr) = &self.rpc_bind_addr {
-            opts.grpc.bind_addr.clone_from(addr);
+        if let Some(addr) = &self.rpc_addr {
+            opts.grpc.addr.clone_from(addr);
         }
 
-        if let Some(server_addr) = &self.rpc_server_addr {
-            opts.grpc.server_addr.clone_from(server_addr);
+        if let Some(hostname) = &self.rpc_hostname {
+            opts.grpc.hostname.clone_from(hostname);
         }
 
         if let Some(node_id) = self.node_id {
@@ -239,7 +237,7 @@ impl StartCommand {
         info!("Flownode options: {:#?}", opts);
 
         let mut opts = opts.component;
-        opts.grpc.detect_server_addr();
+        opts.grpc.detect_hostname();
 
         // TODO(discord9): make it not optionale after cluster id is required
         let cluster_id = opts.cluster_id.unwrap_or(0);

src/cmd/src/frontend.rs

@@ -136,19 +136,13 @@ impl SubCommand {
 
 #[derive(Debug, Default, Parser)]
 pub struct StartCommand {
-    /// The address to bind the gRPC server.
-    #[clap(long, alias = "rpc-addr")]
-    rpc_bind_addr: Option<String>,
-    /// The address advertised to the metasrv, and used for connections from outside the host.
-    /// If left empty or unset, the server will automatically use the IP address of the first network interface
-    /// on the host, with the same port number as the one specified in `rpc_bind_addr`.
-    #[clap(long, alias = "rpc-hostname")]
-    rpc_server_addr: Option<String>,
     #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
     http_timeout: Option<u64>,
     #[clap(long)]
+    rpc_addr: Option<String>,
+    #[clap(long)]
     mysql_addr: Option<String>,
     #[clap(long)]
     postgres_addr: Option<String>,
@@ -224,15 +218,11 @@ impl StartCommand {
             opts.http.disable_dashboard = disable_dashboard;
         }
 
-        if let Some(addr) = &self.rpc_bind_addr {
-            opts.grpc.bind_addr.clone_from(addr);
+        if let Some(addr) = &self.rpc_addr {
+            opts.grpc.addr.clone_from(addr);
             opts.grpc.tls = tls_opts.clone();
         }
 
-        if let Some(addr) = &self.rpc_server_addr {
-            opts.grpc.server_addr.clone_from(addr);
-        }
-
         if let Some(addr) = &self.mysql_addr {
             opts.mysql.enable = true;
             opts.mysql.addr.clone_from(addr);
@@ -279,7 +269,7 @@ impl StartCommand {
 
         let plugin_opts = opts.plugins;
         let mut opts = opts.component;
-        opts.grpc.detect_server_addr();
+        opts.grpc.detect_hostname();
         let mut plugins = Plugins::new();
         plugins::setup_frontend_plugins(&mut plugins, &plugin_opts, &opts)
             .await
@@ -423,7 +413,7 @@ mod tests {
 
         let default_opts = FrontendOptions::default().component;
 
-        assert_eq!(opts.grpc.bind_addr, default_opts.grpc.bind_addr);
+        assert_eq!(opts.grpc.addr, default_opts.grpc.addr);
         assert!(opts.mysql.enable);
         assert_eq!(opts.mysql.runtime_size, default_opts.mysql.runtime_size);
         assert!(opts.postgres.enable);
@@ -614,7 +604,7 @@ mod tests {
                 assert_eq!(fe_opts.http.addr, "127.0.0.1:14000");
 
                 // Should be default value.
-                assert_eq!(fe_opts.grpc.bind_addr, GrpcOptions::default().bind_addr);
+                assert_eq!(fe_opts.grpc.addr, GrpcOptions::default().addr);
             },
         );
     }

src/cmd/src/metasrv.rs

@@ -42,7 +42,7 @@ pub struct Instance {
 }
 
 impl Instance {
-    pub fn new(instance: MetasrvInstance, guard: Vec<WorkerGuard>) -> Self {
+    fn new(instance: MetasrvInstance, guard: Vec<WorkerGuard>) -> Self {
         Self {
             instance,
             _guard: guard,
@@ -133,15 +133,11 @@ impl SubCommand {
 
 #[derive(Debug, Default, Parser)]
 struct StartCommand {
-    /// The address to bind the gRPC server.
-    #[clap(long, alias = "bind-addr")]
-    rpc_bind_addr: Option<String>,
-    /// The communication server address for the frontend and datanode to connect to metasrv.
-    /// If left empty or unset, the server will automatically use the IP address of the first network interface
-    /// on the host, with the same port number as the one specified in `rpc_bind_addr`.
-    #[clap(long, alias = "server-addr")]
-    rpc_server_addr: Option<String>,
-    #[clap(long, alias = "store-addr", value_delimiter = ',', num_args = 1..)]
+    #[clap(long)]
+    bind_addr: Option<String>,
+    #[clap(long)]
+    server_addr: Option<String>,
+    #[clap(long, aliases = ["store-addr"], value_delimiter = ',', num_args = 1..)]
     store_addrs: Option<Vec<String>>,
     #[clap(short, long)]
     config_file: Option<String>,
@@ -205,11 +201,11 @@ impl StartCommand {
             tokio_console_addr: global_options.tokio_console_addr.clone(),
         };
 
-        if let Some(addr) = &self.rpc_bind_addr {
+        if let Some(addr) = &self.bind_addr {
             opts.bind_addr.clone_from(addr);
         }
 
-        if let Some(addr) = &self.rpc_server_addr {
+        if let Some(addr) = &self.server_addr {
             opts.server_addr.clone_from(addr);
         }
 
@@ -273,13 +269,11 @@ impl StartCommand {
         log_versions(version(), short_version(), APP_NAME);
 
         info!("Metasrv start command: {:#?}", self);
+        info!("Metasrv options: {:#?}", opts);
 
         let plugin_opts = opts.plugins;
         let mut opts = opts.component;
         opts.detect_server_addr();
-
-        info!("Metasrv options: {:#?}", opts);
-
         let mut plugins = Plugins::new();
         plugins::setup_metasrv_plugins(&mut plugins, &plugin_opts, &opts)
             .await
@@ -312,8 +306,8 @@ mod tests {
     #[test]
     fn test_read_from_cmd() {
         let cmd = StartCommand {
-            rpc_bind_addr: Some("127.0.0.1:3002".to_string()),
-            rpc_server_addr: Some("127.0.0.1:3002".to_string()),
+            bind_addr: Some("127.0.0.1:3002".to_string()),
+            server_addr: Some("127.0.0.1:3002".to_string()),
             store_addrs: Some(vec!["127.0.0.1:2380".to_string()]),
             selector: Some("LoadBased".to_string()),
             ..Default::default()
@@ -387,8 +381,8 @@ mod tests {
     #[test]
     fn test_load_log_options_from_cli() {
         let cmd = StartCommand {
-            rpc_bind_addr: Some("127.0.0.1:3002".to_string()),
-            rpc_server_addr: Some("127.0.0.1:3002".to_string()),
+            bind_addr: Some("127.0.0.1:3002".to_string()),
+            server_addr: Some("127.0.0.1:3002".to_string()),
             store_addrs: Some(vec!["127.0.0.1:2380".to_string()]),
             selector: Some("LoadBased".to_string()),
             ..Default::default()

src/cmd/src/standalone.rs

@@ -60,8 +60,7 @@ use frontend::instance::builder::FrontendBuilder;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
 use frontend::server::Services;
 use frontend::service_config::{
-    InfluxdbOptions, JaegerOptions, MysqlOptions, OpentsdbOptions, PostgresOptions,
-    PromStoreOptions,
+    InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
 use meta_srv::metasrv::{FLOW_ID_SEQ, TABLE_ID_SEQ};
 use mito2::config::MitoConfig;
@@ -141,7 +140,6 @@ pub struct StandaloneOptions {
     pub postgres: PostgresOptions,
     pub opentsdb: OpentsdbOptions,
     pub influxdb: InfluxdbOptions,
-    pub jaeger: JaegerOptions,
     pub prom_store: PromStoreOptions,
     pub wal: DatanodeWalConfig,
     pub storage: StorageConfig,
@@ -171,7 +169,6 @@ impl Default for StandaloneOptions {
             postgres: PostgresOptions::default(),
             opentsdb: OpentsdbOptions::default(),
             influxdb: InfluxdbOptions::default(),
-            jaeger: JaegerOptions::default(),
             prom_store: PromStoreOptions::default(),
             wal: DatanodeWalConfig::default(),
             storage: StorageConfig::default(),
@@ -220,7 +217,6 @@ impl StandaloneOptions {
             postgres: cloned_opts.postgres,
             opentsdb: cloned_opts.opentsdb,
             influxdb: cloned_opts.influxdb,
-            jaeger: cloned_opts.jaeger,
             prom_store: cloned_opts.prom_store,
             meta_client: None,
             logging: cloned_opts.logging,
@@ -333,8 +329,8 @@ impl App for Instance {
 pub struct StartCommand {
     #[clap(long)]
     http_addr: Option<String>,
-    #[clap(long, alias = "rpc-addr")]
-    rpc_bind_addr: Option<String>,
+    #[clap(long)]
+    rpc_addr: Option<String>,
     #[clap(long)]
     mysql_addr: Option<String>,
     #[clap(long)]
@@ -411,9 +407,9 @@ impl StartCommand {
             opts.storage.data_home.clone_from(data_home);
         }
 
-        if let Some(addr) = &self.rpc_bind_addr {
+        if let Some(addr) = &self.rpc_addr {
             // frontend grpc addr conflict with datanode default grpc addr
-            let datanode_grpc_addr = DatanodeOptions::default().grpc.bind_addr;
+            let datanode_grpc_addr = DatanodeOptions::default().grpc.addr;
             if addr.eq(&datanode_grpc_addr) {
                 return IllegalConfigSnafu {
                     msg: format!(
@@ -421,7 +417,7 @@ impl StartCommand {
                     ),
                 }.fail();
             }
-            opts.grpc.bind_addr.clone_from(addr)
+            opts.grpc.addr.clone_from(addr)
         }
 
         if let Some(addr) = &self.mysql_addr {
@@ -468,7 +464,7 @@ impl StartCommand {
         let mut plugins = Plugins::new();
         let plugin_opts = opts.plugins;
         let mut opts = opts.component;
-        opts.grpc.detect_server_addr();
+        opts.grpc.detect_hostname();
         let fe_opts = opts.frontend_options();
         let dn_opts = opts.datanode_options();
 
@@ -490,8 +486,8 @@ impl StartCommand {
         let metadata_dir = metadata_store_dir(data_home);
         let (kv_backend, procedure_manager) = FeInstance::try_build_standalone_components(
             metadata_dir,
-            opts.metadata_store,
-            opts.procedure,
+            opts.metadata_store.clone(),
+            opts.procedure.clone(),
         )
         .await
         .context(StartFrontendSnafu)?;
@@ -911,7 +907,7 @@ mod tests {
         assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
         assert_eq!(Duration::from_secs(33), fe_opts.http.timeout);
         assert_eq!(ReadableSize::mb(128), fe_opts.http.body_limit);
-        assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.bind_addr);
+        assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.addr);
         assert!(fe_opts.mysql.enable);
         assert_eq!("127.0.0.1:4002", fe_opts.mysql.addr);
         assert_eq!(2, fe_opts.mysql.runtime_size);
@@ -1041,7 +1037,7 @@ mod tests {
                 assert_eq!(ReadableSize::mb(64), fe_opts.http.body_limit);
 
                 // Should be default value.
-                assert_eq!(fe_opts.grpc.bind_addr, GrpcOptions::default().bind_addr);
+                assert_eq!(fe_opts.grpc.addr, GrpcOptions::default().addr);
             },
         );
     }

src/cmd/tests/cli.rs

@@ -63,7 +63,7 @@ mod tests {
             .args([
                 "datanode",
                 "start",
-                "--rpc-bind-addr=0.0.0.0:4321",
+                "--rpc-addr=0.0.0.0:4321",
                 "--node-id=1",
                 &format!("--data-home={}", data_home.path().display()),
                 &format!("--wal-dir={}", wal_dir.path().display()),
@@ -80,7 +80,7 @@ mod tests {
             "--log-level=off",
             "cli",
             "attach",
-            "--grpc-bind-addr=0.0.0.0:4321",
+            "--grpc-addr=0.0.0.0:4321",
             // history commands can sneaky into stdout and mess up our tests, so disable it
             "--disable-helper",
         ]);

src/cmd/tests/load_config_test.rs

@@ -17,6 +17,9 @@ use std::time::Duration;
 use cmd::options::GreptimeOptions;
 use cmd::standalone::StandaloneOptions;
 use common_config::Configurable;
+use common_grpc::channel_manager::{
+    DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
+};
 use common_options::datanode::{ClientOptions, DatanodeClientOptions};
 use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, DEFAULT_OTLP_ENDPOINT};
 use common_wal::config::raft_engine::RaftEngineConfig;
@@ -88,8 +91,13 @@ fn test_load_datanode_example_config() {
                 ..Default::default()
             },
             grpc: GrpcOptions::default()
-                .with_bind_addr("127.0.0.1:3001")
-                .with_server_addr("127.0.0.1:3001"),
+                .with_addr("127.0.0.1:3001")
+                .with_hostname("127.0.0.1:3001"),
+            rpc_addr: Some("127.0.0.1:3001".to_string()),
+            rpc_hostname: Some("127.0.0.1".to_string()),
+            rpc_runtime_size: Some(8),
+            rpc_max_recv_message_size: Some(DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE),
+            rpc_max_send_message_size: Some(DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE),
             ..Default::default()
         },
         ..Default::default()
@@ -136,9 +144,7 @@ fn test_load_frontend_example_config() {
                 remote_write: Some(Default::default()),
                 ..Default::default()
             },
-            grpc: GrpcOptions::default()
-                .with_bind_addr("127.0.0.1:4001")
-                .with_server_addr("127.0.0.1:4001"),
+            grpc: GrpcOptions::default().with_hostname("127.0.0.1:4001"),
             http: HttpOptions {
                 cors_allowed_origins: vec!["https://example.com".to_string()],
                 ..Default::default()

src/common/base/Cargo.toml

@@ -18,7 +18,7 @@ bytes.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 futures.workspace = true
-paste.workspace = true
+paste = "1.0"
 pin-project.workspace = true
 rand.workspace = true
 serde = { version = "1.0", features = ["derive"] }

src/common/config/Cargo.toml

@@ -12,11 +12,9 @@ common-base.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 config.workspace = true
-humantime-serde.workspace = true
 num_cpus.workspace = true
 serde.workspace = true
 serde_json.workspace = true
-serde_with.workspace = true
 snafu.workspace = true
 sysinfo.workspace = true
 toml.workspace = true

src/common/config/src/lib.rs

@@ -16,8 +16,6 @@ pub mod config;
 pub mod error;
 pub mod utils;
 
-use std::time::Duration;
-
 use common_base::readable_size::ReadableSize;
 pub use config::*;
 use serde::{Deserialize, Serialize};
@@ -36,27 +34,22 @@ pub enum Mode {
     Distributed,
 }
 
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct KvBackendConfig {
-    /// The size of the metadata store backend log file.
+    // Kv file size in bytes
     pub file_size: ReadableSize,
-    /// The threshold of the metadata store size to trigger a purge.
+    // Kv purge threshold in bytes
     pub purge_threshold: ReadableSize,
-    /// The interval of the metadata store to trigger a purge.
-    #[serde(with = "humantime_serde")]
-    pub purge_interval: Duration,
 }
 
 impl Default for KvBackendConfig {
     fn default() -> Self {
         Self {
-            // The log file size 64MB
-            file_size: ReadableSize::mb(64),
-            // The log purge threshold 256MB
-            purge_threshold: ReadableSize::mb(256),
-            // The log purge interval 1m
-            purge_interval: Duration::from_secs(60),
+            // log file size 256MB
+            file_size: ReadableSize::mb(256),
+            // purge threshold 4GB
+            purge_threshold: ReadableSize::gb(4),
         }
     }
 }

src/common/datasource/Cargo.toml

@@ -35,7 +35,7 @@ orc-rust = { version = "0.5", default-features = false, features = [
     "async",
 ] }
 parquet.workspace = true
-paste.workspace = true
+paste = "1.0"
 rand.workspace = true
 regex = "1.7"
 serde.workspace = true

src/common/function/Cargo.toml

@@ -12,11 +12,9 @@ default = ["geo"]
 geo = ["geohash", "h3o", "s2", "wkt", "geo-types", "dep:geo"]
 
 [dependencies]
-ahash = "0.8"
 api.workspace = true
 arc-swap = "1.0"
 async-trait.workspace = true
-bincode = "1.3"
 common-base.workspace = true
 common-catalog.workspace = true
 common-error.workspace = true
@@ -34,13 +32,12 @@ geo = { version = "0.29", optional = true }
 geo-types = { version = "0.7", optional = true }
 geohash = { version = "0.13", optional = true }
 h3o = { version = "0.6", optional = true }
-hyperloglogplus = "0.4"
 jsonb.workspace = true
 nalgebra.workspace = true
 num = "0.4"
 num-traits = "0.2"
 once_cell.workspace = true
-paste.workspace = true
+paste = "1.0"
 s2 = { version = "0.0.12", optional = true }
 serde.workspace = true
 serde_json.workspace = true
@@ -50,7 +47,6 @@ sql.workspace = true
 statrs = "0.16"
 store-api.workspace = true
 table.workspace = true
-uddsketch = { git = "https://github.com/GreptimeTeam/timescaledb-toolkit.git", rev = "84828fe8fb494a6a61412a3da96517fc80f7bb20" }
 wkt = { version = "0.11", optional = true }
 
 [dev-dependencies]

src/common/function/src/aggr/hll.rs

@@ -1,319 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use common_query::prelude::*;
-use common_telemetry::trace;
-use datafusion::arrow::array::ArrayRef;
-use datafusion::common::cast::{as_binary_array, as_string_array};
-use datafusion::common::not_impl_err;
-use datafusion::error::{DataFusionError, Result as DfResult};
-use datafusion::logical_expr::function::AccumulatorArgs;
-use datafusion::logical_expr::{Accumulator as DfAccumulator, AggregateUDF};
-use datafusion::prelude::create_udaf;
-use datatypes::arrow::datatypes::DataType;
-use hyperloglogplus::{HyperLogLog, HyperLogLogPlus};
-
-use crate::utils::FixedRandomState;
-
-pub const HLL_NAME: &str = "hll";
-pub const HLL_MERGE_NAME: &str = "hll_merge";
-
-const DEFAULT_PRECISION: u8 = 14;
-
-pub(crate) type HllStateType = HyperLogLogPlus<String, FixedRandomState>;
-
-pub struct HllState {
-    hll: HllStateType,
-}
-
-impl std::fmt::Debug for HllState {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "HllState<Opaque>")
-    }
-}
-
-impl Default for HllState {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl HllState {
-    pub fn new() -> Self {
-        Self {
-            // Safety: the DEFAULT_PRECISION is fixed and valid
-            hll: HllStateType::new(DEFAULT_PRECISION, FixedRandomState::new()).unwrap(),
-        }
-    }
-
-    /// Create a UDF for the `hll` function.
-    ///
-    /// `hll` accepts a string column and aggregates the
-    /// values into a HyperLogLog state.
-    pub fn state_udf_impl() -> AggregateUDF {
-        create_udaf(
-            HLL_NAME,
-            vec![DataType::Utf8],
-            Arc::new(DataType::Binary),
-            Volatility::Immutable,
-            Arc::new(Self::create_accumulator),
-            Arc::new(vec![DataType::Binary]),
-        )
-    }
-
-    /// Create a UDF for the `hll_merge` function.
-    ///
-    /// `hll_merge` accepts a binary column of states generated by `hll`
-    /// and merges them into a single state.
-    pub fn merge_udf_impl() -> AggregateUDF {
-        create_udaf(
-            HLL_MERGE_NAME,
-            vec![DataType::Binary],
-            Arc::new(DataType::Binary),
-            Volatility::Immutable,
-            Arc::new(Self::create_merge_accumulator),
-            Arc::new(vec![DataType::Binary]),
-        )
-    }
-
-    fn update(&mut self, value: &str) {
-        self.hll.insert(value);
-    }
-
-    fn merge(&mut self, raw: &[u8]) {
-        if let Ok(serialized) = bincode::deserialize::<HllStateType>(raw) {
-            if let Ok(()) = self.hll.merge(&serialized) {
-                return;
-            }
-        }
-        trace!("Warning: Failed to merge HyperLogLog from {:?}", raw);
-    }
-
-    fn create_accumulator(acc_args: AccumulatorArgs) -> DfResult<Box<dyn DfAccumulator>> {
-        let data_type = acc_args.exprs[0].data_type(acc_args.schema)?;
-
-        match data_type {
-            DataType::Utf8 => Ok(Box::new(HllState::new())),
-            other => not_impl_err!("{HLL_NAME} does not support data type: {other}"),
-        }
-    }
-
-    fn create_merge_accumulator(acc_args: AccumulatorArgs) -> DfResult<Box<dyn DfAccumulator>> {
-        let data_type = acc_args.exprs[0].data_type(acc_args.schema)?;
-
-        match data_type {
-            DataType::Binary => Ok(Box::new(HllState::new())),
-            other => not_impl_err!("{HLL_MERGE_NAME} does not support data type: {other}"),
-        }
-    }
-}
-
-impl DfAccumulator for HllState {
-    fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> {
-        let array = &values[0];
-
-        match array.data_type() {
-            DataType::Utf8 => {
-                let string_array = as_string_array(array)?;
-                for value in string_array.iter().flatten() {
-                    self.update(value);
-                }
-            }
-            DataType::Binary => {
-                let binary_array = as_binary_array(array)?;
-                for v in binary_array.iter().flatten() {
-                    self.merge(v);
-                }
-            }
-            _ => {
-                return not_impl_err!(
-                    "HLL functions do not support data type: {}",
-                    array.data_type()
-                )
-            }
-        }
-
-        Ok(())
-    }
-
-    fn evaluate(&mut self) -> DfResult<ScalarValue> {
-        Ok(ScalarValue::Binary(Some(
-            bincode::serialize(&self.hll).map_err(|e| {
-                DataFusionError::Internal(format!("Failed to serialize HyperLogLog: {}", e))
-            })?,
-        )))
-    }
-
-    fn size(&self) -> usize {
-        std::mem::size_of_val(&self.hll)
-    }
-
-    fn state(&mut self) -> DfResult<Vec<ScalarValue>> {
-        Ok(vec![ScalarValue::Binary(Some(
-            bincode::serialize(&self.hll).map_err(|e| {
-                DataFusionError::Internal(format!("Failed to serialize HyperLogLog: {}", e))
-            })?,
-        ))])
-    }
-
-    fn merge_batch(&mut self, states: &[ArrayRef]) -> DfResult<()> {
-        let array = &states[0];
-        let binary_array = as_binary_array(array)?;
-        for v in binary_array.iter().flatten() {
-            self.merge(v);
-        }
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use datafusion::arrow::array::{BinaryArray, StringArray};
-
-    use super::*;
-
-    #[test]
-    fn test_hll_basic() {
-        let mut state = HllState::new();
-        state.update("1");
-        state.update("2");
-        state.update("3");
-
-        let result = state.evaluate().unwrap();
-        if let ScalarValue::Binary(Some(bytes)) = result {
-            let mut hll: HllStateType = bincode::deserialize(&bytes).unwrap();
-            assert_eq!(hll.count().trunc() as u32, 3);
-        } else {
-            panic!("Expected binary scalar value");
-        }
-    }
-
-    #[test]
-    fn test_hll_roundtrip() {
-        let mut state = HllState::new();
-        state.update("1");
-        state.update("2");
-
-        // Serialize
-        let serialized = state.evaluate().unwrap();
-
-        // Create new state and merge the serialized data
-        let mut new_state = HllState::new();
-        if let ScalarValue::Binary(Some(bytes)) = &serialized {
-            new_state.merge(bytes);
-
-            // Verify the merged state matches original
-            let result = new_state.evaluate().unwrap();
-            if let ScalarValue::Binary(Some(new_bytes)) = result {
-                let mut original: HllStateType = bincode::deserialize(bytes).unwrap();
-                let mut merged: HllStateType = bincode::deserialize(&new_bytes).unwrap();
-                assert_eq!(original.count(), merged.count());
-            } else {
-                panic!("Expected binary scalar value");
-            }
-        } else {
-            panic!("Expected binary scalar value");
-        }
-    }
-
-    #[test]
-    fn test_hll_batch_update() {
-        let mut state = HllState::new();
-
-        // Test string values
-        let str_values = vec!["a", "b", "c", "d", "e", "f", "g", "h", "i"];
-        let str_array = Arc::new(StringArray::from(str_values)) as ArrayRef;
-        state.update_batch(&[str_array]).unwrap();
-
-        let result = state.evaluate().unwrap();
-        if let ScalarValue::Binary(Some(bytes)) = result {
-            let mut hll: HllStateType = bincode::deserialize(&bytes).unwrap();
-            assert_eq!(hll.count().trunc() as u32, 9);
-        } else {
-            panic!("Expected binary scalar value");
-        }
-    }
-
-    #[test]
-    fn test_hll_merge_batch() {
-        let mut state1 = HllState::new();
-        state1.update("1");
-        let state1_binary = state1.evaluate().unwrap();
-
-        let mut state2 = HllState::new();
-        state2.update("2");
-        let state2_binary = state2.evaluate().unwrap();
-
-        let mut merged_state = HllState::new();
-        if let (ScalarValue::Binary(Some(bytes1)), ScalarValue::Binary(Some(bytes2))) =
-            (&state1_binary, &state2_binary)
-        {
-            let binary_array = Arc::new(BinaryArray::from(vec![
-                bytes1.as_slice(),
-                bytes2.as_slice(),
-            ])) as ArrayRef;
-            merged_state.merge_batch(&[binary_array]).unwrap();
-
-            let result = merged_state.evaluate().unwrap();
-            if let ScalarValue::Binary(Some(bytes)) = result {
-                let mut hll: HllStateType = bincode::deserialize(&bytes).unwrap();
-                assert_eq!(hll.count().trunc() as u32, 2);
-            } else {
-                panic!("Expected binary scalar value");
-            }
-        } else {
-            panic!("Expected binary scalar values");
-        }
-    }
-
-    #[test]
-    fn test_hll_merge_function() {
-        // Create two HLL states with different values
-        let mut state1 = HllState::new();
-        state1.update("1");
-        state1.update("2");
-        let state1_binary = state1.evaluate().unwrap();
-
-        let mut state2 = HllState::new();
-        state2.update("2");
-        state2.update("3");
-        let state2_binary = state2.evaluate().unwrap();
-
-        // Create a merge state and merge both states
-        let mut merge_state = HllState::new();
-        if let (ScalarValue::Binary(Some(bytes1)), ScalarValue::Binary(Some(bytes2))) =
-            (&state1_binary, &state2_binary)
-        {
-            let binary_array = Arc::new(BinaryArray::from(vec![
-                bytes1.as_slice(),
-                bytes2.as_slice(),
-            ])) as ArrayRef;
-            merge_state.update_batch(&[binary_array]).unwrap();
-
-            let result = merge_state.evaluate().unwrap();
-            if let ScalarValue::Binary(Some(bytes)) = result {
-                let mut hll: HllStateType = bincode::deserialize(&bytes).unwrap();
-                // Should have 3 unique values: "1", "2", "3"
-                assert_eq!(hll.count().trunc() as u32, 3);
-            } else {
-                panic!("Expected binary scalar value");
-            }
-        } else {
-            panic!("Expected binary scalar values");
-        }
-    }
-}

src/common/function/src/aggr/uddsketch_state.rs

@@ -1,307 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use common_query::prelude::*;
-use common_telemetry::trace;
-use datafusion::common::cast::{as_binary_array, as_primitive_array};
-use datafusion::common::not_impl_err;
-use datafusion::error::{DataFusionError, Result as DfResult};
-use datafusion::logical_expr::function::AccumulatorArgs;
-use datafusion::logical_expr::{Accumulator as DfAccumulator, AggregateUDF};
-use datafusion::physical_plan::expressions::Literal;
-use datafusion::prelude::create_udaf;
-use datatypes::arrow::array::ArrayRef;
-use datatypes::arrow::datatypes::{DataType, Float64Type};
-use uddsketch::{SketchHashKey, UDDSketch};
-
-pub const UDDSKETCH_STATE_NAME: &str = "uddsketch_state";
-
-#[derive(Debug)]
-pub struct UddSketchState {
-    uddsketch: UDDSketch,
-}
-
-impl UddSketchState {
-    pub fn new(bucket_size: u64, error_rate: f64) -> Self {
-        Self {
-            uddsketch: UDDSketch::new(bucket_size, error_rate),
-        }
-    }
-
-    pub fn udf_impl() -> AggregateUDF {
-        create_udaf(
-            UDDSKETCH_STATE_NAME,
-            vec![DataType::Int64, DataType::Float64, DataType::Float64],
-            Arc::new(DataType::Binary),
-            Volatility::Immutable,
-            Arc::new(|args| {
-                let (bucket_size, error_rate) = downcast_accumulator_args(args)?;
-                Ok(Box::new(UddSketchState::new(bucket_size, error_rate)))
-            }),
-            Arc::new(vec![DataType::Binary]),
-        )
-    }
-
-    fn update(&mut self, value: f64) {
-        self.uddsketch.add_value(value);
-    }
-
-    fn merge(&mut self, raw: &[u8]) {
-        if let Ok(uddsketch) = bincode::deserialize::<UDDSketch>(raw) {
-            if uddsketch.count() != 0 {
-                self.uddsketch.merge_sketch(&uddsketch);
-            }
-        } else {
-            trace!("Warning: Failed to deserialize UDDSketch from {:?}", raw);
-        }
-    }
-}
-
-fn downcast_accumulator_args(args: AccumulatorArgs) -> DfResult<(u64, f64)> {
-    let bucket_size = match args.exprs[0]
-        .as_any()
-        .downcast_ref::<Literal>()
-        .map(|lit| lit.value())
-    {
-        Some(ScalarValue::Int64(Some(value))) => *value as u64,
-        _ => {
-            return not_impl_err!(
-                "{} not supported for bucket size: {}",
-                UDDSKETCH_STATE_NAME,
-                &args.exprs[0]
-            )
-        }
-    };
-
-    let error_rate = match args.exprs[1]
-        .as_any()
-        .downcast_ref::<Literal>()
-        .map(|lit| lit.value())
-    {
-        Some(ScalarValue::Float64(Some(value))) => *value,
-        _ => {
-            return not_impl_err!(
-                "{} not supported for error rate: {}",
-                UDDSKETCH_STATE_NAME,
-                &args.exprs[1]
-            )
-        }
-    };
-
-    Ok((bucket_size, error_rate))
-}
-
-impl DfAccumulator for UddSketchState {
-    fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> {
-        let array = &values[2]; // the third column is data value
-        let f64_array = as_primitive_array::<Float64Type>(array)?;
-        for v in f64_array.iter().flatten() {
-            self.update(v);
-        }
-
-        Ok(())
-    }
-
-    fn evaluate(&mut self) -> DfResult<ScalarValue> {
-        Ok(ScalarValue::Binary(Some(
-            bincode::serialize(&self.uddsketch).map_err(|e| {
-                DataFusionError::Internal(format!("Failed to serialize UDDSketch: {}", e))
-            })?,
-        )))
-    }
-
-    fn size(&self) -> usize {
-        // Base size of UDDSketch struct fields
-        let mut total_size = std::mem::size_of::<f64>() * 3 + // alpha, gamma, values_sum
-                            std::mem::size_of::<u32>() +      // compactions
-                            std::mem::size_of::<u64>() * 2; // max_buckets, num_values
-
-        // Size of buckets (SketchHashMap)
-        // Each bucket entry contains:
-        // - SketchHashKey (enum with i64/Zero/Invalid variants)
-        // - SketchHashEntry (count: u64, next: SketchHashKey)
-        let bucket_entry_size = std::mem::size_of::<SketchHashKey>() + // key
-                               std::mem::size_of::<u64>() +            // count
-                               std::mem::size_of::<SketchHashKey>(); // next
-
-        total_size += self.uddsketch.current_buckets_count() * bucket_entry_size;
-
-        total_size
-    }
-
-    fn state(&mut self) -> DfResult<Vec<ScalarValue>> {
-        Ok(vec![ScalarValue::Binary(Some(
-            bincode::serialize(&self.uddsketch).map_err(|e| {
-                DataFusionError::Internal(format!("Failed to serialize UDDSketch: {}", e))
-            })?,
-        ))])
-    }
-
-    fn merge_batch(&mut self, states: &[ArrayRef]) -> DfResult<()> {
-        let array = &states[0];
-        let binary_array = as_binary_array(array)?;
-        for v in binary_array.iter().flatten() {
-            self.merge(v);
-        }
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use datafusion::arrow::array::{BinaryArray, Float64Array};
-
-    use super::*;
-
-    #[test]
-    fn test_uddsketch_state_basic() {
-        let mut state = UddSketchState::new(10, 0.01);
-        state.update(1.0);
-        state.update(2.0);
-        state.update(3.0);
-
-        let result = state.evaluate().unwrap();
-        if let ScalarValue::Binary(Some(bytes)) = result {
-            let deserialized: UDDSketch = bincode::deserialize(&bytes).unwrap();
-            assert_eq!(deserialized.count(), 3);
-        } else {
-            panic!("Expected binary scalar value");
-        }
-    }
-
-    #[test]
-    fn test_uddsketch_state_roundtrip() {
-        let mut state = UddSketchState::new(10, 0.01);
-        state.update(1.0);
-        state.update(2.0);
-
-        // Serialize
-        let serialized = state.evaluate().unwrap();
-
-        // Create new state and merge the serialized data
-        let mut new_state = UddSketchState::new(10, 0.01);
-        if let ScalarValue::Binary(Some(bytes)) = &serialized {
-            new_state.merge(bytes);
-
-            // Verify the merged state matches original by comparing deserialized values
-            let original_sketch: UDDSketch = bincode::deserialize(bytes).unwrap();
-            let new_result = new_state.evaluate().unwrap();
-            if let ScalarValue::Binary(Some(new_bytes)) = new_result {
-                let new_sketch: UDDSketch = bincode::deserialize(&new_bytes).unwrap();
-                assert_eq!(original_sketch.count(), new_sketch.count());
-                assert_eq!(original_sketch.sum(), new_sketch.sum());
-                assert_eq!(original_sketch.mean(), new_sketch.mean());
-                assert_eq!(original_sketch.max_error(), new_sketch.max_error());
-                // Compare a few quantiles to ensure statistical equivalence
-                for q in [0.1, 0.5, 0.9].iter() {
-                    assert!(
-                        (original_sketch.estimate_quantile(*q) - new_sketch.estimate_quantile(*q))
-                            .abs()
-                            < 1e-10,
-                        "Quantile {} mismatch: original={}, new={}",
-                        q,
-                        original_sketch.estimate_quantile(*q),
-                        new_sketch.estimate_quantile(*q)
-                    );
-                }
-            } else {
-                panic!("Expected binary scalar value");
-            }
-        } else {
-            panic!("Expected binary scalar value");
-        }
-    }
-
-    #[test]
-    fn test_uddsketch_state_batch_update() {
-        let mut state = UddSketchState::new(10, 0.01);
-        let values = vec![1.0f64, 2.0, 3.0];
-        let array = Arc::new(Float64Array::from(values)) as ArrayRef;
-
-        state
-            .update_batch(&[array.clone(), array.clone(), array])
-            .unwrap();
-
-        let result = state.evaluate().unwrap();
-        if let ScalarValue::Binary(Some(bytes)) = result {
-            let deserialized: UDDSketch = bincode::deserialize(&bytes).unwrap();
-            assert_eq!(deserialized.count(), 3);
-        } else {
-            panic!("Expected binary scalar value");
-        }
-    }
-
-    #[test]
-    fn test_uddsketch_state_merge_batch() {
-        let mut state1 = UddSketchState::new(10, 0.01);
-        state1.update(1.0);
-        let state1_binary = state1.evaluate().unwrap();
-
-        let mut state2 = UddSketchState::new(10, 0.01);
-        state2.update(2.0);
-        let state2_binary = state2.evaluate().unwrap();
-
-        let mut merged_state = UddSketchState::new(10, 0.01);
-        if let (ScalarValue::Binary(Some(bytes1)), ScalarValue::Binary(Some(bytes2))) =
-            (&state1_binary, &state2_binary)
-        {
-            let binary_array = Arc::new(BinaryArray::from(vec![
-                bytes1.as_slice(),
-                bytes2.as_slice(),
-            ])) as ArrayRef;
-            merged_state.merge_batch(&[binary_array]).unwrap();
-
-            let result = merged_state.evaluate().unwrap();
-            if let ScalarValue::Binary(Some(bytes)) = result {
-                let deserialized: UDDSketch = bincode::deserialize(&bytes).unwrap();
-                assert_eq!(deserialized.count(), 2);
-            } else {
-                panic!("Expected binary scalar value");
-            }
-        } else {
-            panic!("Expected binary scalar values");
-        }
-    }
-
-    #[test]
-    fn test_uddsketch_state_size() {
-        let mut state = UddSketchState::new(10, 0.01);
-        let initial_size = state.size();
-
-        // Add some values to create buckets
-        state.update(1.0);
-        state.update(2.0);
-        state.update(3.0);
-
-        let size_with_values = state.size();
-        assert!(
-            size_with_values > initial_size,
-            "Size should increase after adding values: initial={}, with_values={}",
-            initial_size,
-            size_with_values
-        );
-
-        // Verify size increases with more buckets
-        state.update(10.0); // This should create a new bucket
-        assert!(
-            state.size() > size_with_values,
-            "Size should increase after adding new bucket: prev={}, new={}",
-            size_with_values,
-            state.size()
-        );
-    }
-}

src/common/function/src/function_registry.rs

@@ -22,12 +22,10 @@ use crate::function::{AsyncFunctionRef, FunctionRef};
 use crate::scalars::aggregate::{AggregateFunctionMetaRef, AggregateFunctions};
 use crate::scalars::date::DateFunction;
 use crate::scalars::expression::ExpressionFunction;
-use crate::scalars::hll_count::HllCalcFunction;
 use crate::scalars::json::JsonFunction;
 use crate::scalars::matches::MatchesFunction;
 use crate::scalars::math::MathFunction;
 use crate::scalars::timestamp::TimestampFunction;
-use crate::scalars::uddsketch_calc::UddSketchCalcFunction;
 use crate::scalars::vector::VectorFunction;
 use crate::system::SystemFunction;
 use crate::table::TableFunction;
@@ -107,8 +105,6 @@ pub static FUNCTION_REGISTRY: Lazy<Arc<FunctionRegistry>> = Lazy::new(|| {
     TimestampFunction::register(&function_registry);
     DateFunction::register(&function_registry);
     ExpressionFunction::register(&function_registry);
-    UddSketchCalcFunction::register(&function_registry);
-    HllCalcFunction::register(&function_registry);
 
     // Aggregate functions
     AggregateFunctions::register(&function_registry);

src/common/function/src/lib.rs

@@ -21,7 +21,6 @@ pub mod scalars;
 mod system;
 mod table;
 
-pub mod aggr;
 pub mod function;
 pub mod function_registry;
 pub mod handlers;

src/common/function/src/scalars.rs

@@ -22,9 +22,7 @@ pub mod matches;
 pub mod math;
 pub mod vector;
 
-pub(crate) mod hll_count;
 #[cfg(test)]
 pub(crate) mod test;
 pub(crate) mod timestamp;
-pub(crate) mod uddsketch_calc;
 pub mod udf;

src/common/function/src/scalars/hll_count.rs

@@ -1,175 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Implementation of the scalar function `hll_count`.
-
-use std::fmt;
-use std::fmt::Display;
-use std::sync::Arc;
-
-use common_query::error::{DowncastVectorSnafu, InvalidFuncArgsSnafu, Result};
-use common_query::prelude::{Signature, Volatility};
-use datatypes::data_type::ConcreteDataType;
-use datatypes::prelude::Vector;
-use datatypes::scalars::{ScalarVector, ScalarVectorBuilder};
-use datatypes::vectors::{BinaryVector, MutableVector, UInt64VectorBuilder, VectorRef};
-use hyperloglogplus::HyperLogLog;
-use snafu::OptionExt;
-
-use crate::aggr::HllStateType;
-use crate::function::{Function, FunctionContext};
-use crate::function_registry::FunctionRegistry;
-
-const NAME: &str = "hll_count";
-
-/// HllCalcFunction implements the scalar function `hll_count`.
-///
-/// It accepts one argument:
-/// 1. The serialized HyperLogLogPlus state, as produced by the aggregator (binary).
-///
-/// For each row, it deserializes the sketch and returns the estimated cardinality.
-#[derive(Debug, Default)]
-pub struct HllCalcFunction;
-
-impl HllCalcFunction {
-    pub fn register(registry: &FunctionRegistry) {
-        registry.register(Arc::new(HllCalcFunction));
-    }
-}
-
-impl Display for HllCalcFunction {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", NAME.to_ascii_uppercase())
-    }
-}
-
-impl Function for HllCalcFunction {
-    fn name(&self) -> &str {
-        NAME
-    }
-
-    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::uint64_datatype())
-    }
-
-    fn signature(&self) -> Signature {
-        // Only argument: HyperLogLogPlus state (binary)
-        Signature::exact(
-            vec![ConcreteDataType::binary_datatype()],
-            Volatility::Immutable,
-        )
-    }
-
-    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        if columns.len() != 1 {
-            return InvalidFuncArgsSnafu {
-                err_msg: format!("hll_count expects 1 argument, got {}", columns.len()),
-            }
-            .fail();
-        }
-
-        let hll_vec = columns[0]
-            .as_any()
-            .downcast_ref::<BinaryVector>()
-            .with_context(|| DowncastVectorSnafu {
-                err_msg: format!("expect BinaryVector, got {}", columns[0].vector_type_name()),
-            })?;
-        let len = hll_vec.len();
-        let mut builder = UInt64VectorBuilder::with_capacity(len);
-
-        for i in 0..len {
-            let hll_opt = hll_vec.get_data(i);
-
-            if hll_opt.is_none() {
-                builder.push_null();
-                continue;
-            }
-
-            let hll_bytes = hll_opt.unwrap();
-
-            // Deserialize the HyperLogLogPlus from its bincode representation
-            let mut hll: HllStateType = match bincode::deserialize(hll_bytes) {
-                Ok(h) => h,
-                Err(e) => {
-                    common_telemetry::trace!("Failed to deserialize HyperLogLogPlus: {}", e);
-                    builder.push_null();
-                    continue;
-                }
-            };
-
-            builder.push(Some(hll.count().round() as u64));
-        }
-
-        Ok(builder.to_vector())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use datatypes::vectors::BinaryVector;
-
-    use super::*;
-    use crate::utils::FixedRandomState;
-
-    #[test]
-    fn test_hll_count_function() {
-        let function = HllCalcFunction;
-        assert_eq!("hll_count", function.name());
-        assert_eq!(
-            ConcreteDataType::uint64_datatype(),
-            function
-                .return_type(&[ConcreteDataType::uint64_datatype()])
-                .unwrap()
-        );
-
-        // Create a test HLL
-        let mut hll = HllStateType::new(14, FixedRandomState::new()).unwrap();
-        for i in 1..=10 {
-            hll.insert(&i.to_string());
-        }
-
-        let serialized_bytes = bincode::serialize(&hll).unwrap();
-        let args: Vec<VectorRef> = vec![Arc::new(BinaryVector::from(vec![Some(serialized_bytes)]))];
-
-        let result = function.eval(FunctionContext::default(), &args).unwrap();
-        assert_eq!(result.len(), 1);
-
-        // Test cardinality estimate
-        if let datatypes::value::Value::UInt64(v) = result.get(0) {
-            assert_eq!(v, 10);
-        } else {
-            panic!("Expected uint64 value");
-        }
-    }
-
-    #[test]
-    fn test_hll_count_function_errors() {
-        let function = HllCalcFunction;
-
-        // Test with invalid number of arguments
-        let args: Vec<VectorRef> = vec![];
-        let result = function.eval(FunctionContext::default(), &args);
-        assert!(result.is_err());
-        assert!(result
-            .unwrap_err()
-            .to_string()
-            .contains("hll_count expects 1 argument"));
-
-        // Test with invalid binary data
-        let args: Vec<VectorRef> = vec![Arc::new(BinaryVector::from(vec![Some(vec![1, 2, 3])]))]; // Invalid binary data
-        let result = function.eval(FunctionContext::default(), &args).unwrap();
-        assert_eq!(result.len(), 1);
-        assert!(matches!(result.get(0), datatypes::value::Value::Null));
-    }
-}

src/common/function/src/scalars/json.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use std::sync::Arc;
-pub mod json_get;
+mod json_get;
 mod json_is;
 mod json_path_exists;
 mod json_path_match;

src/common/function/src/scalars/uddsketch_calc.rs

@@ -1,211 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Implementation of the scalar function `uddsketch_calc`.
-
-use std::fmt;
-use std::fmt::Display;
-use std::sync::Arc;
-
-use common_query::error::{DowncastVectorSnafu, InvalidFuncArgsSnafu, Result};
-use common_query::prelude::{Signature, Volatility};
-use datatypes::data_type::ConcreteDataType;
-use datatypes::prelude::Vector;
-use datatypes::scalars::{ScalarVector, ScalarVectorBuilder};
-use datatypes::vectors::{BinaryVector, Float64VectorBuilder, MutableVector, VectorRef};
-use snafu::OptionExt;
-use uddsketch::UDDSketch;
-
-use crate::function::{Function, FunctionContext};
-use crate::function_registry::FunctionRegistry;
-
-const NAME: &str = "uddsketch_calc";
-
-/// UddSketchCalcFunction implements the scalar function `uddsketch_calc`.
-///
-/// It accepts two arguments:
-/// 1. A percentile (as f64) for which to compute the estimated quantile (e.g. 0.95 for p95).
-/// 2. The serialized UDDSketch state, as produced by the aggregator (binary).
-///
-/// For each row, it deserializes the sketch and returns the computed quantile value.
-#[derive(Debug, Default)]
-pub struct UddSketchCalcFunction;
-
-impl UddSketchCalcFunction {
-    pub fn register(registry: &FunctionRegistry) {
-        registry.register(Arc::new(UddSketchCalcFunction));
-    }
-}
-
-impl Display for UddSketchCalcFunction {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", NAME.to_ascii_uppercase())
-    }
-}
-
-impl Function for UddSketchCalcFunction {
-    fn name(&self) -> &str {
-        NAME
-    }
-
-    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::float64_datatype())
-    }
-
-    fn signature(&self) -> Signature {
-        // First argument: percentile (float64)
-        // Second argument: UDDSketch state (binary)
-        Signature::exact(
-            vec![
-                ConcreteDataType::float64_datatype(),
-                ConcreteDataType::binary_datatype(),
-            ],
-            Volatility::Immutable,
-        )
-    }
-
-    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        if columns.len() != 2 {
-            return InvalidFuncArgsSnafu {
-                err_msg: format!("uddsketch_calc expects 2 arguments, got {}", columns.len()),
-            }
-            .fail();
-        }
-
-        let perc_vec = &columns[0];
-        let sketch_vec = columns[1]
-            .as_any()
-            .downcast_ref::<BinaryVector>()
-            .with_context(|| DowncastVectorSnafu {
-                err_msg: format!("expect BinaryVector, got {}", columns[1].vector_type_name()),
-            })?;
-        let len = sketch_vec.len();
-        let mut builder = Float64VectorBuilder::with_capacity(len);
-
-        for i in 0..len {
-            let perc_opt = perc_vec.get(i).as_f64_lossy();
-            let sketch_opt = sketch_vec.get_data(i);
-
-            if sketch_opt.is_none() || perc_opt.is_none() {
-                builder.push_null();
-                continue;
-            }
-
-            let sketch_bytes = sketch_opt.unwrap();
-            let perc = perc_opt.unwrap();
-
-            // Deserialize the UDDSketch from its bincode representation
-            let sketch: UDDSketch = match bincode::deserialize(sketch_bytes) {
-                Ok(s) => s,
-                Err(e) => {
-                    common_telemetry::trace!("Failed to deserialize UDDSketch: {}", e);
-                    builder.push_null();
-                    continue;
-                }
-            };
-
-            // Compute the estimated quantile from the sketch
-            let result = sketch.estimate_quantile(perc);
-            builder.push(Some(result));
-        }
-
-        Ok(builder.to_vector())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use datatypes::vectors::{BinaryVector, Float64Vector};
-
-    use super::*;
-
-    #[test]
-    fn test_uddsketch_calc_function() {
-        let function = UddSketchCalcFunction;
-        assert_eq!("uddsketch_calc", function.name());
-        assert_eq!(
-            ConcreteDataType::float64_datatype(),
-            function
-                .return_type(&[ConcreteDataType::float64_datatype()])
-                .unwrap()
-        );
-
-        // Create a test sketch
-        let mut sketch = UDDSketch::new(128, 0.01);
-        sketch.add_value(10.0);
-        sketch.add_value(20.0);
-        sketch.add_value(30.0);
-        sketch.add_value(40.0);
-        sketch.add_value(50.0);
-        sketch.add_value(60.0);
-        sketch.add_value(70.0);
-        sketch.add_value(80.0);
-        sketch.add_value(90.0);
-        sketch.add_value(100.0);
-
-        // Get expected values directly from the sketch
-        let expected_p50 = sketch.estimate_quantile(0.5);
-        let expected_p90 = sketch.estimate_quantile(0.9);
-        let expected_p95 = sketch.estimate_quantile(0.95);
-
-        let serialized = bincode::serialize(&sketch).unwrap();
-        let percentiles = vec![0.5, 0.9, 0.95];
-
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Float64Vector::from_vec(percentiles.clone())),
-            Arc::new(BinaryVector::from(vec![Some(serialized.clone()); 3])),
-        ];
-
-        let result = function.eval(FunctionContext::default(), &args).unwrap();
-        assert_eq!(result.len(), 3);
-
-        // Test median (p50)
-        assert!(
-            matches!(result.get(0), datatypes::value::Value::Float64(v) if (v - expected_p50).abs() < 1e-10)
-        );
-        // Test p90
-        assert!(
-            matches!(result.get(1), datatypes::value::Value::Float64(v) if (v - expected_p90).abs() < 1e-10)
-        );
-        // Test p95
-        assert!(
-            matches!(result.get(2), datatypes::value::Value::Float64(v) if (v - expected_p95).abs() < 1e-10)
-        );
-    }
-
-    #[test]
-    fn test_uddsketch_calc_function_errors() {
-        let function = UddSketchCalcFunction;
-
-        // Test with invalid number of arguments
-        let args: Vec<VectorRef> = vec![Arc::new(Float64Vector::from_vec(vec![0.95]))];
-        let result = function.eval(FunctionContext::default(), &args);
-        assert!(result.is_err());
-        assert!(result
-            .unwrap_err()
-            .to_string()
-            .contains("uddsketch_calc expects 2 arguments"));
-
-        // Test with invalid binary data
-        let args: Vec<VectorRef> = vec![
-            Arc::new(Float64Vector::from_vec(vec![0.95])),
-            Arc::new(BinaryVector::from(vec![Some(vec![1, 2, 3])])), // Invalid binary data
-        ];
-        let result = function.eval(FunctionContext::default(), &args).unwrap();
-        assert_eq!(result.len(), 1);
-        assert!(matches!(result.get(0), datatypes::value::Value::Null));
-    }
-}

src/common/function/src/scalars/vector.rs

@@ -20,12 +20,11 @@ pub mod impl_conv;
 pub(crate) mod product;
 mod scalar_add;
 mod scalar_mul;
+mod sub;
 pub(crate) mod sum;
-mod vector_add;
 mod vector_div;
 mod vector_mul;
 mod vector_norm;
-mod vector_sub;
 
 use std::sync::Arc;
 
@@ -49,11 +48,10 @@ impl VectorFunction {
         registry.register(Arc::new(scalar_mul::ScalarMulFunction));
 
         // vector calculation
-        registry.register(Arc::new(vector_add::VectorAddFunction));
-        registry.register(Arc::new(vector_sub::VectorSubFunction));
         registry.register(Arc::new(vector_mul::VectorMulFunction));
-        registry.register(Arc::new(vector_div::VectorDivFunction));
         registry.register(Arc::new(vector_norm::VectorNormFunction));
+        registry.register(Arc::new(vector_div::VectorDivFunction));
+        registry.register(Arc::new(sub::SubFunction));
         registry.register(Arc::new(elem_sum::ElemSumFunction));
         registry.register(Arc::new(elem_product::ElemProductFunction));
     }

src/common/function/src/scalars/vector/sub.rs

@@ -42,10 +42,19 @@ const NAME: &str = "vec_sub";
 /// | [0,-1]                                                        |
 /// +---------------------------------------------------------------+
 ///
+/// -- Negative scalar to simulate subtraction
+/// SELECT vec_to_string(vec_sub('[-1.0, -1.0]', '[1.0, 2.0]'));
+///
+/// +-----------------------------------------------------------------+
+/// | vec_to_string(vec_sub(Utf8("[-1.0, -1.0]"),Utf8("[1.0, 2.0]"))) |
+/// +-----------------------------------------------------------------+
+/// | [-2,-3]                                                         |
+/// +-----------------------------------------------------------------+
+///
 #[derive(Debug, Clone, Default)]
-pub struct VectorSubFunction;
+pub struct SubFunction;
 
-impl Function for VectorSubFunction {
+impl Function for SubFunction {
     fn name(&self) -> &str {
         NAME
     }
@@ -133,7 +142,7 @@ impl Function for VectorSubFunction {
     }
 }
 
-impl Display for VectorSubFunction {
+impl Display for SubFunction {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", NAME.to_ascii_uppercase())
     }
@@ -150,7 +159,7 @@ mod tests {
 
     #[test]
     fn test_sub() {
-        let func = VectorSubFunction;
+        let func = SubFunction;
 
         let input0 = Arc::new(StringVector::from(vec![
             Some("[1.0,2.0,3.0]".to_string()),
@@ -185,7 +194,7 @@ mod tests {
 
     #[test]
     fn test_sub_error() {
-        let func = VectorSubFunction;
+        let func = SubFunction;
 
         let input0 = Arc::new(StringVector::from(vec![
             Some("[1.0,2.0,3.0]".to_string()),

src/common/function/src/scalars/vector/vector_add.rs

@@ -1,214 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::borrow::Cow;
-use std::fmt::Display;
-
-use common_query::error::InvalidFuncArgsSnafu;
-use common_query::prelude::Signature;
-use datatypes::prelude::ConcreteDataType;
-use datatypes::scalars::ScalarVectorBuilder;
-use datatypes::vectors::{BinaryVectorBuilder, MutableVector, VectorRef};
-use nalgebra::DVectorView;
-use snafu::ensure;
-
-use crate::function::{Function, FunctionContext};
-use crate::helper;
-use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
-
-const NAME: &str = "vec_add";
-
-/// Adds corresponding elements of two vectors, returns a vector.
-///
-/// # Example
-///
-/// ```sql
-/// SELECT vec_to_string(vec_add("[1.0, 1.0]", "[1.0, 2.0]")) as result;
-///
-/// +---------------------------------------------------------------+
-/// | vec_to_string(vec_add(Utf8("[1.0, 1.0]"),Utf8("[1.0, 2.0]"))) |
-/// +---------------------------------------------------------------+
-/// | [2,3]                                                         |
-/// +---------------------------------------------------------------+
-///
-#[derive(Debug, Clone, Default)]
-pub struct VectorAddFunction;
-
-impl Function for VectorAddFunction {
-    fn name(&self) -> &str {
-        NAME
-    }
-
-    fn return_type(
-        &self,
-        _input_types: &[ConcreteDataType],
-    ) -> common_query::error::Result<ConcreteDataType> {
-        Ok(ConcreteDataType::binary_datatype())
-    }
-
-    fn signature(&self) -> Signature {
-        helper::one_of_sigs2(
-            vec![
-                ConcreteDataType::string_datatype(),
-                ConcreteDataType::binary_datatype(),
-            ],
-            vec![
-                ConcreteDataType::string_datatype(),
-                ConcreteDataType::binary_datatype(),
-            ],
-        )
-    }
-
-    fn eval(
-        &self,
-        _func_ctx: FunctionContext,
-        columns: &[VectorRef],
-    ) -> common_query::error::Result<VectorRef> {
-        ensure!(
-            columns.len() == 2,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect exactly two, have: {}",
-                    columns.len()
-                )
-            }
-        );
-        let arg0 = &columns[0];
-        let arg1 = &columns[1];
-
-        ensure!(
-            arg0.len() == arg1.len(),
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The lengths of the vector are not aligned, args 0: {}, args 1: {}",
-                    arg0.len(),
-                    arg1.len(),
-                )
-            }
-        );
-
-        let len = arg0.len();
-        let mut result = BinaryVectorBuilder::with_capacity(len);
-        if len == 0 {
-            return Ok(result.to_vector());
-        }
-
-        let arg0_const = as_veclit_if_const(arg0)?;
-        let arg1_const = as_veclit_if_const(arg1)?;
-
-        for i in 0..len {
-            let arg0 = match arg0_const.as_ref() {
-                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
-                None => as_veclit(arg0.get_ref(i))?,
-            };
-            let arg1 = match arg1_const.as_ref() {
-                Some(arg1) => Some(Cow::Borrowed(arg1.as_ref())),
-                None => as_veclit(arg1.get_ref(i))?,
-            };
-            let (Some(arg0), Some(arg1)) = (arg0, arg1) else {
-                result.push_null();
-                continue;
-            };
-            let vec0 = DVectorView::from_slice(&arg0, arg0.len());
-            let vec1 = DVectorView::from_slice(&arg1, arg1.len());
-
-            let vec_res = vec0 + vec1;
-            let veclit = vec_res.as_slice();
-            let binlit = veclit_to_binlit(veclit);
-            result.push(Some(&binlit));
-        }
-
-        Ok(result.to_vector())
-    }
-}
-
-impl Display for VectorAddFunction {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", NAME.to_ascii_uppercase())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use common_query::error::Error;
-    use datatypes::vectors::StringVector;
-
-    use super::*;
-
-    #[test]
-    fn test_sub() {
-        let func = VectorAddFunction;
-
-        let input0 = Arc::new(StringVector::from(vec![
-            Some("[1.0,2.0,3.0]".to_string()),
-            Some("[4.0,5.0,6.0]".to_string()),
-            None,
-            Some("[2.0,3.0,3.0]".to_string()),
-        ]));
-        let input1 = Arc::new(StringVector::from(vec![
-            Some("[1.0,1.0,1.0]".to_string()),
-            Some("[6.0,5.0,4.0]".to_string()),
-            Some("[3.0,2.0,2.0]".to_string()),
-            None,
-        ]));
-
-        let result = func
-            .eval(FunctionContext::default(), &[input0, input1])
-            .unwrap();
-
-        let result = result.as_ref();
-        assert_eq!(result.len(), 4);
-        assert_eq!(
-            result.get_ref(0).as_binary().unwrap(),
-            Some(veclit_to_binlit(&[2.0, 3.0, 4.0]).as_slice())
-        );
-        assert_eq!(
-            result.get_ref(1).as_binary().unwrap(),
-            Some(veclit_to_binlit(&[10.0, 10.0, 10.0]).as_slice())
-        );
-        assert!(result.get_ref(2).is_null());
-        assert!(result.get_ref(3).is_null());
-    }
-
-    #[test]
-    fn test_sub_error() {
-        let func = VectorAddFunction;
-
-        let input0 = Arc::new(StringVector::from(vec![
-            Some("[1.0,2.0,3.0]".to_string()),
-            Some("[4.0,5.0,6.0]".to_string()),
-            None,
-            Some("[2.0,3.0,3.0]".to_string()),
-        ]));
-        let input1 = Arc::new(StringVector::from(vec![
-            Some("[1.0,1.0,1.0]".to_string()),
-            Some("[6.0,5.0,4.0]".to_string()),
-            Some("[3.0,2.0,2.0]".to_string()),
-        ]));
-
-        let result = func.eval(FunctionContext::default(), &[input0, input1]);
-
-        match result {
-            Err(Error::InvalidFuncArgs { err_msg, .. }) => {
-                assert_eq!(
-                    err_msg,
-                    "The lengths of the vector are not aligned, args 0: 4, args 1: 3"
-                )
-            }
-            _ => unreachable!(),
-        }
-    }
-}

src/common/function/src/utils.rs

@@ -12,11 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::hash::BuildHasher;
-
-use ahash::RandomState;
-use serde::{Deserialize, Serialize};
-
 /// Escapes special characters in the provided pattern string for `LIKE`.
 ///
 /// Specifically, it prefixes the backslash (`\`), percent (`%`), and underscore (`_`)
@@ -37,71 +32,6 @@ pub fn escape_like_pattern(pattern: &str) -> String {
         })
         .collect::<String>()
 }
-
-/// A random state with fixed seeds.
-///
-/// This is used to ensure that the hash values are consistent across
-/// different processes, and easy to serialize and deserialize.
-#[derive(Debug)]
-pub struct FixedRandomState {
-    state: RandomState,
-}
-
-impl FixedRandomState {
-    // some random seeds
-    const RANDOM_SEED_0: u64 = 0x517cc1b727220a95;
-    const RANDOM_SEED_1: u64 = 0x428a2f98d728ae22;
-    const RANDOM_SEED_2: u64 = 0x7137449123ef65cd;
-    const RANDOM_SEED_3: u64 = 0xb5c0fbcfec4d3b2f;
-
-    pub fn new() -> Self {
-        Self {
-            state: ahash::RandomState::with_seeds(
-                Self::RANDOM_SEED_0,
-                Self::RANDOM_SEED_1,
-                Self::RANDOM_SEED_2,
-                Self::RANDOM_SEED_3,
-            ),
-        }
-    }
-}
-
-impl Default for FixedRandomState {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl BuildHasher for FixedRandomState {
-    type Hasher = ahash::AHasher;
-
-    fn build_hasher(&self) -> Self::Hasher {
-        self.state.build_hasher()
-    }
-
-    fn hash_one<T: std::hash::Hash>(&self, x: T) -> u64 {
-        self.state.hash_one(x)
-    }
-}
-
-impl Serialize for FixedRandomState {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        serializer.serialize_unit()
-    }
-}
-
-impl<'de> Deserialize<'de> for FixedRandomState {
-    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        Ok(Self::new())
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

src/common/grpc-expr/Cargo.toml

@@ -22,4 +22,4 @@ store-api.workspace = true
 table.workspace = true
 
 [dev-dependencies]
-paste.workspace = true
+paste = "1.0"

src/common/grpc-expr/src/alter.rs

@@ -15,14 +15,13 @@
 use api::helper::ColumnDataTypeWrapper;
 use api::v1::add_column_location::LocationType;
 use api::v1::alter_table_expr::Kind;
-use api::v1::column_def::{as_fulltext_option, as_skipping_index_type};
+use api::v1::column_def::as_fulltext_option;
 use api::v1::{
     column_def, AddColumnLocation as Location, AlterTableExpr, Analyzer, CreateTableExpr,
     DropColumns, ModifyColumnTypes, RenameTable, SemanticType,
-    SkippingIndexType as PbSkippingIndexType,
 };
 use common_query::AddColumnLocation;
-use datatypes::schema::{ColumnSchema, FulltextOptions, RawSchema, SkippingIndexOptions};
+use datatypes::schema::{ColumnSchema, FulltextOptions, RawSchema};
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::region_request::{SetRegionOption, UnsetRegionOption};
 use table::metadata::TableId;
@@ -32,8 +31,7 @@ use table::requests::{
 };
 
 use crate::error::{
-    InvalidColumnDefSnafu, InvalidSetFulltextOptionRequestSnafu,
-    InvalidSetSkippingIndexOptionRequestSnafu, InvalidSetTableOptionRequestSnafu,
+    InvalidColumnDefSnafu, InvalidSetFulltextOptionRequestSnafu, InvalidSetTableOptionRequestSnafu,
     InvalidUnsetTableOptionRequestSnafu, MissingAlterIndexOptionSnafu, MissingFieldSnafu,
     MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu,
 };
@@ -139,18 +137,6 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterTableExpr) -> Result<
                         column_name: i.column_name,
                     },
                 },
-                api::v1::set_index::Options::Skipping(s) => AlterKind::SetIndex {
-                    options: SetIndexOptions::Skipping {
-                        column_name: s.column_name,
-                        options: SkippingIndexOptions {
-                            granularity: s.granularity as u32,
-                            index_type: as_skipping_index_type(
-                                PbSkippingIndexType::try_from(s.skipping_index_type)
-                                    .context(InvalidSetSkippingIndexOptionRequestSnafu)?,
-                            ),
-                        },
-                    },
-                },
             },
             None => return MissingAlterIndexOptionSnafu.fail(),
         },
@@ -166,11 +152,6 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterTableExpr) -> Result<
                         column_name: i.column_name,
                     },
                 },
-                api::v1::unset_index::Options::Skipping(s) => AlterKind::UnsetIndex {
-                    options: UnsetIndexOptions::Skipping {
-                        column_name: s.column_name,
-                    },
-                },
             },
             None => return MissingAlterIndexOptionSnafu.fail(),
         },

src/common/grpc-expr/src/error.rs

@@ -140,14 +140,6 @@ pub enum Error {
         error: prost::UnknownEnumValue,
     },
 
-    #[snafu(display("Invalid set skipping index option request"))]
-    InvalidSetSkippingIndexOptionRequest {
-        #[snafu(implicit)]
-        location: Location,
-        #[snafu(source)]
-        error: prost::UnknownEnumValue,
-    },
-
     #[snafu(display("Missing alter index options"))]
     MissingAlterIndexOption {
         #[snafu(implicit)]
@@ -179,7 +171,6 @@ impl ErrorExt for Error {
             Error::InvalidSetTableOptionRequest { .. }
             | Error::InvalidUnsetTableOptionRequest { .. }
             | Error::InvalidSetFulltextOptionRequest { .. }
-            | Error::InvalidSetSkippingIndexOptionRequest { .. }
             | Error::MissingAlterIndexOption { .. } => StatusCode::InvalidArguments,
         }
     }

src/common/grpc-expr/src/insert.rs

@@ -14,12 +14,37 @@
 
 use api::helper;
 use api::v1::column::Values;
+use api::v1::{Column, CreateTableExpr};
 use common_base::BitVec;
 use datatypes::data_type::{ConcreteDataType, DataType};
 use datatypes::prelude::VectorRef;
 use snafu::{ensure, ResultExt};
+use table::metadata::TableId;
+use table::table_reference::TableReference;
 
 use crate::error::{CreateVectorSnafu, Result, UnexpectedValuesLengthSnafu};
+use crate::util;
+use crate::util::ColumnExpr;
+
+/// Try to build create table request from insert data.
+pub fn build_create_expr_from_insertion(
+    catalog_name: &str,
+    schema_name: &str,
+    table_id: Option<TableId>,
+    table_name: &str,
+    columns: &[Column],
+    engine: &str,
+) -> Result<CreateTableExpr> {
+    let table_name = TableReference::full(catalog_name, schema_name, table_name);
+    let column_exprs = ColumnExpr::from_columns(columns);
+    util::build_create_table_expr(
+        table_id,
+        &table_name,
+        column_exprs,
+        engine,
+        "Created on insertion",
+    )
+}
 
 pub(crate) fn add_values_to_builder(
     data_type: ConcreteDataType,
@@ -62,7 +87,276 @@ fn is_null(null_mask: &BitVec, idx: usize) -> Option<bool> {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
+    use std::{assert_eq, vec};
+
+    use api::helper::ColumnDataTypeWrapper;
+    use api::v1::column::Values;
+    use api::v1::column_data_type_extension::TypeExt;
+    use api::v1::{
+        Column, ColumnDataType, ColumnDataTypeExtension, Decimal128, DecimalTypeExtension,
+        IntervalMonthDayNano, SemanticType,
+    };
+    use common_base::BitVec;
+    use common_catalog::consts::MITO_ENGINE;
+    use common_time::interval::IntervalUnit;
+    use common_time::timestamp::TimeUnit;
+    use datatypes::data_type::ConcreteDataType;
+    use datatypes::schema::{ColumnSchema, SchemaBuilder};
+    use snafu::ResultExt;
+
     use super::*;
+    use crate::error;
+    use crate::error::ColumnDataTypeSnafu;
+
+    #[inline]
+    fn build_column_schema(
+        column_name: &str,
+        datatype: i32,
+        nullable: bool,
+    ) -> error::Result<ColumnSchema> {
+        let datatype_wrapper =
+            ColumnDataTypeWrapper::try_new(datatype, None).context(ColumnDataTypeSnafu)?;
+
+        Ok(ColumnSchema::new(
+            column_name,
+            datatype_wrapper.into(),
+            nullable,
+        ))
+    }
+
+    #[test]
+    fn test_build_create_table_request() {
+        let table_id = Some(10);
+        let table_name = "test_metric";
+
+        assert!(
+            build_create_expr_from_insertion("", "", table_id, table_name, &[], MITO_ENGINE)
+                .is_err()
+        );
+
+        let insert_batch = mock_insert_batch();
+
+        let create_expr = build_create_expr_from_insertion(
+            "",
+            "",
+            table_id,
+            table_name,
+            &insert_batch.0,
+            MITO_ENGINE,
+        )
+        .unwrap();
+
+        assert_eq!(table_id, create_expr.table_id.map(|x| x.id));
+        assert_eq!(table_name, create_expr.table_name);
+        assert_eq!("Created on insertion".to_string(), create_expr.desc);
+        assert_eq!(
+            vec![create_expr.column_defs[0].name.clone()],
+            create_expr.primary_keys
+        );
+
+        let column_defs = create_expr.column_defs;
+        assert_eq!(column_defs[5].name, create_expr.time_index);
+        assert_eq!(7, column_defs.len());
+
+        assert_eq!(
+            ConcreteDataType::string_datatype(),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    column_defs
+                        .iter()
+                        .find(|c| c.name == "host")
+                        .unwrap()
+                        .data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+
+        assert_eq!(
+            ConcreteDataType::float64_datatype(),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    column_defs
+                        .iter()
+                        .find(|c| c.name == "cpu")
+                        .unwrap()
+                        .data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+
+        assert_eq!(
+            ConcreteDataType::float64_datatype(),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    column_defs
+                        .iter()
+                        .find(|c| c.name == "memory")
+                        .unwrap()
+                        .data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+
+        assert_eq!(
+            ConcreteDataType::time_datatype(TimeUnit::Millisecond),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    column_defs
+                        .iter()
+                        .find(|c| c.name == "time")
+                        .unwrap()
+                        .data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+
+        assert_eq!(
+            ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    column_defs
+                        .iter()
+                        .find(|c| c.name == "interval")
+                        .unwrap()
+                        .data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+
+        assert_eq!(
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    column_defs
+                        .iter()
+                        .find(|c| c.name == "ts")
+                        .unwrap()
+                        .data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+
+        let decimal_column = column_defs.iter().find(|c| c.name == "decimals").unwrap();
+        assert_eq!(
+            ConcreteDataType::decimal128_datatype(38, 10),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    decimal_column.data_type,
+                    decimal_column.datatype_extension,
+                )
+                .unwrap()
+            )
+        );
+    }
+
+    #[test]
+    fn test_find_new_columns() {
+        let mut columns = Vec::with_capacity(1);
+        let cpu_column = build_column_schema("cpu", 10, true).unwrap();
+        let ts_column = build_column_schema("ts", 15, false)
+            .unwrap()
+            .with_time_index(true);
+        columns.push(cpu_column);
+        columns.push(ts_column);
+
+        let schema = Arc::new(SchemaBuilder::try_from(columns).unwrap().build().unwrap());
+
+        assert!(
+            util::extract_new_columns(&schema, ColumnExpr::from_columns(&[]))
+                .unwrap()
+                .is_none()
+        );
+
+        let insert_batch = mock_insert_batch();
+
+        let add_columns =
+            util::extract_new_columns(&schema, ColumnExpr::from_columns(&insert_batch.0))
+                .unwrap()
+                .unwrap();
+
+        assert_eq!(5, add_columns.add_columns.len());
+        let host_column = &add_columns.add_columns[0];
+        assert_eq!(
+            ConcreteDataType::string_datatype(),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    host_column.column_def.as_ref().unwrap().data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+        assert!(host_column.add_if_not_exists);
+
+        let memory_column = &add_columns.add_columns[1];
+        assert_eq!(
+            ConcreteDataType::float64_datatype(),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    memory_column.column_def.as_ref().unwrap().data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+        assert!(host_column.add_if_not_exists);
+
+        let time_column = &add_columns.add_columns[2];
+        assert_eq!(
+            ConcreteDataType::time_datatype(TimeUnit::Millisecond),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    time_column.column_def.as_ref().unwrap().data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+        assert!(host_column.add_if_not_exists);
+
+        let interval_column = &add_columns.add_columns[3];
+        assert_eq!(
+            ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    interval_column.column_def.as_ref().unwrap().data_type,
+                    None
+                )
+                .unwrap()
+            )
+        );
+        assert!(host_column.add_if_not_exists);
+
+        let decimal_column = &add_columns.add_columns[4];
+        assert_eq!(
+            ConcreteDataType::decimal128_datatype(38, 10),
+            ConcreteDataType::from(
+                ColumnDataTypeWrapper::try_new(
+                    decimal_column.column_def.as_ref().unwrap().data_type,
+                    decimal_column
+                        .column_def
+                        .as_ref()
+                        .unwrap()
+                        .datatype_extension
+                )
+                .unwrap()
+            )
+        );
+        assert!(host_column.add_if_not_exists);
+    }
 
     #[test]
     fn test_is_null() {
@@ -77,4 +371,127 @@ mod tests {
         assert_eq!(None, is_null(&null_mask, 16));
         assert_eq!(None, is_null(&null_mask, 99));
     }
+
+    fn mock_insert_batch() -> (Vec<Column>, u32) {
+        let row_count = 2;
+
+        let host_vals = Values {
+            string_values: vec!["host1".to_string(), "host2".to_string()],
+            ..Default::default()
+        };
+        let host_column = Column {
+            column_name: "host".to_string(),
+            semantic_type: SemanticType::Tag as i32,
+            values: Some(host_vals),
+            null_mask: vec![0],
+            datatype: ColumnDataType::String as i32,
+            ..Default::default()
+        };
+
+        let cpu_vals = Values {
+            f64_values: vec![0.31],
+            ..Default::default()
+        };
+        let cpu_column = Column {
+            column_name: "cpu".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(cpu_vals),
+            null_mask: vec![2],
+            datatype: ColumnDataType::Float64 as i32,
+            ..Default::default()
+        };
+
+        let mem_vals = Values {
+            f64_values: vec![0.1],
+            ..Default::default()
+        };
+        let mem_column = Column {
+            column_name: "memory".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(mem_vals),
+            null_mask: vec![1],
+            datatype: ColumnDataType::Float64 as i32,
+            ..Default::default()
+        };
+
+        let time_vals = Values {
+            time_millisecond_values: vec![100, 101],
+            ..Default::default()
+        };
+        let time_column = Column {
+            column_name: "time".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(time_vals),
+            null_mask: vec![0],
+            datatype: ColumnDataType::TimeMillisecond as i32,
+            ..Default::default()
+        };
+
+        let interval1 = IntervalMonthDayNano {
+            months: 1,
+            days: 2,
+            nanoseconds: 3,
+        };
+        let interval2 = IntervalMonthDayNano {
+            months: 4,
+            days: 5,
+            nanoseconds: 6,
+        };
+        let interval_vals = Values {
+            interval_month_day_nano_values: vec![interval1, interval2],
+            ..Default::default()
+        };
+        let interval_column = Column {
+            column_name: "interval".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(interval_vals),
+            null_mask: vec![0],
+            datatype: ColumnDataType::IntervalMonthDayNano as i32,
+            ..Default::default()
+        };
+
+        let ts_vals = Values {
+            timestamp_millisecond_values: vec![100, 101],
+            ..Default::default()
+        };
+        let ts_column = Column {
+            column_name: "ts".to_string(),
+            semantic_type: SemanticType::Timestamp as i32,
+            values: Some(ts_vals),
+            null_mask: vec![0],
+            datatype: ColumnDataType::TimestampMillisecond as i32,
+            ..Default::default()
+        };
+        let decimal_vals = Values {
+            decimal128_values: vec![Decimal128 { hi: 0, lo: 123 }, Decimal128 { hi: 0, lo: 456 }],
+            ..Default::default()
+        };
+        let decimal_column = Column {
+            column_name: "decimals".to_string(),
+            semantic_type: SemanticType::Field as i32,
+            values: Some(decimal_vals),
+            null_mask: vec![0],
+            datatype: ColumnDataType::Decimal128 as i32,
+            datatype_extension: Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::DecimalType(DecimalTypeExtension {
+                    precision: 38,
+                    scale: 10,
+                })),
+            }),
+            options: None,
+        };
+
+        (
+            vec![
+                host_column,
+                cpu_column,
+                mem_column,
+                time_column,
+                interval_column,
+                ts_column,
+                decimal_column,
+            ],
+            row_count,
+        )
+    }
 }

src/common/grpc-expr/src/lib.rs

@@ -19,3 +19,4 @@ pub mod insert;
 pub mod util;
 
 pub use alter::{alter_expr_to_request, create_table_schema};
+pub use insert::build_create_expr_from_insertion;

src/common/grpc-expr/src/util.rs

@@ -236,414 +236,3 @@ pub fn extract_new_columns(
         }))
     }
 }
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-    use std::{assert_eq, vec};
-
-    use api::helper::ColumnDataTypeWrapper;
-    use api::v1::column::Values;
-    use api::v1::column_data_type_extension::TypeExt;
-    use api::v1::{
-        Column, ColumnDataType, ColumnDataTypeExtension, Decimal128, DecimalTypeExtension,
-        IntervalMonthDayNano, SemanticType,
-    };
-    use common_catalog::consts::MITO_ENGINE;
-    use common_time::interval::IntervalUnit;
-    use common_time::timestamp::TimeUnit;
-    use datatypes::data_type::ConcreteDataType;
-    use datatypes::schema::{ColumnSchema, SchemaBuilder};
-    use snafu::ResultExt;
-
-    use super::*;
-    use crate::error;
-    use crate::error::ColumnDataTypeSnafu;
-
-    #[inline]
-    fn build_column_schema(
-        column_name: &str,
-        datatype: i32,
-        nullable: bool,
-    ) -> error::Result<ColumnSchema> {
-        let datatype_wrapper =
-            ColumnDataTypeWrapper::try_new(datatype, None).context(ColumnDataTypeSnafu)?;
-
-        Ok(ColumnSchema::new(
-            column_name,
-            datatype_wrapper.into(),
-            nullable,
-        ))
-    }
-
-    fn build_create_expr_from_insertion(
-        catalog_name: &str,
-        schema_name: &str,
-        table_id: Option<TableId>,
-        table_name: &str,
-        columns: &[Column],
-        engine: &str,
-    ) -> Result<CreateTableExpr> {
-        let table_name = TableReference::full(catalog_name, schema_name, table_name);
-        let column_exprs = ColumnExpr::from_columns(columns);
-        build_create_table_expr(
-            table_id,
-            &table_name,
-            column_exprs,
-            engine,
-            "Created on insertion",
-        )
-    }
-
-    #[test]
-    fn test_build_create_table_request() {
-        let table_id = Some(10);
-        let table_name = "test_metric";
-
-        assert!(
-            build_create_expr_from_insertion("", "", table_id, table_name, &[], MITO_ENGINE)
-                .is_err()
-        );
-
-        let insert_batch = mock_insert_batch();
-
-        let create_expr = build_create_expr_from_insertion(
-            "",
-            "",
-            table_id,
-            table_name,
-            &insert_batch.0,
-            MITO_ENGINE,
-        )
-        .unwrap();
-
-        assert_eq!(table_id, create_expr.table_id.map(|x| x.id));
-        assert_eq!(table_name, create_expr.table_name);
-        assert_eq!("Created on insertion".to_string(), create_expr.desc);
-        assert_eq!(
-            vec![create_expr.column_defs[0].name.clone()],
-            create_expr.primary_keys
-        );
-
-        let column_defs = create_expr.column_defs;
-        assert_eq!(column_defs[5].name, create_expr.time_index);
-        assert_eq!(7, column_defs.len());
-
-        assert_eq!(
-            ConcreteDataType::string_datatype(),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    column_defs
-                        .iter()
-                        .find(|c| c.name == "host")
-                        .unwrap()
-                        .data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-
-        assert_eq!(
-            ConcreteDataType::float64_datatype(),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    column_defs
-                        .iter()
-                        .find(|c| c.name == "cpu")
-                        .unwrap()
-                        .data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-
-        assert_eq!(
-            ConcreteDataType::float64_datatype(),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    column_defs
-                        .iter()
-                        .find(|c| c.name == "memory")
-                        .unwrap()
-                        .data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-
-        assert_eq!(
-            ConcreteDataType::time_datatype(TimeUnit::Millisecond),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    column_defs
-                        .iter()
-                        .find(|c| c.name == "time")
-                        .unwrap()
-                        .data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-
-        assert_eq!(
-            ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    column_defs
-                        .iter()
-                        .find(|c| c.name == "interval")
-                        .unwrap()
-                        .data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-
-        assert_eq!(
-            ConcreteDataType::timestamp_millisecond_datatype(),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    column_defs
-                        .iter()
-                        .find(|c| c.name == "ts")
-                        .unwrap()
-                        .data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-
-        let decimal_column = column_defs.iter().find(|c| c.name == "decimals").unwrap();
-        assert_eq!(
-            ConcreteDataType::decimal128_datatype(38, 10),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    decimal_column.data_type,
-                    decimal_column.datatype_extension,
-                )
-                .unwrap()
-            )
-        );
-    }
-
-    #[test]
-    fn test_find_new_columns() {
-        let mut columns = Vec::with_capacity(1);
-        let cpu_column = build_column_schema("cpu", 10, true).unwrap();
-        let ts_column = build_column_schema("ts", 15, false)
-            .unwrap()
-            .with_time_index(true);
-        columns.push(cpu_column);
-        columns.push(ts_column);
-
-        let schema = Arc::new(SchemaBuilder::try_from(columns).unwrap().build().unwrap());
-
-        assert!(extract_new_columns(&schema, ColumnExpr::from_columns(&[]))
-            .unwrap()
-            .is_none());
-
-        let insert_batch = mock_insert_batch();
-
-        let add_columns = extract_new_columns(&schema, ColumnExpr::from_columns(&insert_batch.0))
-            .unwrap()
-            .unwrap();
-
-        assert_eq!(5, add_columns.add_columns.len());
-        let host_column = &add_columns.add_columns[0];
-        assert_eq!(
-            ConcreteDataType::string_datatype(),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    host_column.column_def.as_ref().unwrap().data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-        assert!(host_column.add_if_not_exists);
-
-        let memory_column = &add_columns.add_columns[1];
-        assert_eq!(
-            ConcreteDataType::float64_datatype(),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    memory_column.column_def.as_ref().unwrap().data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-        assert!(host_column.add_if_not_exists);
-
-        let time_column = &add_columns.add_columns[2];
-        assert_eq!(
-            ConcreteDataType::time_datatype(TimeUnit::Millisecond),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    time_column.column_def.as_ref().unwrap().data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-        assert!(host_column.add_if_not_exists);
-
-        let interval_column = &add_columns.add_columns[3];
-        assert_eq!(
-            ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    interval_column.column_def.as_ref().unwrap().data_type,
-                    None
-                )
-                .unwrap()
-            )
-        );
-        assert!(host_column.add_if_not_exists);
-
-        let decimal_column = &add_columns.add_columns[4];
-        assert_eq!(
-            ConcreteDataType::decimal128_datatype(38, 10),
-            ConcreteDataType::from(
-                ColumnDataTypeWrapper::try_new(
-                    decimal_column.column_def.as_ref().unwrap().data_type,
-                    decimal_column
-                        .column_def
-                        .as_ref()
-                        .unwrap()
-                        .datatype_extension
-                )
-                .unwrap()
-            )
-        );
-        assert!(host_column.add_if_not_exists);
-    }
-
-    fn mock_insert_batch() -> (Vec<Column>, u32) {
-        let row_count = 2;
-
-        let host_vals = Values {
-            string_values: vec!["host1".to_string(), "host2".to_string()],
-            ..Default::default()
-        };
-        let host_column = Column {
-            column_name: "host".to_string(),
-            semantic_type: SemanticType::Tag as i32,
-            values: Some(host_vals),
-            null_mask: vec![0],
-            datatype: ColumnDataType::String as i32,
-            ..Default::default()
-        };
-
-        let cpu_vals = Values {
-            f64_values: vec![0.31],
-            ..Default::default()
-        };
-        let cpu_column = Column {
-            column_name: "cpu".to_string(),
-            semantic_type: SemanticType::Field as i32,
-            values: Some(cpu_vals),
-            null_mask: vec![2],
-            datatype: ColumnDataType::Float64 as i32,
-            ..Default::default()
-        };
-
-        let mem_vals = Values {
-            f64_values: vec![0.1],
-            ..Default::default()
-        };
-        let mem_column = Column {
-            column_name: "memory".to_string(),
-            semantic_type: SemanticType::Field as i32,
-            values: Some(mem_vals),
-            null_mask: vec![1],
-            datatype: ColumnDataType::Float64 as i32,
-            ..Default::default()
-        };
-
-        let time_vals = Values {
-            time_millisecond_values: vec![100, 101],
-            ..Default::default()
-        };
-        let time_column = Column {
-            column_name: "time".to_string(),
-            semantic_type: SemanticType::Field as i32,
-            values: Some(time_vals),
-            null_mask: vec![0],
-            datatype: ColumnDataType::TimeMillisecond as i32,
-            ..Default::default()
-        };
-
-        let interval1 = IntervalMonthDayNano {
-            months: 1,
-            days: 2,
-            nanoseconds: 3,
-        };
-        let interval2 = IntervalMonthDayNano {
-            months: 4,
-            days: 5,
-            nanoseconds: 6,
-        };
-        let interval_vals = Values {
-            interval_month_day_nano_values: vec![interval1, interval2],
-            ..Default::default()
-        };
-        let interval_column = Column {
-            column_name: "interval".to_string(),
-            semantic_type: SemanticType::Field as i32,
-            values: Some(interval_vals),
-            null_mask: vec![0],
-            datatype: ColumnDataType::IntervalMonthDayNano as i32,
-            ..Default::default()
-        };
-
-        let ts_vals = Values {
-            timestamp_millisecond_values: vec![100, 101],
-            ..Default::default()
-        };
-        let ts_column = Column {
-            column_name: "ts".to_string(),
-            semantic_type: SemanticType::Timestamp as i32,
-            values: Some(ts_vals),
-            null_mask: vec![0],
-            datatype: ColumnDataType::TimestampMillisecond as i32,
-            ..Default::default()
-        };
-        let decimal_vals = Values {
-            decimal128_values: vec![Decimal128 { hi: 0, lo: 123 }, Decimal128 { hi: 0, lo: 456 }],
-            ..Default::default()
-        };
-        let decimal_column = Column {
-            column_name: "decimals".to_string(),
-            semantic_type: SemanticType::Field as i32,
-            values: Some(decimal_vals),
-            null_mask: vec![0],
-            datatype: ColumnDataType::Decimal128 as i32,
-            datatype_extension: Some(ColumnDataTypeExtension {
-                type_ext: Some(TypeExt::DecimalType(DecimalTypeExtension {
-                    precision: 38,
-                    scale: 10,
-                })),
-            }),
-            options: None,
-        };
-
-        (
-            vec![
-                host_column,
-                cpu_column,
-                mem_column,
-                time_column,
-                interval_column,
-                ts_column,
-                decimal_column,
-            ],
-            row_count,
-        )
-    }
-}

src/common/meta/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 
 [features]
 testing = []
-pg_kvbackend = ["dep:tokio-postgres", "dep:backon", "dep:deadpool-postgres", "dep:deadpool"]
+pg_kvbackend = ["dep:tokio-postgres", "dep:backon"]
 
 [lints]
 workspace = true
@@ -36,8 +36,8 @@ common-wal.workspace = true
 datafusion-common.workspace = true
 datafusion-expr.workspace = true
 datatypes.workspace = true
-deadpool = { workspace = true, optional = true }
-deadpool-postgres = { workspace = true, optional = true }
+deadpool.workspace = true
+deadpool-postgres.workspace = true
 derive_builder.workspace = true
 etcd-client.workspace = true
 futures.workspace = true

src/common/meta/src/cache/flow/table_flownode.rs

@@ -16,6 +16,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use futures::future::BoxFuture;
+use futures::TryStreamExt;
 use moka::future::Cache;
 use moka::ops::compute::Op;
 use table::metadata::TableId;
@@ -53,13 +54,9 @@ fn init_factory(table_flow_manager: TableFlowManagerRef) -> Initializer<TableId,
         Box::pin(async move {
             table_flow_manager
                 .flows(table_id)
+                .map_ok(|(key, value)| (key.flownode_id(), value.peer))
+                .try_collect::<HashMap<_, _>>()
                 .await
-                .map(|flows| {
-                    flows
-                        .into_iter()
-                        .map(|(key, value)| (key.flownode_id(), value.peer))
-                        .collect::<HashMap<_, _>>()
-                })
                 // We must cache the `HashSet` even if it's empty,
                 // to avoid future requests to the remote storage next time;
                 // If the value is added to the remote storage,

src/common/meta/src/cluster.rs

@@ -12,10 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::hash::{DefaultHasher, Hash, Hasher};
 use std::str::FromStr;
 
-use api::v1::meta::HeartbeatRequest;
 use common_error::ext::ErrorExt;
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -57,10 +55,12 @@ pub trait ClusterInfo {
 }
 
 /// The key of [NodeInfo] in the storage. The format is `__meta_cluster_node_info-{cluster_id}-{role}-{node_id}`.
-#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
+///
+/// This key cannot be used to describe the `Metasrv` because the `Metasrv` does not have
+/// a `cluster_id`, it serves multiple clusters.
+#[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]
 pub struct NodeInfoKey {
     /// The cluster id.
-    // todo(hl): remove cluster_id as it is not assigned anywhere.
     pub cluster_id: ClusterId,
     /// The role of the node. It can be `[Role::Datanode]` or `[Role::Frontend]`.
     pub role: Role,
@@ -69,28 +69,6 @@ pub struct NodeInfoKey {
 }
 
 impl NodeInfoKey {
-    /// Try to create a `NodeInfoKey` from a "good" heartbeat request. "good" as in every needed
-    /// piece of information is provided and valid.  
-    pub fn new(request: &HeartbeatRequest) -> Option<Self> {
-        let HeartbeatRequest { header, peer, .. } = request;
-        let header = header.as_ref()?;
-        let peer = peer.as_ref()?;
-
-        let role = header.role.try_into().ok()?;
-        let node_id = match role {
-            // Because the Frontend is stateless, it's too easy to neglect choosing a unique id
-            // for it when setting up a cluster. So we calculate its id from its address.
-            Role::Frontend => calculate_node_id(&peer.addr),
-            _ => peer.id,
-        };
-
-        Some(NodeInfoKey {
-            cluster_id: header.cluster_id,
-            role,
-            node_id,
-        })
-    }
-
     pub fn key_prefix_with_cluster_id(cluster_id: u64) -> String {
         format!("{}-{}-", CLUSTER_NODE_INFO_PREFIX, cluster_id)
     }
@@ -105,13 +83,6 @@ impl NodeInfoKey {
     }
 }
 
-/// Calculate (by using the DefaultHasher) the node's id from its address.
-fn calculate_node_id(addr: &str) -> u64 {
-    let mut hasher = DefaultHasher::new();
-    addr.hash(&mut hasher);
-    hasher.finish()
-}
-
 /// The information of a node in the cluster.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct NodeInfo {
@@ -129,7 +100,7 @@ pub struct NodeInfo {
     pub start_time_ms: u64,
 }
 
-#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]
 pub enum Role {
     Datanode,
     Frontend,
@@ -230,8 +201,8 @@ impl TryFrom<Vec<u8>> for NodeInfoKey {
     }
 }
 
-impl From<&NodeInfoKey> for Vec<u8> {
-    fn from(key: &NodeInfoKey) -> Self {
+impl From<NodeInfoKey> for Vec<u8> {
+    fn from(key: NodeInfoKey) -> Self {
         format!(
             "{}-{}-{}-{}",
             CLUSTER_NODE_INFO_PREFIX,
@@ -300,7 +271,6 @@ impl TryFrom<i32> for Role {
 mod tests {
     use std::assert_matches::assert_matches;
 
-    use super::*;
     use crate::cluster::Role::{Datanode, Frontend};
     use crate::cluster::{DatanodeStatus, NodeInfo, NodeInfoKey, NodeStatus};
     use crate::peer::Peer;
@@ -313,7 +283,7 @@ mod tests {
             node_id: 2,
         };
 
-        let key_bytes: Vec<u8> = (&key).into();
+        let key_bytes: Vec<u8> = key.into();
         let new_key: NodeInfoKey = key_bytes.try_into().unwrap();
 
         assert_eq!(1, new_key.cluster_id);
@@ -368,26 +338,4 @@ mod tests {
         let prefix = NodeInfoKey::key_prefix_with_role(2, Frontend);
         assert_eq!(prefix, "__meta_cluster_node_info-2-1-");
     }
-
-    #[test]
-    fn test_calculate_node_id_from_addr() {
-        // Test empty string
-        assert_eq!(calculate_node_id(""), calculate_node_id(""));
-
-        // Test same addresses return same ids
-        let addr1 = "127.0.0.1:8080";
-        let id1 = calculate_node_id(addr1);
-        let id2 = calculate_node_id(addr1);
-        assert_eq!(id1, id2);
-
-        // Test different addresses return different ids
-        let addr2 = "127.0.0.1:8081";
-        let id3 = calculate_node_id(addr2);
-        assert_ne!(id1, id3);
-
-        // Test long address
-        let long_addr = "very.long.domain.name.example.com:9999";
-        let id4 = calculate_node_id(long_addr);
-        assert!(id4 > 0);
-    }
 }

src/common/meta/src/ddl/create_flow.rs

@@ -15,7 +15,6 @@
 mod metadata;
 
 use std::collections::BTreeMap;
-use std::fmt;
 
 use api::v1::flow::flow_request::Body as PbFlowRequest;
 use api::v1::flow::{CreateRequest, FlowRequest, FlowRequestHeader};
@@ -29,6 +28,7 @@ use common_procedure::{
 use common_telemetry::info;
 use common_telemetry::tracing_context::TracingContext;
 use futures::future::join_all;
+use futures::TryStreamExt;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, ResultExt};
@@ -77,7 +77,6 @@ impl CreateFlowProcedure {
                 query_context,
                 state: CreateFlowState::Prepare,
                 prev_flow_info_value: None,
-                flow_type: None,
             },
         }
     }
@@ -105,7 +104,7 @@ impl CreateFlowProcedure {
         if create_if_not_exists && or_replace {
             // this is forbidden because not clear what does that mean exactly
             return error::UnsupportedSnafu {
-                operation: "Create flow with both `IF NOT EXISTS` and `OR REPLACE`",
+                operation: "Create flow with both `IF NOT EXISTS` and `OR REPLACE`".to_string(),
             }
             .fail();
         }
@@ -130,10 +129,9 @@ impl CreateFlowProcedure {
                 .flow_metadata_manager
                 .flow_route_manager()
                 .routes(flow_id)
-                .await?
-                .into_iter()
-                .map(|(_, value)| value.peer)
-                .collect::<Vec<_>>();
+                .map_ok(|(_, value)| value.peer)
+                .try_collect::<Vec<_>>()
+                .await?;
             self.data.flow_id = Some(flow_id);
             self.data.peers = peers;
             info!("Replacing flow, flow_id: {}", flow_id);
@@ -177,8 +175,6 @@ impl CreateFlowProcedure {
             self.allocate_flow_id().await?;
         }
         self.data.state = CreateFlowState::CreateFlows;
-        // determine flow type
-        self.data.flow_type = Some(determine_flow_type(&self.data.task));
 
         Ok(Status::executing(true))
     }
@@ -313,11 +309,6 @@ impl Procedure for CreateFlowProcedure {
     }
 }
 
-pub fn determine_flow_type(_flow_task: &CreateFlowTask) -> FlowType {
-    // TODO(discord9): determine flow type
-    FlowType::RecordingRule
-}
-
 /// The state of [CreateFlowProcedure].
 #[derive(Debug, Clone, Serialize, Deserialize, AsRefStr, PartialEq)]
 pub enum CreateFlowState {
@@ -331,35 +322,6 @@ pub enum CreateFlowState {
     CreateMetadata,
 }
 
-/// The type of flow.
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
-pub enum FlowType {
-    /// The flow is a recording rule task.
-    RecordingRule,
-    /// The flow is a streaming task.
-    Streaming,
-}
-
-impl FlowType {
-    pub const RECORDING_RULE: &str = "recording_rule";
-    pub const STREAMING: &str = "streaming";
-}
-
-impl Default for FlowType {
-    fn default() -> Self {
-        Self::RecordingRule
-    }
-}
-
-impl fmt::Display for FlowType {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            FlowType::RecordingRule => write!(f, "{}", FlowType::RECORDING_RULE),
-            FlowType::Streaming => write!(f, "{}", FlowType::STREAMING),
-        }
-    }
-}
-
 /// The serializable data.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct CreateFlowData {
@@ -373,7 +335,6 @@ pub struct CreateFlowData {
     /// For verify if prev value is consistent when need to update flow metadata.
     /// only set when `or_replace` is true.
     pub(crate) prev_flow_info_value: Option<DeserializedValueWithBytes<FlowInfoValue>>,
-    pub(crate) flow_type: Option<FlowType>,
 }
 
 impl From<&CreateFlowData> for CreateRequest {
@@ -381,7 +342,7 @@ impl From<&CreateFlowData> for CreateRequest {
         let flow_id = value.flow_id.unwrap();
         let source_table_ids = &value.source_table_ids;
 
-        let mut req = CreateRequest {
+        CreateRequest {
             flow_id: Some(api::v1::FlowId { id: flow_id }),
             source_table_ids: source_table_ids
                 .iter()
@@ -395,11 +356,7 @@ impl From<&CreateFlowData> for CreateRequest {
             comment: value.task.comment.clone(),
             sql: value.task.sql.clone(),
             flow_options: value.task.flow_options.clone(),
-        };
-
-        let flow_type = value.flow_type.unwrap_or_default().to_string();
-        req.flow_options.insert("flow_type".to_string(), flow_type);
-        req
+        }
     }
 }
 
@@ -412,7 +369,7 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
             expire_after,
             comment,
             sql,
-            flow_options: mut options,
+            flow_options: options,
             ..
         } = value.task.clone();
 
@@ -429,21 +386,19 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
             .map(|(idx, peer)| (idx as u32, FlowRouteValue { peer: peer.clone() }))
             .collect::<Vec<_>>();
 
-        let flow_type = value.flow_type.unwrap_or_default().to_string();
-        options.insert("flow_type".to_string(), flow_type);
-
-        let flow_info = FlowInfoValue {
-            source_table_ids: value.source_table_ids.clone(),
-            sink_table_name,
-            flownode_ids,
-            catalog_name,
-            flow_name,
-            raw_sql: sql,
-            expire_after,
-            comment,
-            options,
-        };
-
-        (flow_info, flow_routes)
+        (
+            FlowInfoValue {
+                source_table_ids: value.source_table_ids.clone(),
+                sink_table_name,
+                flownode_ids,
+                catalog_name,
+                flow_name,
+                raw_sql: sql,
+                expire_after,
+                comment,
+                options,
+            },
+            flow_routes,
+        )
     }
 }

src/common/meta/src/ddl/drop_database/executor.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::any::Any;
-use std::collections::HashMap;
 
 use common_procedure::Status;
 use common_telemetry::info;
@@ -26,7 +25,6 @@ use super::cursor::DropDatabaseCursor;
 use super::{DropDatabaseContext, DropTableTarget};
 use crate::ddl::drop_database::State;
 use crate::ddl::drop_table::executor::DropTableExecutor;
-use crate::ddl::utils::extract_region_wal_options;
 use crate::ddl::DdlContext;
 use crate::error::{self, Result};
 use crate::key::table_route::TableRouteValue;
@@ -109,26 +107,12 @@ impl State for DropDatabaseExecutor {
             self.physical_table_id,
             self.physical_region_routes.clone(),
         );
-
-        // Deletes topic-region mapping if dropping physical table
-        let region_wal_options =
-            if let TableRouteValue::Physical(table_route_value) = &table_route_value {
-                let datanode_table_values = ddl_ctx
-                    .table_metadata_manager
-                    .datanode_table_manager()
-                    .regions(self.physical_table_id, table_route_value)
-                    .await?;
-                extract_region_wal_options(&datanode_table_values)?
-            } else {
-                HashMap::new()
-            };
-
         executor
-            .on_destroy_metadata(ddl_ctx, &table_route_value, &region_wal_options)
+            .on_destroy_metadata(ddl_ctx, &table_route_value)
             .await?;
         executor.invalidate_table_cache(ddl_ctx).await?;
         executor
-            .on_drop_regions(ddl_ctx, &self.physical_region_routes, true)
+            .on_drop_regions(ddl_ctx, &self.physical_region_routes)
             .await?;
         info!("Table: {}({}) is dropped", self.table_name, self.table_id);
 

src/common/meta/src/ddl/drop_flow/metadata.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use common_catalog::format_full_flow_name;
+use futures::TryStreamExt;
 use snafu::{ensure, OptionExt};
 
 use crate::ddl::drop_flow::DropFlowProcedure;
@@ -38,10 +39,9 @@ impl DropFlowProcedure {
             .flow_metadata_manager
             .flow_route_manager()
             .routes(self.data.task.flow_id)
-            .await?
-            .into_iter()
-            .map(|(_, value)| value)
-            .collect::<Vec<_>>();
+            .map_ok(|(_, value)| value)
+            .try_collect::<Vec<_>>()
+            .await?;
         ensure!(
             !flow_route_values.is_empty(),
             error::FlowRouteNotFoundSnafu {

src/common/meta/src/ddl/drop_table.rs

@@ -15,8 +15,6 @@
 pub(crate) mod executor;
 mod metadata;
 
-use std::collections::HashMap;
-
 use async_trait::async_trait;
 use common_error::ext::BoxedError;
 use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu};
@@ -26,10 +24,8 @@ use common_procedure::{
 };
 use common_telemetry::info;
 use common_telemetry::tracing::warn;
-use common_wal::options::WalOptions;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
-use store_api::storage::RegionNumber;
 use strum::AsRefStr;
 use table::metadata::TableId;
 use table::table_reference::TableReference;
@@ -135,11 +131,7 @@ impl DropTableProcedure {
         );
         // Deletes table metadata logically.
         self.executor
-            .on_delete_metadata(
-                &self.context,
-                table_route_value,
-                &self.data.region_wal_options,
-            )
+            .on_delete_metadata(&self.context, table_route_value)
             .await?;
         info!("Deleted table metadata for table {table_id}");
         self.data.state = DropTableState::InvalidateTableCache;
@@ -156,7 +148,7 @@ impl DropTableProcedure {
 
     pub async fn on_datanode_drop_regions(&mut self) -> Result<Status> {
         self.executor
-            .on_drop_regions(&self.context, &self.data.physical_region_routes, false)
+            .on_drop_regions(&self.context, &self.data.physical_region_routes)
             .await?;
         self.data.state = DropTableState::DeleteTombstone;
         Ok(Status::executing(true))
@@ -171,11 +163,7 @@ impl DropTableProcedure {
             self.data.physical_region_routes.clone(),
         );
         self.executor
-            .on_delete_metadata_tombstone(
-                &self.context,
-                table_route_value,
-                &self.data.region_wal_options,
-            )
+            .on_delete_metadata_tombstone(&self.context, table_route_value)
             .await?;
 
         self.dropping_regions.clear();
@@ -255,11 +243,7 @@ impl Procedure for DropTableProcedure {
             self.data.physical_region_routes.clone(),
         );
         self.executor
-            .on_restore_metadata(
-                &self.context,
-                table_route_value,
-                &self.data.region_wal_options,
-            )
+            .on_restore_metadata(&self.context, table_route_value)
             .await
             .map_err(ProcedureError::external)
     }
@@ -273,8 +257,6 @@ pub struct DropTableData {
     pub physical_region_routes: Vec<RegionRoute>,
     pub physical_table_id: Option<TableId>,
     #[serde(default)]
-    pub region_wal_options: HashMap<RegionNumber, WalOptions>,
-    #[serde(default)]
     pub allow_rollback: bool,
 }
 
@@ -286,7 +268,6 @@ impl DropTableData {
             task,
             physical_region_routes: vec![],
             physical_table_id: None,
-            region_wal_options: HashMap::new(),
             allow_rollback: false,
         }
     }

src/common/meta/src/ddl/drop_table/executor.rs

@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-
 use api::v1::region::{
     region_request, DropRequest as PbDropRegionRequest, RegionRequest, RegionRequestHeader,
 };
@@ -21,10 +19,9 @@ use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_telemetry::debug;
 use common_telemetry::tracing_context::TracingContext;
-use common_wal::options::WalOptions;
 use futures::future::join_all;
 use snafu::ensure;
-use store_api::storage::{RegionId, RegionNumber};
+use store_api::storage::RegionId;
 use table::metadata::TableId;
 use table::table_name::TableName;
 
@@ -116,15 +113,9 @@ impl DropTableExecutor {
         &self,
         ctx: &DdlContext,
         table_route_value: &TableRouteValue,
-        region_wal_options: &HashMap<RegionNumber, WalOptions>,
     ) -> Result<()> {
         ctx.table_metadata_manager
-            .delete_table_metadata(
-                self.table_id,
-                &self.table,
-                table_route_value,
-                region_wal_options,
-            )
+            .delete_table_metadata(self.table_id, &self.table, table_route_value)
             .await
     }
 
@@ -133,15 +124,9 @@ impl DropTableExecutor {
         &self,
         ctx: &DdlContext,
         table_route_value: &TableRouteValue,
-        region_wal_options: &HashMap<u32, WalOptions>,
     ) -> Result<()> {
         ctx.table_metadata_manager
-            .delete_table_metadata_tombstone(
-                self.table_id,
-                &self.table,
-                table_route_value,
-                region_wal_options,
-            )
+            .delete_table_metadata_tombstone(self.table_id, &self.table, table_route_value)
             .await
     }
 
@@ -150,15 +135,9 @@ impl DropTableExecutor {
         &self,
         ctx: &DdlContext,
         table_route_value: &TableRouteValue,
-        region_wal_options: &HashMap<u32, WalOptions>,
     ) -> Result<()> {
         ctx.table_metadata_manager
-            .destroy_table_metadata(
-                self.table_id,
-                &self.table,
-                table_route_value,
-                region_wal_options,
-            )
+            .destroy_table_metadata(self.table_id, &self.table, table_route_value)
             .await?;
 
         let detecting_regions = if table_route_value.is_physical() {
@@ -177,15 +156,9 @@ impl DropTableExecutor {
         &self,
         ctx: &DdlContext,
         table_route_value: &TableRouteValue,
-        region_wal_options: &HashMap<u32, WalOptions>,
     ) -> Result<()> {
         ctx.table_metadata_manager
-            .restore_table_metadata(
-                self.table_id,
-                &self.table,
-                table_route_value,
-                region_wal_options,
-            )
+            .restore_table_metadata(self.table_id, &self.table, table_route_value)
             .await
     }
 
@@ -214,7 +187,6 @@ impl DropTableExecutor {
         &self,
         ctx: &DdlContext,
         region_routes: &[RegionRoute],
-        fast_path: bool,
     ) -> Result<()> {
         let leaders = find_leaders(region_routes);
         let mut drop_region_tasks = Vec::with_capacity(leaders.len());
@@ -237,7 +209,6 @@ impl DropTableExecutor {
                     }),
                     body: Some(region_request::Body::Drop(PbDropRegionRequest {
                         region_id: region_id.as_u64(),
-                        fast_path,
                     })),
                 };
                 let datanode = datanode.clone();

src/common/meta/src/ddl/drop_table/metadata.rs

@@ -17,7 +17,6 @@ use snafu::OptionExt;
 use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
 
 use crate::ddl::drop_table::DropTableProcedure;
-use crate::ddl::utils::extract_region_wal_options;
 use crate::error::{self, Result};
 
 impl DropTableProcedure {
@@ -31,6 +30,9 @@ impl DropTableProcedure {
             .get_physical_table_route(task.table_id)
             .await?;
 
+        self.data.physical_region_routes = physical_table_route_value.region_routes;
+        self.data.physical_table_id = Some(physical_table_id);
+
         if physical_table_id == self.data.table_id() {
             let table_info_value = self
                 .context
@@ -45,21 +47,9 @@ impl DropTableProcedure {
 
             let engine = table_info_value.table_info.meta.engine;
             // rollback only if dropping the metric physical table fails
-            self.data.allow_rollback = engine.as_str() == METRIC_ENGINE_NAME;
-
-            // Deletes topic-region mapping if dropping physical table
-            let datanode_table_values = self
-                .context
-                .table_metadata_manager
-                .datanode_table_manager()
-                .regions(physical_table_id, &physical_table_route_value)
-                .await?;
-            self.data.region_wal_options = extract_region_wal_options(&datanode_table_values)?;
+            self.data.allow_rollback = engine.as_str() == METRIC_ENGINE_NAME
         }
 
-        self.data.physical_region_routes = physical_table_route_value.region_routes;
-        self.data.physical_table_id = Some(physical_table_id);
-
         Ok(())
     }
 }

src/common/meta/src/ddl/tests/create_view.rs

@@ -219,7 +219,7 @@ async fn test_replace_view_metadata() {
         assert_eq!(err.status_code(), StatusCode::TableAlreadyExists);
     }
 
-    // Set `or_replace` to be `true` and try again
+    // Set `or_replce` to be `true` and try again
     task.create_view.or_replace = true;
     task.create_view.logical_plan = vec![4, 5, 6];
     task.create_view.definition = "new_definition".to_string();

src/common/meta/src/ddl/utils.rs

@@ -12,23 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-
 use common_catalog::consts::METRIC_ENGINE;
 use common_error::ext::BoxedError;
 use common_procedure::error::Error as ProcedureError;
-use common_wal::options::WalOptions;
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::metric_engine_consts::LOGICAL_TABLE_METADATA_KEY;
-use store_api::storage::RegionNumber;
 use table::metadata::TableId;
 use table::table_reference::TableReference;
 
 use crate::ddl::DetectingRegion;
-use crate::error::{
-    Error, OperateDatanodeSnafu, ParseWalOptionsSnafu, Result, TableNotFoundSnafu, UnsupportedSnafu,
-};
-use crate::key::datanode_table::DatanodeTableValue;
+use crate::error::{Error, OperateDatanodeSnafu, Result, TableNotFoundSnafu, UnsupportedSnafu};
 use crate::key::table_name::TableNameKey;
 use crate::key::TableMetadataManagerRef;
 use crate::peer::Peer;
@@ -158,32 +151,6 @@ pub fn convert_region_routes_to_detecting_regions(
         .collect::<Vec<_>>()
 }
 
-/// Parses [WalOptions] from serialized strings in hashmap.
-pub fn parse_region_wal_options(
-    serialized_options: &HashMap<RegionNumber, String>,
-) -> Result<HashMap<RegionNumber, WalOptions>> {
-    let mut region_wal_options = HashMap::with_capacity(serialized_options.len());
-    for (region_number, wal_options) in serialized_options {
-        let wal_option = serde_json::from_str::<WalOptions>(wal_options)
-            .context(ParseWalOptionsSnafu { wal_options })?;
-        region_wal_options.insert(*region_number, wal_option);
-    }
-    Ok(region_wal_options)
-}
-
-/// Extracts region wal options from [DatanodeTableValue]s.
-pub fn extract_region_wal_options(
-    datanode_table_values: &Vec<DatanodeTableValue>,
-) -> Result<HashMap<RegionNumber, WalOptions>> {
-    let mut region_wal_options = HashMap::new();
-    for value in datanode_table_values {
-        let serialized_options = &value.region_info.region_wal_options;
-        let parsed_options = parse_region_wal_options(serialized_options)?;
-        region_wal_options.extend(parsed_options);
-    }
-    Ok(region_wal_options)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;