refactor: address review comments

fix: toml format
fix: lint error
2025-12-27 00:19:58 +00:00 · 2024-09-02 11:43:33 -07:00 · 2024-08-31 10:32:25 -07:00 · 2024-08-31 10:31:45 -07:00 · 2024-08-31 10:28:20 -07:00 · 2024-08-31 10:25:19 -07:00
206 changed files with 3734 additions and 1631 deletions
--- a/.github/actions/build-linux-artifacts/action.yml
+++ b/.github/actions/build-linux-artifacts/action.yml
@@ -17,6 +17,12 @@ inputs:
    description: Enable dev mode, only build standard greptime
    required: false
    default: "false"
+  image-namespace:
+    description: Image Namespace
+    required: true
+  image-registry:
+    description: Image Registry
+    required: true
  working-dir:
    description: Working directory to build the artifacts
    required: false
@@ -31,8 +37,8 @@ runs:
      run: |
        cd ${{ inputs.working-dir }} && \
        make run-it-in-container BUILD_JOBS=4 \
-        IMAGE_NAMESPACE=i8k6a5e1/greptime \
-        IMAGE_REGISTRY=public.ecr.aws
+        IMAGE_NAMESPACE=${{ inputs.image-namespace }} \
+        IMAGE_REGISTRY=${{ inputs.image-registry }}

    - name: Upload sqlness logs
      if: ${{ failure() && inputs.disable-run-tests == 'false' }} # Only upload logs when the integration tests failed.
@@ -51,8 +57,8 @@ runs:
        artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }}
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime        
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}

    - name: Build greptime without pyo3
      if: ${{ inputs.dev-mode == 'false' }}
@@ -64,8 +70,8 @@ runs:
        artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}

    - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build.
      shell: bash
@@ -82,8 +88,8 @@ runs:
        artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}

    - name: Build greptime on android base image
      uses: ./.github/actions/build-greptime-binary
@@ -94,5 +100,5 @@ runs:
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
        build-android-artifacts: true
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}
--- a/.github/actions/build-macos-artifacts/action.yml
+++ b/.github/actions/build-macos-artifacts/action.yml
@@ -4,9 +4,6 @@ inputs:
  arch:
    description: Architecture to build
    required: true
-  rust-toolchain:
-    description: Rust toolchain to use
-    required: true
  cargo-profile:
    description: Cargo profile to build
    required: true
@@ -43,9 +40,8 @@ runs:
        brew install protobuf

    - name: Install rust toolchain
-      uses: dtolnay/rust-toolchain@master
+      uses: actions-rust-lang/setup-rust-toolchain@v1
      with:
-        toolchain: ${{ inputs.rust-toolchain }}
        targets: ${{ inputs.arch }}

    - name: Start etcd # For integration tests.
--- a/.github/actions/build-windows-artifacts/action.yml
+++ b/.github/actions/build-windows-artifacts/action.yml
@@ -4,9 +4,6 @@ inputs:
  arch:
    description: Architecture to build
    required: true
-  rust-toolchain:
-    description: Rust toolchain to use
-    required: true
  cargo-profile:
    description: Cargo profile to build
    required: true
@@ -28,9 +25,8 @@ runs:
    - uses: arduino/setup-protoc@v3

    - name: Install rust toolchain
-      uses: dtolnay/rust-toolchain@master
+      uses: actions-rust-lang/setup-rust-toolchain@v1
      with:
-        toolchain: ${{ inputs.rust-toolchain }}
        targets: ${{ inputs.arch }}
        components: llvm-tools-preview

--- a/.github/workflows/apidoc.yml
+++ b/.github/workflows/apidoc.yml
@@ -12,9 +12,6 @@ on:

 name: Build API docs

-env:
-  RUST_TOOLCHAIN: nightly-2024-04-20
-
 jobs:
  apidoc:
    runs-on: ubuntu-20.04
@@ -23,9 +20,7 @@ jobs:
    - uses: arduino/setup-protoc@v3
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - uses: dtolnay/rust-toolchain@master
-      with:
-        toolchain: ${{ env.RUST_TOOLCHAIN }}
+    - uses: actions-rust-lang/setup-rust-toolchain@v1
    - run: cargo doc --workspace --no-deps --document-private-items
    - run: |
        cat <<EOF > target/doc/index.html
--- a/.github/workflows/dev-build.yml
+++ b/.github/workflows/dev-build.yml
@@ -177,6 +177,8 @@ jobs:
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          dev-mode: true # Only build the standard greptime binary.
          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-linux-arm64-artifacts:
    name: Build linux-arm64 artifacts
@@ -206,6 +208,8 @@ jobs:
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          dev-mode: true # Only build the standard greptime binary.
          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  release-images-to-dockerhub:
    name: Build and push images to DockerHub
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -29,9 +29,6 @@ concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

-env:
-  RUST_TOOLCHAIN: nightly-2024-04-20
-
 jobs:
  check-typos-and-docs:
    name: Check typos and docs
@@ -64,9 +61,7 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
@@ -82,9 +77,7 @@ jobs:
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: stable
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
@@ -107,9 +100,7 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - uses: Swatinem/rust-cache@v2
        with:
          # Shares across multiple jobs
@@ -161,9 +152,7 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
@@ -181,7 +170,7 @@ jobs:
          name: bins
          path: .
      - name: Unzip binaries
-        run: | 
+        run: |
          tar -xvf ./bins.tar.gz
          rm ./bins.tar.gz
      - name: Run GreptimeDB
@@ -221,9 +210,7 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
@@ -274,9 +261,7 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - uses: Swatinem/rust-cache@v2
        with:
          # Shares across multiple jobs
@@ -287,7 +272,7 @@ jobs:
      - name: Build greptime bianry
        shell: bash
        # `cargo gc` will invoke `cargo build` with specified args
-        run: cargo gc --profile ci -- --bin greptime 
+        run: cargo gc --profile ci -- --bin greptime
      - name: Pack greptime binary
        shell: bash
        run: |
@@ -301,7 +286,7 @@ jobs:
          artifacts-dir: bin
          version: current

-  distributed-fuzztest: 
+  distributed-fuzztest:
    name: Fuzz Test (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
    runs-on: ubuntu-latest
    needs:  build-greptime-ci
@@ -344,9 +329,7 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
@@ -416,12 +399,12 @@ jobs:
      - name: Describe Nodes
        if: failure()
        shell: bash
-        run: | 
-          kubectl describe nodes      
+        run: |
+          kubectl describe nodes
      - name: Export kind logs
        if: failure()
        shell: bash
-        run: | 
+        run: |
          kind export logs /tmp/kind
      - name: Upload logs
        if: failure()
@@ -433,13 +416,13 @@ jobs:
      - name: Delete cluster
        if: success()
        shell: bash
-        run: | 
+        run: |
          kind delete cluster
          docker stop $(docker ps -a -q)
          docker rm $(docker ps -a -q)
          docker system prune -f

-  distributed-fuzztest-with-chaos: 
+  distributed-fuzztest-with-chaos:
    name: Fuzz Test with Chaos (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
    runs-on: ubuntu-latest
    needs:  build-greptime-ci
@@ -447,7 +430,7 @@ jobs:
    strategy:
      matrix:
        target: ["fuzz_migrate_mito_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
-        mode: 
+        mode:
          - name: "Remote WAL"
            minio: true
            kafka: true
@@ -484,9 +467,7 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
@@ -557,12 +538,12 @@ jobs:
      - name: Describe Nodes
        if: failure()
        shell: bash
-        run: | 
-          kubectl describe nodes      
+        run: |
+          kubectl describe nodes
      - name: Export kind logs
        if: failure()
        shell: bash
-        run: | 
+        run: |
          kind export logs /tmp/kind
      - name: Upload logs
        if: failure()
@@ -574,7 +555,7 @@ jobs:
      - name: Delete cluster
        if: success()
        shell: bash
-        run: | 
+        run: |
          kind delete cluster
          docker stop $(docker ps -a -q)
          docker rm $(docker ps -a -q)
@@ -627,9 +608,8 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
          components: rustfmt
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
@@ -648,9 +628,8 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
          components: clippy
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
@@ -674,9 +653,8 @@ jobs:
        with:
          version: "14.0"
      - name: Install toolchain
-        uses: dtolnay/rust-toolchain@master
+        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
          components: llvm-tools-preview
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
--- a/.github/workflows/nightly-build.yml
+++ b/.github/workflows/nightly-build.yml
@@ -154,6 +154,8 @@ jobs:
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-linux-arm64-artifacts:
    name: Build linux-arm64 artifacts
@@ -173,6 +175,8 @@ jobs:
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  release-images-to-dockerhub:
    name: Build and push images to DockerHub
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -9,9 +9,6 @@ concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

-env:
-  RUST_TOOLCHAIN: nightly-2024-04-20
-
 permissions:
  issues: write

@@ -52,9 +49,7 @@ jobs:
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
      - name: Run sqlness
@@ -85,9 +80,8 @@ jobs:
        with:
          version: "14.0"
      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@master
+        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
          components: llvm-tools-preview
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
--- a/.github/workflows/release-dev-builder-images.yaml
+++ b/.github/workflows/release-dev-builder-images.yaml
@@ -1,12 +1,14 @@
 name: Release dev-builder images

 on:
+  push:
+    branches:
+      - main
+    paths:
+      - rust-toolchain.toml
+      - 'docker/dev-builder/**'
  workflow_dispatch: # Allows you to run this workflow manually.
    inputs:
-      version:
-        description: Version of the dev-builder
-        required: false
-        default: latest
      release_dev_builder_ubuntu_image:
        type: boolean
        description: Release dev-builder-ubuntu image
@@ -28,22 +30,103 @@ jobs:
    name: Release dev builder images
    if: ${{ inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }} # Only manually trigger this job.
    runs-on: ubuntu-20.04-16-cores
+    outputs:
+      version: ${{ steps.set-version.outputs.version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

+      - name: Configure build image version
+        id: set-version
+        shell: bash
+        run: |
+          commitShortSHA=`echo ${{ github.sha }} | cut -c1-8`
+          buildTime=`date +%Y%m%d%H%M%S`
+          BUILD_VERSION="$commitShortSHA-$buildTime"
+          RUST_TOOLCHAIN_VERSION=$(cat rust-toolchain.toml | grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}')
+          IMAGE_VERSION="${RUST_TOOLCHAIN_VERSION}-${BUILD_VERSION}"
+          echo "VERSION=${IMAGE_VERSION}" >> $GITHUB_ENV
+          echo "version=$IMAGE_VERSION" >> $GITHUB_OUTPUT
+
      - name: Build and push dev builder images
        uses: ./.github/actions/build-dev-builder-images
        with:
-          version: ${{ inputs.version }}
+          version: ${{ env.VERSION }}
          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
          build-dev-builder-ubuntu: ${{ inputs.release_dev_builder_ubuntu_image }}
          build-dev-builder-centos: ${{ inputs.release_dev_builder_centos_image }}
          build-dev-builder-android: ${{ inputs.release_dev_builder_android_image }}

+  release-dev-builder-images-ecr:
+    name: Release dev builder images to AWS ECR
+    runs-on: ubuntu-20.04
+    needs: [
+      release-dev-builder-images
+    ]
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.ECR_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr-public
+        uses: aws-actions/amazon-ecr-login@v2
+        env:
+          AWS_REGION: ${{ vars.ECR_REGION }}
+        with:
+          registry-type: public
+
+      - name: Push dev-builder-ubuntu image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_ubuntu_image }}
+        run: |
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }}
+
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-ubuntu:latest
+      - name: Push dev-builder-centos image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_centos_image }}
+        run: |
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }}
+
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-centos:latest
+      - name: Push dev-builder-android image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_android_image }}
+        run: |
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }}
+
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-android:latest
  release-dev-builder-images-cn: # Note: Be careful issue: https://github.com/containers/skopeo/issues/1874 and we decide to use the latest stable skopeo container.
    name: Release dev builder images to CN region
    runs-on: ubuntu-20.04
@@ -51,35 +134,39 @@ jobs:
      release-dev-builder-images
    ]
    steps:
+      - name: Login to AliCloud Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          username: ${{ secrets.ALICLOUD_USERNAME }}
+          password: ${{ secrets.ALICLOUD_PASSWORD }}
+
      - name: Push dev-builder-ubuntu image
        shell: bash
        if: ${{ inputs.release_dev_builder_ubuntu_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
-            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }}
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }}

      - name: Push dev-builder-centos image
        shell: bash
        if: ${{ inputs.release_dev_builder_centos_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
-            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }}
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }}

      - name: Push dev-builder-android image
        shell: bash
        if: ${{ inputs.release_dev_builder_android_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
-            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }}
+          docker run -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \
+            -e "REGISTRY_AUTH_FILE=/root/.docker/config.json" \
+            quay.io/skopeo/stable:latest \
+            copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }}
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -82,7 +82,6 @@ on:
 # Use env variables to control all the release process.
 env:
  # The arguments of building greptime.
-  RUST_TOOLCHAIN: nightly-2024-04-20
  CARGO_PROFILE: nightly

  # Controls whether to run tests, include unit-test, integration-test and sqlness.
@@ -99,6 +98,16 @@ permissions:
  contents: write # Allows the action to create a release.

 jobs:
+  check-builder-rust-version:
+    name: Check rust version in builder
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check Rust toolchain version
+        shell: bash
+        run: |
+          ./scripts/check-builder-rust-version.sh
+
  allocate-runners:
    name: Allocate runners
    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
@@ -183,6 +192,8 @@ jobs:
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-linux-arm64-artifacts:
    name: Build linux-arm64 artifacts
@@ -202,6 +213,8 @@ jobs:
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-macos-artifacts:
    name: Build macOS artifacts
@@ -240,7 +253,6 @@ jobs:
      - uses: ./.github/actions/build-macos-artifacts
        with:
          arch: ${{ matrix.arch }}
-          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
          cargo-profile: ${{ env.CARGO_PROFILE }}
          features: ${{ matrix.features }}
          version: ${{ needs.allocate-runners.outputs.version }}
@@ -283,7 +295,6 @@ jobs:
      - uses: ./.github/actions/build-windows-artifacts
        with:
          arch: ${{ matrix.arch }}
-          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
          cargo-profile: ${{ env.CARGO_PROFILE }}
          features: ${{ matrix.features }}
          version: ${{ needs.allocate-runners.outputs.version }}
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -897,6 +897,15 @@ dependencies = [
 "tokio",
 ]

+[[package]]
+name = "backon"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2caf634d05fe0642d0fb1ab43497fa627088ecd93f84b2d0f2a5d7b91f7730db"
+dependencies = [
+ "fastrand",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.73"
@@ -1941,6 +1950,8 @@ dependencies = [
 "common-version",
 "datafusion",
 "datatypes",
+ "geohash",
+ "h3o",
 "num",
 "num-traits",
 "once_cell",
@@ -1954,6 +1965,7 @@ dependencies = [
 "statrs",
 "store-api",
 "table",
+ "tokio",
 ]

 [[package]]
@@ -2110,7 +2122,7 @@ version = "0.9.2"
 dependencies = [
 "async-stream",
 "async-trait",
- "backon",
+ "backon 1.0.2",
 "common-base",
 "common-error",
 "common-macro",
@@ -3803,6 +3815,12 @@ dependencies = [
 "num-traits",
 ]

+[[package]]
+name = "float_eq"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853"
+
 [[package]]
 name = "flow"
 version = "0.9.2"
@@ -4201,6 +4219,27 @@ dependencies = [
 "version_check",
 ]

+[[package]]
+name = "geo-types"
+version = "0.7.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ff16065e5720f376fbced200a5ae0f47ace85fd70b7e54269790281353b6d61"
+dependencies = [
+ "approx",
+ "num-traits",
+ "serde",
+]
+
+[[package]]
+name = "geohash"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fb94b1a65401d6cbf22958a9040aa364812c26674f841bee538b12c135db1e6"
+dependencies = [
+ "geo-types",
+ "libm",
+]
+
 [[package]]
 name = "gethostname"
 version = "0.2.3"
@@ -4291,6 +4330,25 @@ dependencies = [
 "tracing",
 ]

+[[package]]
+name = "h3o"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0de3592e1f699692aa0525c42ff7879ec3ee7e36329af20967bc910a1cdc39c7"
+dependencies = [
+ "ahash 0.8.11",
+ "either",
+ "float_eq",
+ "h3o-bit",
+ "libm",
+]
+
+[[package]]
+name = "h3o-bit"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fb45e8060378c0353781abf67e1917b545a6b710d0342d85b70c125af7ef320"
+
 [[package]]
 name = "half"
 version = "1.8.3"
@@ -4568,9 +4626,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"

 [[package]]
 name = "human-panic"
-version = "1.2.3"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4f016c89920bbb30951a8405ecacbb4540db5524313b9445736e7e1855cf370"
+checksum = "1c5a08ed290eac04006e21e63d32e90086b6182c7cd0452d10f4264def1fec9a"
 dependencies = [
 "anstream",
 "anstyle",
@@ -5816,6 +5874,7 @@ dependencies = [
 "common-time",
 "common-wal",
 "delta-encoding",
+ "derive_builder 0.12.0",
 "futures",
 "futures-util",
 "itertools 0.10.5",
@@ -7010,13 +7069,13 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"

 [[package]]
 name = "opendal"
-version = "0.48.0"
+version = "0.49.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "615d41187deea0ea7fab5b48e9afef6ae8fc742fdcfa248846ee3d92ff71e986"
+checksum = "ba615070686c8781ce97376fdafca29d7c42f47b31d2230d7c8c1642ec823950"
 dependencies = [
 "anyhow",
 "async-trait",
- "backon",
+ "backon 0.4.4",
 "base64 0.22.1",
 "bytes",
 "chrono",
@@ -9170,8 +9229,9 @@ dependencies = [

 [[package]]
 name = "rsasl"
-version = "2.0.2"
-source = "git+https://github.com/wenyxu/rsasl.git?rev=06ebb683d5539c3410de4ce9fa37ff9b97e790a4#06ebb683d5539c3410de4ce9fa37ff9b97e790a4"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45035615cdd68c71daac89aef75b130d4b2cad29599966e1b4671f8fbb463559"
 dependencies = [
 "base64 0.22.1",
 "core2",
@@ -9188,9 +9248,8 @@ dependencies = [
 [[package]]
 name = "rskafka"
 version = "0.5.0"
-source = "git+https://github.com/WenyXu/rskafka.git?rev=940c6030012c5b746fad819fb72e3325b26e39de#940c6030012c5b746fad819fb72e3325b26e39de"
+source = "git+https://github.com/influxdata/rskafka.git?rev=75535b5ad9bae4a5dbb582c82e44dfd81ec10105#75535b5ad9bae4a5dbb582c82e44dfd81ec10105"
 dependencies = [
- "async-trait",
 "bytes",
 "chrono",
 "crc32c",
@@ -9199,7 +9258,6 @@ dependencies = [
 "integer-encoding 4.0.0",
 "lz4",
 "parking_lot 0.12.3",
- "pin-project-lite",
 "rand",
 "rsasl",
 "rustls 0.23.10",
@@ -11800,9 +11858,9 @@ dependencies = [

 [[package]]
 name = "tikv-jemalloc-ctl"
-version = "0.5.4"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "619bfed27d807b54f7f776b9430d4f8060e66ee138a28632ca898584d462c31c"
+checksum = "f21f216790c8df74ce3ab25b534e0718da5a1916719771d3fec23315c99e468b"
 dependencies = [
 "libc",
 "paste",
@@ -11811,9 +11869,9 @@ dependencies = [

 [[package]]
 name = "tikv-jemalloc-sys"
-version = "0.5.4+5.3.0-patched"
+version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1"
+checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d"
 dependencies = [
 "cc",
 "libc",
@@ -11821,9 +11879,9 @@ dependencies = [

 [[package]]
 name = "tikv-jemallocator"
-version = "0.5.4"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca"
+checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865"
 dependencies = [
 "libc",
 "tikv-jemalloc-sys",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -77,6 +77,7 @@ clippy.readonly_write_lock = "allow"
 rust.unknown_lints = "deny"
 # Remove this after https://github.com/PyO3/pyo3/issues/4094
 rust.non_local_definitions = "allow"
+rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }

 [workspace.dependencies]
 # We turn off default-features for some dependencies here so the workspaces which inherit them can
@@ -151,8 +152,7 @@ reqwest = { version = "0.12", default-features = false, features = [
    "stream",
    "multipart",
 ] }
-# SCRAM-SHA-512 requires https://github.com/dequbed/rsasl/pull/48, https://github.com/influxdata/rskafka/pull/247 
-rskafka = { git = "https://github.com/WenyXu/rskafka.git", rev = "940c6030012c5b746fad819fb72e3325b26e39de", features = [
+rskafka = { git = "https://github.com/influxdata/rskafka.git", rev = "75535b5ad9bae4a5dbb582c82e44dfd81ec10105", features = [
    "transport-tls",
 ] }
 rstest = "0.21"
@@ -251,7 +251,7 @@ debug = 1

 [profile.nightly]
 inherits = "release"
-strip = true
+strip = "debuginfo"
 lto = "thin"
 debug = false
 incremental = false
--- a/13
+++ b/13
@@ -8,6 +8,7 @@ CARGO_BUILD_OPTS := --locked
 IMAGE_REGISTRY ?= docker.io
 IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
+DEV_BUILDER_IMAGE_TAG ?= 2024-06-06-b4b105ad-20240827021230
 BUILDX_MULTI_PLATFORM_BUILD ?= false
 BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu
@@ -77,7 +78,7 @@ build: ## Build debug version greptime.
 build-by-dev-builder: ## Build greptime by dev-builder.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${DEV_BUILDER_IMAGE_TAG} \
 	make build \
 	CARGO_EXTENSION="${CARGO_EXTENSION}" \
 	CARGO_PROFILE=${CARGO_PROFILE} \
@@ -91,7 +92,7 @@ build-by-dev-builder: ## Build greptime by dev-builder.
 build-android-bin: ## Build greptime binary for android.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:${DEV_BUILDER_IMAGE_TAG} \
 	make build \
 	CARGO_EXTENSION="ndk --platform 23 -t aarch64-linux-android" \
 	CARGO_PROFILE=release \
@@ -105,8 +106,8 @@ build-android-bin: ## Build greptime binary for android.
 strip-android-bin: build-android-bin ## Strip greptime binary for android.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \
-	bash -c '$${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip /greptimedb/target/aarch64-linux-android/release/greptime'
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:${DEV_BUILDER_IMAGE_TAG} \
+	bash -c '$${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip --strip-debug /greptimedb/target/aarch64-linux-android/release/greptime'

 .PHONY: clean
 clean: ## Clean the project.
@@ -145,7 +146,7 @@ dev-builder: multi-platform-buildx ## Build dev-builder image.
 	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
 	--build-arg="RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" \
 	-f docker/dev-builder/${BASE_IMAGE}/Dockerfile \
-	-t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
+	-t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${DEV_BUILDER_IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .

 .PHONY: multi-platform-buildx
 multi-platform-buildx: ## Create buildx multi-platform builder.
@@ -203,7 +204,7 @@ stop-etcd: ## Stop single node etcd for testing purpose.
 run-it-in-container: start-etcd ## Run integration tests in dev-builder.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v /tmp:/tmp \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${DEV_BUILDER_IMAGE_TAG} \
 	make test sqlness-test BUILD_JOBS=${BUILD_JOBS}

 .PHONY: start-cluster
--- a/README.md
+++ b/README.md
@@ -150,7 +150,7 @@ Our official Grafana dashboard is available at [grafana](grafana/README.md) dire

 ## Project Status

-The current version has not yet reached the standards for General Availability. 
+The current version has not yet reached the standards for General Availability.
 According to our Greptime 2024 Roadmap, we aim to achieve a production-level version with the release of v1.0 by the end of 2024. [Join Us](https://github.com/GreptimeTeam/greptimedb/issues/3412)

 We welcome you to test and use GreptimeDB. Some users have already adopted it in their production environments. If you're interested in trying it out, please use the latest stable release available.
@@ -172,6 +172,13 @@ In addition, you may:
 - Connect us with [Linkedin](https://www.linkedin.com/company/greptime/)
 - Follow us on [Twitter](https://twitter.com/greptime)

+## Commerial Support
+
+If you are running GreptimeDB OSS in your organization, we offer additional
+enterprise addons, installation service, training and consulting. [Contact
+us](https://greptime.com/contactus) and we will reach out to you with more
+detail of our commerial license.
+
 ## License

 GreptimeDB uses the [Apache License 2.0](https://apache.org/licenses/LICENSE-2.0.txt) to strike a balance between
--- a/config/config.md
+++ b/config/config.md
@@ -67,9 +67,10 @@
 | `wal.prefill_log_files` | Bool | `false` | Whether to pre-create log files on start up.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.sync_period` | String | `10s` | Duration for fsyncing log files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.broker_endpoints` | Array | -- | The Kafka broker endpoints.<br/>**It's only used when the provider is `kafka`**. |
-| `wal.num_topics` | Integer | `64` | Number of topics to be created upon start.<br/>**It's only used when the provider is `kafka`**. |
+| `wal.auto_create_topics` | Bool | `true` | Automatically create topics for WAL.<br/>Set to `true` to automatically create topics for WAL.<br/>Otherwise, use topics named `topic_name_prefix_[0..num_topics)` |
+| `wal.num_topics` | Integer | `64` | Number of topics.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.selector_type` | String | `round_robin` | Topic selector type.<br/>Available selector types:<br/>- `round_robin` (default)<br/>**It's only used when the provider is `kafka`**. |
-| `wal.topic_name_prefix` | String | `greptimedb_wal_topic` | A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.<br/>**It's only used when the provider is `kafka`**. |
+| `wal.topic_name_prefix` | String | `greptimedb_wal_topic` | A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.<br/>i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.replication_factor` | Integer | `1` | Expected number of replicas of each partition.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.create_topic_timeout` | String | `30s` | Above which a topic creation operation will be cancelled.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.max_batch_bytes` | String | `1MB` | The max size of a single producer batch.<br/>Warning: Kafka has a default limit of 1MB per message in a topic.<br/>**It's only used when the provider is `kafka`**. |
@@ -116,11 +117,12 @@
 | `region_engine.mito.global_write_buffer_reject_size` | String | `2GB` | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` |
 | `region_engine.mito.sst_meta_cache_size` | String | `128MB` | Cache size for SST metadata. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. |
 | `region_engine.mito.vector_cache_size` | String | `512MB` | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
-| `region_engine.mito.page_cache_size` | String | `512MB` | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
+| `region_engine.mito.page_cache_size` | String | `512MB` | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
+| `region_engine.mito.selector_result_cache_size` | String | `512MB` | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. |
 | `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. |
 | `region_engine.mito.experimental_write_cache_size` | String | `512MB` | Capacity for write cache. |
-| `region_engine.mito.experimental_write_cache_ttl` | String | `1h` | TTL for write cache. |
+| `region_engine.mito.experimental_write_cache_ttl` | String | `None` | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.scan_parallelism` | Integer | `0` | Parallelism to scan a region (default: 1/4 of cpu cores).<br/>- `0`: using the default value (1/4 of cpu cores).<br/>- `1`: scan in current thread.<br/>- `n`: scan in parallelism n. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
@@ -158,10 +160,10 @@
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
-| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself |
+| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
 | `export_metrics.self_import.db` | String | `None` | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
-| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. |
+| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
@@ -243,10 +245,10 @@
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
-| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself |
+| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
 | `export_metrics.self_import.db` | String | `None` | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
-| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. |
+| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
@@ -259,12 +261,13 @@
 | `data_home` | String | `/tmp/metasrv/` | The working home directory. |
 | `bind_addr` | String | `127.0.0.1:3002` | The bind address of metasrv. |
 | `server_addr` | String | `127.0.0.1:3002` | The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost. |
-| `store_addr` | String | `127.0.0.1:2379` | Etcd server address. |
+| `store_addr` | String | `127.0.0.1:2379` | Store server address default to etcd store. |
 | `selector` | String | `round_robin` | Datanode selector type.<br/>- `round_robin` (default value)<br/>- `lease_based`<br/>- `load_based`<br/>For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector". |
 | `use_memory_store` | Bool | `false` | Store data in memory. |
 | `enable_telemetry` | Bool | `true` | Whether to enable greptimedb telemetry. |
 | `store_key_prefix` | String | `""` | If it's not empty, the metasrv will store all data with this key prefix. |
 | `enable_region_failover` | Bool | `false` | Whether to enable region failover.<br/>This feature is only available on GreptimeDB running on cluster mode and<br/>- Using Remote WAL<br/>- Using shared storage (e.g., s3). |
+| `backend` | String | `EtcdStore` | The datastore for meta server. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
@@ -285,9 +288,10 @@
 | `wal` | -- | -- | -- |
 | `wal.provider` | String | `raft_engine` | -- |
 | `wal.broker_endpoints` | Array | -- | The broker endpoints of the Kafka cluster. |
-| `wal.num_topics` | Integer | `64` | Number of topics to be created upon start. |
+| `wal.auto_create_topics` | Bool | `true` | Automatically create topics for WAL.<br/>Set to `true` to automatically create topics for WAL.<br/>Otherwise, use topics named `topic_name_prefix_[0..num_topics)` |
+| `wal.num_topics` | Integer | `64` | Number of topics. |
 | `wal.selector_type` | String | `round_robin` | Topic selector type.<br/>Available selector types:<br/>- `round_robin` (default) |
-| `wal.topic_name_prefix` | String | `greptimedb_wal_topic` | A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`. |
+| `wal.topic_name_prefix` | String | `greptimedb_wal_topic` | A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.<br/>i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1. |
 | `wal.replication_factor` | Integer | `1` | Expected number of replicas of each partition. |
 | `wal.create_topic_timeout` | String | `30s` | Above which a topic creation operation will be cancelled. |
 | `wal.backoff_init` | String | `500ms` | The initial backoff for kafka clients. |
@@ -305,10 +309,10 @@
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
-| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself |
+| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
 | `export_metrics.self_import.db` | String | `None` | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
-| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. |
+| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
@@ -329,6 +333,10 @@
 | `rpc_runtime_size` | Integer | `None` | Deprecated, use `grpc.runtime_size` instead. |
 | `rpc_max_recv_message_size` | String | `None` | Deprecated, use `grpc.rpc_max_recv_message_size` instead. |
 | `rpc_max_send_message_size` | String | `None` | Deprecated, use `grpc.rpc_max_send_message_size` instead. |
+| `http` | -- | -- | The HTTP server options. |
+| `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
+| `http.timeout` | String | `30s` | HTTP request timeout. Set to 0 to disable timeout. |
+| `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:3001` | The address to bind the gRPC server. |
 | `grpc.hostname` | String | `127.0.0.1` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
@@ -408,11 +416,12 @@
 | `region_engine.mito.global_write_buffer_reject_size` | String | `2GB` | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` |
 | `region_engine.mito.sst_meta_cache_size` | String | `128MB` | Cache size for SST metadata. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. |
 | `region_engine.mito.vector_cache_size` | String | `512MB` | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
-| `region_engine.mito.page_cache_size` | String | `512MB` | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
+| `region_engine.mito.page_cache_size` | String | `512MB` | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
+| `region_engine.mito.selector_result_cache_size` | String | `512MB` | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. |
 | `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. |
 | `region_engine.mito.experimental_write_cache_size` | String | `512MB` | Capacity for write cache. |
-| `region_engine.mito.experimental_write_cache_ttl` | String | `1h` | TTL for write cache. |
+| `region_engine.mito.experimental_write_cache_ttl` | String | `None` | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.scan_parallelism` | Integer | `0` | Parallelism to scan a region (default: 1/4 of cpu cores).<br/>- `0`: using the default value (1/4 of cpu cores).<br/>- `1`: scan in current thread.<br/>- `n`: scan in parallelism n. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
@@ -448,10 +457,10 @@
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
-| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself |
+| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
 | `export_metrics.self_import.db` | String | `None` | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
-| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. |
+| `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -39,6 +39,18 @@ rpc_max_recv_message_size = "512MB"
 ## +toml2docs:none-default
 rpc_max_send_message_size = "512MB"

+
+## The HTTP server options.
+[http]
+## The address to bind the HTTP server.
+addr = "127.0.0.1:4000"
+## HTTP request timeout. Set to 0 to disable timeout.
+timeout = "30s"
+## HTTP request body limit.
+## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
+## Set to 0 to disable limit.
+body_limit = "64MB"
+
 ## The gRPC server options.
 [grpc]
 ## The address to bind the gRPC server.
@@ -394,9 +406,13 @@ sst_meta_cache_size = "128MB"
 vector_cache_size = "512MB"

 ## Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
-## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
+## If not set, it's default to 1/8 of OS memory.
 page_cache_size = "512MB"

+## Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.
+## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
+selector_result_cache_size = "512MB"
+
 ## Whether to enable the experimental write cache.
 enable_experimental_write_cache = false

@@ -407,7 +423,8 @@ experimental_write_cache_path = ""
 experimental_write_cache_size = "512MB"

 ## TTL for write cache.
-experimental_write_cache_ttl = "1h"
+## +toml2docs:none-default
+experimental_write_cache_ttl = "8h"

 ## Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
@@ -547,12 +564,13 @@ enable = false
 write_interval = "30s"

 ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
+## You must create the database before enabling it.
 [export_metrics.self_import]
 ## +toml2docs:none-default
-db = "information_schema"
+db = "greptime_metrics"

 [export_metrics.remote_write]
-## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`.
+## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
 url = ""

 ## HTTP headers of Prometheus remote-write carry.
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -199,12 +199,13 @@ enable = false
 write_interval = "30s"

 ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
+## You must create the database before enabling it.
 [export_metrics.self_import]
 ## +toml2docs:none-default
-db = "information_schema"
+db = "greptime_metrics"

 [export_metrics.remote_write]
-## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`.
+## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
 url = ""

 ## HTTP headers of Prometheus remote-write carry.
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -7,7 +7,7 @@ bind_addr = "127.0.0.1:3002"
 ## The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost.
 server_addr = "127.0.0.1:3002"

-## Etcd server address.
+## Store server address default to etcd store.
 store_addr = "127.0.0.1:2379"

 ## Datanode selector type.
@@ -32,6 +32,9 @@ store_key_prefix = ""
 ## - Using shared storage (e.g., s3).
 enable_region_failover = false

+## The datastore for meta server.
+backend = "EtcdStore"
+
 ## The runtime options.
 [runtime]
 ## The number of threads to execute the runtime for global read operations.
@@ -96,7 +99,12 @@ provider = "raft_engine"
 ## The broker endpoints of the Kafka cluster.
 broker_endpoints = ["127.0.0.1:9092"]

-## Number of topics to be created upon start.
+## Automatically create topics for WAL.
+## Set to `true` to automatically create topics for WAL.
+## Otherwise, use topics named `topic_name_prefix_[0..num_topics)`
+auto_create_topics = true
+
+## Number of topics.
 num_topics = 64

 ## Topic selector type.
@@ -105,6 +113,7 @@ num_topics = 64
 selector_type = "round_robin"

 ## A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
+## i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1.
 topic_name_prefix = "greptimedb_wal_topic"

 ## Expected number of replicas of each partition.
@@ -177,12 +186,13 @@ enable = false
 write_interval = "30s"

 ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
+## You must create the database before enabling it.
 [export_metrics.self_import]
 ## +toml2docs:none-default
-db = "information_schema"
+db = "greptime_metrics"

 [export_metrics.remote_write]
-## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`.
+## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
 url = ""

 ## HTTP headers of Prometheus remote-write carry.
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -171,7 +171,12 @@ sync_period = "10s"
 ## **It's only used when the provider is `kafka`**.
 broker_endpoints = ["127.0.0.1:9092"]

-## Number of topics to be created upon start.
+## Automatically create topics for WAL.
+## Set to `true` to automatically create topics for WAL.
+## Otherwise, use topics named `topic_name_prefix_[0..num_topics)`
+auto_create_topics = true
+
+## Number of topics.
 ## **It's only used when the provider is `kafka`**.
 num_topics = 64

@@ -182,6 +187,7 @@ num_topics = 64
 selector_type = "round_robin"

 ## A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
+## i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1.
 ## **It's only used when the provider is `kafka`**.
 topic_name_prefix = "greptimedb_wal_topic"

@@ -431,9 +437,13 @@ sst_meta_cache_size = "128MB"
 vector_cache_size = "512MB"

 ## Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
-## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
+## If not set, it's default to 1/8 of OS memory.
 page_cache_size = "512MB"

+## Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.
+## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
+selector_result_cache_size = "512MB"
+
 ## Whether to enable the experimental write cache.
 enable_experimental_write_cache = false

@@ -444,7 +454,8 @@ experimental_write_cache_path = ""
 experimental_write_cache_size = "512MB"

 ## TTL for write cache.
-experimental_write_cache_ttl = "1h"
+## +toml2docs:none-default
+experimental_write_cache_ttl = "8h"

 ## Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
@@ -590,12 +601,13 @@ enable = false
 write_interval = "30s"

 ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
+## You must create the database before enabling it.
 [export_metrics.self_import]
 ## +toml2docs:none-default
-db = "information_schema"
+db = "greptime_metrics"

 [export_metrics.remote_write]
-## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`.
+## The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`.
 url = ""

 ## HTTP headers of Prometheus remote-write carry.
--- a/docs/benchmarks/log/README.md
+++ b/docs/benchmarks/log/README.md
@@ -0,0 +1,51 @@
+# Log benchmark configuration
+This repo holds the configuration we used to benchmark GreptimeDB, Clickhouse and Elastic Search.
+
+Here are the versions of databases we used in the benchmark
+
+| name          | version    |
+| :------------ | :--------- |
+| GreptimeDB    | v0.9.2     |
+| Clickhouse    | 24.9.1.219 |
+| Elasticsearch | 8.15.0     |
+
+## Structured model vs Unstructured model
+We divide test into two parts, using structured model and unstructured model accordingly. You can also see the difference in create table clause.
+
+__Structured model__
+
+The log data is pre-processed into columns by vector. For example an insert request looks like following
+```SQL
+INSERT INTO test_table (bytes, http_version, ip, method, path, status, user, timestamp) VALUES ()
+```
+The goal is to test string/text support for each database. In real scenarios it means the datasource(or log data producers) have separate fields defined, or have already processed the raw input.
+
+__Unstructured model__
+
+The log data is inserted as a long string, and then we build fulltext index upon these strings. For example an insert request looks like following
+```SQL
+INSERT INTO test_table (message, timestamp) VALUES ()
+```
+The goal is to test fuzzy search performance for each database. In real scenarios it means the log is produced by some kind of middleware and inserted directly into the database.
+
+## Creating tables
+See [here](./create_table.sql) for GreptimeDB and Clickhouse's create table clause.
+The mapping of Elastic search is created automatically.
+
+## Vector Configuration
+We use vector to generate random log data and send inserts to databases.
+Please refer to [structured config](./structured_vector.toml) and [unstructured config](./unstructured_vector.toml) for detailed configuration.
+
+## SQLs and payloads
+Please refer to [SQL query](./query.sql) for GreptimeDB and Clickhouse, and [query payload](./query.md) for Elastic search.
+
+## Steps to reproduce
+0. Decide whether to run structured model test or unstructured mode test.
+1. Build vector binary(see vector's config file for specific branch) and databases binaries accordingly.
+2. Create table in GreptimeDB and Clickhouse in advance.
+3. Run vector to insert data.
+4. When data insertion is finished, run queries against each database. Note: you'll need to update timerange value after data insertion.
+
+## Addition
+- You can tune GreptimeDB's configuration to get better performance.
+- You can setup GreptimeDB to use S3 as storage, see [here](https://docs.greptime.com/user-guide/operations/configuration/#storage-options).
--- a/docs/benchmarks/log/create_table.sql
+++ b/docs/benchmarks/log/create_table.sql
@@ -0,0 +1,56 @@
+-- GreptimeDB create table clause
+-- structured test, use vector to pre-process log data into fields
+CREATE TABLE IF NOT EXISTS `test_table` (
+    `bytes` Int64 NULL,
+    `http_version` STRING NULL,
+    `ip` STRING NULL,
+    `method` STRING NULL,
+    `path` STRING NULL,
+    `status` SMALLINT UNSIGNED NULL,
+    `user` STRING NULL,
+    `timestamp` TIMESTAMP(3) NOT NULL,
+    PRIMARY KEY (`user`, `path`, `status`),
+    TIME INDEX (`timestamp`)
+)
+ENGINE=mito
+WITH(
+    append_mode = 'true'
+);
+
+-- unstructured test, build fulltext index on message column
+CREATE TABLE IF NOT EXISTS `test_table` (
+    `message` STRING NULL FULLTEXT WITH(analyzer = 'English', case_sensitive = 'false'),
+    `timestamp` TIMESTAMP(3) NOT NULL,
+    TIME INDEX (`timestamp`)
+)
+ENGINE=mito
+WITH(
+    append_mode = 'true'
+);
+
+-- Clickhouse create table clause
+-- structured test
+CREATE TABLE IF NOT EXISTS test_table
+(
+    bytes UInt64 NOT NULL,
+    http_version String NOT NULL,
+    ip String NOT NULL,
+    method String NOT NULL,
+    path String NOT NULL,
+    status UInt8 NOT NULL,
+    user String NOT NULL,
+    timestamp String NOT NULL,
+)
+ENGINE = MergeTree()
+ORDER BY (user, path, status);
+
+-- unstructured test
+SET allow_experimental_full_text_index = true;
+CREATE TABLE IF NOT EXISTS test_table
+(
+    message String,
+    timestamp String,
+    INDEX inv_idx(message) TYPE full_text(0) GRANULARITY 1
+)
+ENGINE = MergeTree()
+ORDER BY tuple();
--- a/docs/benchmarks/log/query.md
+++ b/docs/benchmarks/log/query.md
@@ -0,0 +1,199 @@
+# Query URL and payload for Elastic Search
+## Count
+URL: `http://127.0.0.1:9200/_count`
+
+## Query by timerange
+URL: `http://127.0.0.1:9200/_search`
+
+You can use the following payload to get the full timerange first.
+```JSON
+{"size":0,"aggs":{"max_timestamp":{"max":{"field":"timestamp"}},"min_timestamp":{"min":{"field":"timestamp"}}}}
+```
+
+And then use this payload to query by timerange.
+```JSON
+{
+  "from": 0,
+  "size": 1000,
+  "query": {
+    "range": {
+      "timestamp": {
+        "gte": "2024-08-16T04:30:44.000Z",
+        "lte": "2024-08-16T04:51:52.000Z"
+      }
+    }
+  }
+}
+```
+
+## Query by condition
+URL: `http://127.0.0.1:9200/_search`
+### Structured payload
+```JSON
+{
+  "from": 0,
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": [
+        {
+          "term": {
+            "user.keyword": "CrucifiX"
+          }
+        },
+        {
+          "term": {
+            "method.keyword": "OPTION"
+          }
+        },
+        {
+          "term": {
+            "path.keyword": "/user/booperbot124"
+          }
+        },
+        {
+          "term": {
+            "http_version.keyword": "HTTP/1.1"
+          }
+        },
+        {
+          "term": {
+            "status": "401"
+          }
+        }
+      ]
+    }
+  }
+}
+```
+### Unstructured payload
+```JSON
+{
+  "from": 0,
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": [
+        {
+          "match_phrase": {
+            "message": "CrucifiX"
+          }
+        },
+        {
+          "match_phrase": {
+            "message": "OPTION"
+          }
+        },
+        {
+          "match_phrase": {
+            "message": "/user/booperbot124"
+          }
+        },
+        {
+          "match_phrase": {
+            "message": "HTTP/1.1"
+          }
+        },
+        {
+          "match_phrase": {
+            "message": "401"
+          }
+        }
+      ]
+    }
+  }
+}
+```
+
+## Query by condition and timerange
+URL: `http://127.0.0.1:9200/_search`
+### Structured payload
+```JSON
+{
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": [
+        {
+          "term": {
+            "user.keyword": "CrucifiX"
+          }
+        },
+        {
+          "term": {
+            "method.keyword": "OPTION"
+          }
+        },
+        {
+          "term": {
+            "path.keyword": "/user/booperbot124"
+          }
+        },
+        {
+          "term": {
+            "http_version.keyword": "HTTP/1.1"
+          }
+        },
+        {
+          "term": {
+            "status": "401"
+          }
+        },
+        {
+          "range": {
+            "timestamp": {
+              "gte": "2024-08-19T07:03:37.383Z",
+              "lte": "2024-08-19T07:24:58.883Z"
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+```
+### Unstructured payload
+```JSON
+{
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": [
+        {
+          "match_phrase": {
+            "message": "CrucifiX"
+          }
+        },
+        {
+          "match_phrase": {
+            "message": "OPTION"
+          }
+        },
+        {
+          "match_phrase": {
+            "message": "/user/booperbot124"
+          }
+        },
+        {
+          "match_phrase": {
+            "message": "HTTP/1.1"
+          }
+        },
+        {
+          "match_phrase": {
+            "message": "401"
+          }
+        },
+        {
+          "range": {
+            "timestamp": {
+              "gte": "2024-08-19T05:16:17.099Z",
+              "lte": "2024-08-19T05:46:02.722Z"
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+```
--- a/docs/benchmarks/log/query.sql
+++ b/docs/benchmarks/log/query.sql
@@ -0,0 +1,50 @@
+-- Structured query for GreptimeDB and Clickhouse
+
+-- query count
+select count(*) from test_table;
+
+-- query by timerange. Note: place the timestamp range in the where clause
+-- GreptimeDB
+-- you can use `select max(timestamp)::bigint from test_table;` and `select min(timestamp)::bigint from test_table;`
+-- to get the full timestamp range
+select * from test_table where timestamp between 1723710843619 and 1723711367588;
+-- Clickhouse
+-- you can use `select max(timestamp) from test_table;` and `select min(timestamp) from test_table;`
+-- to get the full timestamp range
+select * from test_table where timestamp between '2024-08-16T03:58:46Z' and '2024-08-16T04:03:50Z';
+
+-- query by condition
+SELECT * FROM test_table WHERE user = 'CrucifiX' and method = 'OPTION' and path = '/user/booperbot124' and http_version = 'HTTP/1.1' and status = 401;
+
+-- query by condition and timerange
+-- GreptimeDB
+SELECT * FROM test_table WHERE user = "CrucifiX" and method = "OPTION" and path = "/user/booperbot124" and http_version = "HTTP/1.1" and status = 401 
+and timestamp between 1723774396760 and 1723774788760;
+-- Clickhouse
+SELECT * FROM test_table WHERE user = 'CrucifiX' and method = 'OPTION' and path = '/user/booperbot124' and http_version = 'HTTP/1.1' and status = 401 
+and timestamp between '2024-08-16T03:58:46Z' and '2024-08-16T04:03:50Z';
+
+-- Unstructured query for GreptimeDB and Clickhouse
+
+
+-- query by condition
+-- GreptimeDB
+SELECT * FROM test_table WHERE MATCHES(message, "+CrucifiX +OPTION +/user/booperbot124 +HTTP/1.1 +401");
+-- Clickhouse
+SELECT * FROM test_table WHERE (message LIKE '%CrucifiX%') 
+AND (message LIKE '%OPTION%') 
+AND (message LIKE '%/user/booperbot124%') 
+AND (message LIKE '%HTTP/1.1%') 
+AND (message LIKE '%401%');
+
+-- query by condition and timerange
+-- GreptimeDB
+SELECT * FROM test_table WHERE MATCHES(message, "+CrucifiX +OPTION +/user/booperbot124 +HTTP/1.1 +401") 
+and timestamp between 1723710843619 and 1723711367588;
+-- Clickhouse
+SELECT * FROM test_table WHERE (message LIKE '%CrucifiX%') 
+AND (message LIKE '%OPTION%') 
+AND (message LIKE '%/user/booperbot124%') 
+AND (message LIKE '%HTTP/1.1%') 
+AND (message LIKE '%401%') 
+AND timestamp between '2024-08-15T10:25:26.524000000Z' AND '2024-08-15T10:31:31.746000000Z';
--- a/docs/benchmarks/log/structured_vector.toml
+++ b/docs/benchmarks/log/structured_vector.toml
@@ -0,0 +1,57 @@
+# Please note we use patched branch to build vector
+# https://github.com/shuiyisong/vector/tree/chore/greptime_log_ingester_logitem
+
+[sources.demo_logs]
+type = "demo_logs"
+format = "apache_common"
+# interval value = 1 / rps
+# say you want to insert at 20k/s, that is 1 / 20000 = 0.00005
+# set to 0 to run as fast as possible
+interval = 0
+# total rows to insert
+count = 100000000
+lines = [ "line1" ]
+
+[transforms.parse_logs]
+type = "remap"
+inputs = ["demo_logs"]
+source = '''
+. = parse_regex!(.message, r'^(?P<ip>\S+) - (?P<user>\S+) \[(?P<timestamp>[^\]]+)\] "(?P<method>\S+) (?P<path>\S+) (?P<http_version>\S+)" (?P<status>\d+) (?P<bytes>\d+)$')
+
+# Convert timestamp to a standard format
+.timestamp = parse_timestamp!(.timestamp, format: "%d/%b/%Y:%H:%M:%S %z")
+
+# Convert status and bytes to integers
+.status = to_int!(.status)
+.bytes = to_int!(.bytes)
+'''
+
+[sinks.sink_greptime_logs]
+type = "greptimedb_logs"
+# The table to insert into
+table = "test_table"
+pipeline_name = "demo_pipeline"
+compression = "none"
+inputs = [ "parse_logs" ]
+endpoint = "http://127.0.0.1:4000"
+# Batch size for each insertion
+batch.max_events = 4000
+
+[sinks.clickhouse]
+type = "clickhouse"
+inputs = [ "parse_logs" ]
+database = "default"
+endpoint = "http://127.0.0.1:8123"
+format = "json_each_row"
+# The table to insert into
+table = "test_table"
+
+[sinks.sink_elasticsearch]
+type = "elasticsearch"
+inputs = [ "parse_logs" ]
+api_version = "auto"
+compression = "none"
+doc_type = "_doc"
+endpoints = [ "http://127.0.0.1:9200" ]
+id_key = "id"
+mode = "bulk"
--- a/docs/benchmarks/log/unstructured_vector.toml
+++ b/docs/benchmarks/log/unstructured_vector.toml
@@ -0,0 +1,43 @@
+# Please note we use patched branch to build vector
+# https://github.com/shuiyisong/vector/tree/chore/greptime_log_ingester_ft
+
+[sources.demo_logs]
+type = "demo_logs"
+format = "apache_common"
+# interval value = 1 / rps
+# say you want to insert at 20k/s, that is 1 / 20000 = 0.00005
+# set to 0 to run as fast as possible
+interval = 0
+# total rows to insert
+count = 100000000
+lines = [ "line1" ]
+
+[sinks.sink_greptime_logs]
+type = "greptimedb_logs"
+# The table to insert into
+table = "test_table"
+pipeline_name = "demo_pipeline"
+compression = "none"
+inputs = [ "demo_logs" ]
+endpoint = "http://127.0.0.1:4000"
+# Batch size for each insertion
+batch.max_events = 500
+
+[sinks.clickhouse]
+type = "clickhouse"
+inputs = [ "demo_logs" ]
+database = "default"
+endpoint = "http://127.0.0.1:8123"
+format = "json_each_row"
+# The table to insert into
+table = "test_table"
+
+[sinks.sink_elasticsearch]
+type = "elasticsearch"
+inputs = [ "demo_logs" ]
+api_version = "auto"
+compression = "none"
+doc_type = "_doc"
+endpoints = [ "http://127.0.0.1:9200" ]
+id_key = "id"
+mode = "bulk"
--- a/docs/benchmarks/tsbs/v0.9.1.md
+++ b/docs/benchmarks/tsbs/v0.9.1.md
--- a/grafana/greptimedb-cluster.json
+++ b/grafana/greptimedb-cluster.json
@@ -157,26 +157,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -326,26 +306,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -497,26 +457,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -668,26 +608,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -852,26 +772,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -1023,26 +923,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -1194,26 +1074,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -1365,26 +1225,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -1536,26 +1376,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -1707,26 +1527,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -1878,26 +1678,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -2062,26 +1842,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -2233,26 +1993,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -2417,26 +2157,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -2569,26 +2289,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -2751,26 +2451,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -3091,28 +2771,7 @@
          },
          "unit": "s"
        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
+        "overrides": []
      },
      "gridPos": {
        "h": 8,
@@ -3242,26 +2901,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -3429,26 +3068,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -3598,26 +3217,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -3740,28 +3339,7 @@
          },
          "unit": "s"
        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
+        "overrides": []
      },
      "gridPos": {
        "h": 8,
@@ -4101,26 +3679,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -4270,26 +3828,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -4439,26 +3977,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -4608,26 +4126,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -4777,26 +4275,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -4946,26 +4424,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -5115,26 +4573,6 @@
                }
              }
            ]
-          },
-          {
-            "matcher": {
-              "id": "byValue",
-              "options": {
-                "op": "gte",
-                "reducer": "allIsZero",
-                "value": 0
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": true,
-                  "tooltip": true,
-                  "viz": true
-                }
-              }
-            ]
          }
        ]
      },
@@ -5241,4 +4679,4 @@
  "uid": "ea35efe5-918e-44fa-9743-e9aa1a340a3f",
  "version": 11,
  "weekStart": ""
-}
+}
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,2 +1,3 @@
 [toolchain]
-channel = "nightly-2024-04-20"
+channel = "nightly-2024-06-06"
+
--- a/scripts/check-builder-rust-version.sh
+++ b/scripts/check-builder-rust-version.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -e
+
+RUST_TOOLCHAIN_VERSION_FILE="rust-toolchain.toml"
+DEV_BUILDER_UBUNTU_REGISTRY="docker.io"
+DEV_BUILDER_UBUNTU_NAMESPACE="greptime"
+DEV_BUILDER_UBUNTU_NAME="dev-builder-ubuntu"
+
+function check_rust_toolchain_version() {
+  DEV_BUILDER_IMAGE_TAG=$(grep "DEV_BUILDER_IMAGE_TAG ?= " Makefile | cut -d= -f2 | sed 's/^[ \t]*//')
+  if [ -z "$DEV_BUILDER_IMAGE_TAG" ]; then
+    echo "Error: No DEV_BUILDER_IMAGE_TAG found in Makefile"
+    exit 1
+  fi
+
+  DEV_BUILDER_UBUNTU_IMAGE="$DEV_BUILDER_UBUNTU_REGISTRY/$DEV_BUILDER_UBUNTU_NAMESPACE/$DEV_BUILDER_UBUNTU_NAME:$DEV_BUILDER_IMAGE_TAG"
+
+  CURRENT_VERSION=$(grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}' "$RUST_TOOLCHAIN_VERSION_FILE")
+  if [ -z "$CURRENT_VERSION" ]; then
+    echo "Error: No rust toolchain version found in $RUST_TOOLCHAIN_VERSION_FILE"
+    exit 1
+  fi
+
+  RUST_TOOLCHAIN_VERSION_IN_BUILDER=$(docker run "$DEV_BUILDER_UBUNTU_IMAGE" rustc --version | grep  -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}')
+  if [ -z "$RUST_TOOLCHAIN_VERSION_IN_BUILDER" ]; then
+    echo "Error: No rustc version found in $DEV_BUILDER_UBUNTU_IMAGE"
+    exit 1
+  fi
+
+  # Compare the version and the difference should be less than 1 day.
+  current_rust_toolchain_seconds=$(date -d "$CURRENT_VERSION" +%s)
+  rust_toolchain_in_dev_builder_ubuntu_seconds=$(date -d "$RUST_TOOLCHAIN_VERSION_IN_BUILDER" +%s)
+  date_diff=$(( (current_rust_toolchain_seconds - rust_toolchain_in_dev_builder_ubuntu_seconds) / 86400 ))
+
+  if [ $date_diff -gt 1 ]; then
+    echo "Error: The rust toolchain '$RUST_TOOLCHAIN_VERSION_IN_BUILDER' in builder '$DEV_BUILDER_UBUNTU_IMAGE' maybe outdated, please update it to '$CURRENT_VERSION'"
+    exit 1
+  fi
+}
+
+check_rust_toolchain_version
--- a/src/client/src/database.rs
+++ b/src/client/src/database.rs
@@ -91,7 +91,7 @@ impl Database {
    ///
    /// - the name of database when using GreptimeDB standalone or cluster
    /// - the name provided by GreptimeCloud or other multi-tenant GreptimeDB
-    /// environment
+    ///   environment
    pub fn new_with_dbname(dbname: impl Into<String>, client: Client) -> Self {
        Self {
            catalog: String::default(),
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -51,7 +51,7 @@ file-engine.workspace = true
 flow.workspace = true
 frontend = { workspace = true, default-features = false }
 futures.workspace = true
-human-panic = "1.2.2"
+human-panic = "2.0"
 lazy_static.workspace = true
 meta-client.workspace = true
 meta-srv.workspace = true
@@ -80,7 +80,7 @@ tonic.workspace = true
 tracing-appender = "0.2"

 [target.'cfg(not(windows))'.dependencies]
-tikv-jemallocator = "0.5"
+tikv-jemallocator = "0.6"

 [dev-dependencies]
 client = { workspace = true, features = ["testing"] }
--- a/src/cmd/src/bin/greptime.rs
+++ b/src/cmd/src/bin/greptime.rs
@@ -139,13 +139,10 @@ async fn start(cli: Command) -> Result<()> {
 }

 fn setup_human_panic() {
-    let metadata = human_panic::Metadata {
-        version: env!("CARGO_PKG_VERSION").into(),
-        name: "GreptimeDB".into(),
-        authors: Default::default(),
-        homepage: "https://github.com/GreptimeTeam/greptimedb/discussions".into(),
-    };
-    human_panic::setup_panic!(metadata);
+    human_panic::setup_panic!(
+        human_panic::Metadata::new("GreptimeDB", env!("CARGO_PKG_VERSION"))
+            .homepage("https://github.com/GreptimeTeam/greptimedb/discussions")
+    );

    common_telemetry::set_panic_hook();
 }
--- a/src/cmd/src/datanode.rs
+++ b/src/cmd/src/datanode.rs
@@ -267,7 +267,7 @@ impl StartCommand {
            &opts.component.tracing,
            opts.component.node_id.map(|x| x.to_string()),
        );
-        log_versions(version(), short_version());
+        log_versions(version(), short_version(), APP_NAME);

        info!("Datanode start command: {:#?}", self);
        info!("Datanode options: {:#?}", opts);
--- a/src/cmd/src/flownode.rs
+++ b/src/cmd/src/flownode.rs
@@ -215,7 +215,7 @@ impl StartCommand {
            &opts.component.tracing,
            opts.component.node_id.map(|x| x.to_string()),
        );
-        log_versions(version(), short_version());
+        log_versions(version(), short_version(), APP_NAME);

        info!("Flownode start command: {:#?}", self);
        info!("Flownode options: {:#?}", opts);
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -261,7 +261,7 @@ impl StartCommand {
            &opts.component.tracing,
            opts.component.node_id.clone(),
        );
-        log_versions(version(), short_version());
+        log_versions(version(), short_version(), APP_NAME);

        info!("Frontend start command: {:#?}", self);
        info!("Frontend options: {:#?}", opts);
--- a/src/cmd/src/lib.rs
+++ b/src/cmd/src/lib.rs
@@ -30,7 +30,7 @@ pub mod standalone;

 lazy_static::lazy_static! {
    static ref APP_VERSION: prometheus::IntGaugeVec =
-        prometheus::register_int_gauge_vec!("greptime_app_version", "app version", &["version", "short_version"]).unwrap();
+        prometheus::register_int_gauge_vec!("greptime_app_version", "app version", &["version", "short_version", "app"]).unwrap();
 }

 #[async_trait]
@@ -76,10 +76,10 @@ pub trait App: Send {
 /// Log the versions of the application, and the arguments passed to the cli.
 /// `version` should be the same as the output of cli "--version";
 /// and the `short_version` is the short version of the codes, often consist of git branch and commit.
-pub fn log_versions(version: &str, short_version: &str) {
+pub fn log_versions(version: &str, short_version: &str, app: &str) {
    // Report app version as gauge.
    APP_VERSION
-        .with_label_values(&[env!("CARGO_PKG_VERSION"), short_version])
+        .with_label_values(&[env!("CARGO_PKG_VERSION"), short_version, app])
        .inc();

    // Log version and argument flags.
--- a/src/cmd/src/metasrv.rs
+++ b/src/cmd/src/metasrv.rs
@@ -244,7 +244,7 @@ impl StartCommand {
            &opts.component.tracing,
            None,
        );
-        log_versions(version(), short_version());
+        log_versions(version(), short_version(), APP_NAME);

        info!("Metasrv start command: {:#?}", self);
        info!("Metasrv options: {:#?}", opts);
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -178,6 +178,16 @@ impl Configurable for StandaloneOptions {
    }
 }

+/// The [`StandaloneOptions`] is only defined in cmd crate,
+/// we don't want to make `frontend` depends on it, so impl [`Into`]
+/// rather than [`From`].
+#[allow(clippy::from_over_into)]
+impl Into<FrontendOptions> for StandaloneOptions {
+    fn into(self) -> FrontendOptions {
+        self.frontend_options()
+    }
+}
+
 impl StandaloneOptions {
    pub fn frontend_options(&self) -> FrontendOptions {
        let cloned_opts = self.clone();
@@ -415,7 +425,7 @@ impl StartCommand {
            &opts.component.tracing,
            None,
        );
-        log_versions(version(), short_version());
+        log_versions(version(), short_version(), APP_NAME);

        info!("Standalone start command: {:#?}", self);
        info!("Standalone options: {opts:#?}");
@@ -510,7 +520,7 @@ impl StartCommand {
                .build(),
        );
        let wal_options_allocator = Arc::new(WalOptionsAllocator::new(
-            opts.wal.into(),
+            opts.wal.clone().into(),
            kv_backend.clone(),
        ));
        let table_meta_allocator = Arc::new(TableMetadataAllocator::new(
@@ -533,7 +543,7 @@ impl StartCommand {
        .await?;

        let mut frontend = FrontendBuilder::new(
-            fe_opts.clone(),
+            fe_opts,
            kv_backend.clone(),
            layered_cache_registry.clone(),
            catalog_manager.clone(),
@@ -561,7 +571,7 @@ impl StartCommand {

        let (tx, _rx) = broadcast::channel(1);

-        let servers = Services::new(fe_opts, Arc::new(frontend.clone()), plugins)
+        let servers = Services::new(opts, Arc::new(frontend.clone()), plugins)
            .build()
            .await
            .context(StartFrontendSnafu)?;
--- a/src/cmd/tests/load_config_test.rs
+++ b/src/cmd/tests/load_config_test.rs
@@ -82,6 +82,7 @@ fn test_load_datanode_example_config() {
                    vector_cache_size: ReadableSize::mb(512),
                    page_cache_size: ReadableSize::mb(512),
                    max_background_jobs: 4,
+                    experimental_write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                    ..Default::default()
                }),
                RegionEngineConfig::File(EngineConfig {}),
@@ -217,7 +218,9 @@ fn test_load_standalone_example_config() {
                    sst_meta_cache_size: ReadableSize::mb(128),
                    vector_cache_size: ReadableSize::mb(512),
                    page_cache_size: ReadableSize::mb(512),
+                    selector_result_cache_size: ReadableSize::mb(512),
                    max_background_jobs: 4,
+                    experimental_write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                    ..Default::default()
                }),
                RegionEngineConfig::File(EngineConfig {}),
--- a/src/common/catalog/src/lib.rs
+++ b/src/common/catalog/src/lib.rs
@@ -48,19 +48,19 @@ pub fn build_db_string(catalog: &str, schema: &str) -> String {
 /// The database name may come from different sources:
 ///
 /// - MySQL `schema` name in MySQL protocol login request: it's optional and user
-/// and switch database using `USE` command
+///   and switch database using `USE` command
 /// - Postgres `database` parameter in Postgres wire protocol, required
 /// - HTTP RESTful API: the database parameter, optional
 /// - gRPC: the dbname field in header, optional but has a higher priority than
-/// original catalog/schema
+///   original catalog/schema
 ///
 /// When database name is provided, we attempt to parse catalog and schema from
 /// it. We assume the format `[<catalog>-]<schema>`:
 ///
 /// - If `[<catalog>-]` part is not provided, we use whole database name as
-/// schema name
+///   schema name
 /// - if `[<catalog>-]` is provided, we split database name with `-` and use
-/// `<catalog>` and `<schema>`.
+///   `<catalog>` and `<schema>`.
 pub fn parse_catalog_and_schema_from_db_string(db: &str) -> (String, String) {
    match parse_optional_catalog_and_schema_from_db_string(db) {
        (Some(catalog), schema) => (catalog, schema),
--- a/src/common/datasource/src/object_store/fs.rs
+++ b/src/common/datasource/src/object_store/fs.rs
@@ -13,6 +13,7 @@
 // limitations under the License.

 use object_store::services::Fs;
+use object_store::util::DefaultLoggingInterceptor;
 use object_store::ObjectStore;
 use snafu::ResultExt;

@@ -22,13 +23,9 @@ pub fn build_fs_backend(root: &str) -> Result<ObjectStore> {
    let builder = Fs::default();
    let object_store = ObjectStore::new(builder.root(root))
        .context(BuildBackendSnafu)?
-        .layer(
-            object_store::layers::LoggingLayer::default()
-                // Print the expected error only in DEBUG level.
-                // See https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html#method.with_error_level
-                .with_error_level(Some("debug"))
-                .expect("input error level must be valid"),
-        )
+        .layer(object_store::layers::LoggingLayer::new(
+            DefaultLoggingInterceptor,
+        ))
        .layer(object_store::layers::TracingLayer)
        .layer(object_store::layers::PrometheusMetricsLayer::new(true))
        .finish();
--- a/src/common/datasource/src/object_store/s3.rs
+++ b/src/common/datasource/src/object_store/s3.rs
@@ -15,6 +15,7 @@
 use std::collections::HashMap;

 use object_store::services::S3;
+use object_store::util::DefaultLoggingInterceptor;
 use object_store::ObjectStore;
 use snafu::ResultExt;

@@ -84,13 +85,9 @@ pub fn build_s3_backend(
    // TODO(weny): Consider finding a better way to eliminate duplicate code.
    Ok(ObjectStore::new(builder)
        .context(error::BuildBackendSnafu)?
-        .layer(
-            object_store::layers::LoggingLayer::default()
-                // Print the expected error only in DEBUG level.
-                // See https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html#method.with_error_level
-                .with_error_level(Some("debug"))
-                .expect("input error level must be valid"),
-        )
+        .layer(object_store::layers::LoggingLayer::new(
+            DefaultLoggingInterceptor,
+        ))
        .layer(object_store::layers::TracingLayer)
        .layer(object_store::layers::PrometheusMetricsLayer::new(true))
        .finish())
--- a/src/common/function/Cargo.toml
+++ b/src/common/function/Cargo.toml
@@ -7,6 +7,10 @@ license.workspace = true
 [lints]
 workspace = true

+[features]
+default = ["geo"]
+geo = ["geohash", "h3o"]
+
 [dependencies]
 api.workspace = true
 arc-swap = "1.0"
@@ -23,6 +27,8 @@ common-time.workspace = true
 common-version.workspace = true
 datafusion.workspace = true
 datatypes.workspace = true
+geohash = { version = "0.13", optional = true }
+h3o = { version = "0.6", optional = true }
 num = "0.4"
 num-traits = "0.2"
 once_cell.workspace = true
@@ -39,3 +45,4 @@ table.workspace = true
 [dev-dependencies]
 ron = "0.7"
 serde = { version = "1.0", features = ["derive"] }
+tokio.workspace = true
--- a/src/common/function/src/flush_flow.rs
+++ b/src/common/function/src/flush_flow.rs
@@ -110,7 +110,7 @@ mod test {
    use session::context::QueryContext;

    use super::*;
-    use crate::function::{Function, FunctionContext};
+    use crate::function::{AsyncFunction, FunctionContext};

    #[test]
    fn test_flush_flow_metadata() {
@@ -130,8 +130,8 @@ mod test {
        );
    }

-    #[test]
-    fn test_missing_flow_service() {
+    #[tokio::test]
+    async fn test_missing_flow_service() {
        let f = FlushFlowFunction;

        let args = vec!["flow_name"];
@@ -140,7 +140,7 @@ mod test {
            .map(|arg| Arc::new(StringVector::from_slice(&[arg])) as _)
            .collect::<Vec<_>>();

-        let result = f.eval(FunctionContext::default(), &args).unwrap_err();
+        let result = f.eval(FunctionContext::default(), &args).await.unwrap_err();
        assert_eq!(
            "Missing FlowServiceHandler, not expected",
            result.to_string()
--- a/src/common/function/src/function.rs
+++ b/src/common/function/src/function.rs
@@ -32,7 +32,7 @@ pub struct FunctionContext {

 impl FunctionContext {
    /// Create a mock [`FunctionContext`] for test.
-    #[cfg(any(test, feature = "testing"))]
+    #[cfg(test)]
    pub fn mock() -> Self {
        Self {
            query_ctx: QueryContextBuilder::default().build().into(),
@@ -56,8 +56,10 @@ pub trait Function: fmt::Display + Sync + Send {
    /// Returns the name of the function, should be unique.
    fn name(&self) -> &str;

+    /// The returned data type of function execution.
    fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType>;

+    /// The signature of function.
    fn signature(&self) -> Signature;

    /// Evaluate the function, e.g. run/execute the function.
@@ -65,3 +67,22 @@ pub trait Function: fmt::Display + Sync + Send {
 }

 pub type FunctionRef = Arc<dyn Function>;
+
+/// Async Scalar function trait
+#[async_trait::async_trait]
+pub trait AsyncFunction: fmt::Display + Sync + Send {
+    /// Returns the name of the function, should be unique.
+    fn name(&self) -> &str;
+
+    /// The returned data type of function execution.
+    fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType>;
+
+    /// The signature of function.
+    fn signature(&self) -> Signature;
+
+    /// Evaluate the function, e.g. run/execute the function.
+    /// TODO(dennis): simplify the signature and refactor all the admin functions.
+    async fn eval(&self, _func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef>;
+}
+
+pub type AsyncFunctionRef = Arc<dyn AsyncFunction>;
--- a/src/common/function/src/function_registry.rs
+++ b/src/common/function/src/function_registry.rs
@@ -18,7 +18,7 @@ use std::sync::{Arc, RwLock};

 use once_cell::sync::Lazy;

-use crate::function::FunctionRef;
+use crate::function::{AsyncFunctionRef, FunctionRef};
 use crate::scalars::aggregate::{AggregateFunctionMetaRef, AggregateFunctions};
 use crate::scalars::date::DateFunction;
 use crate::scalars::expression::ExpressionFunction;
@@ -32,6 +32,7 @@ use crate::table::TableFunction;
 #[derive(Default)]
 pub struct FunctionRegistry {
    functions: RwLock<HashMap<String, FunctionRef>>,
+    async_functions: RwLock<HashMap<String, AsyncFunctionRef>>,
    aggregate_functions: RwLock<HashMap<String, AggregateFunctionMetaRef>>,
 }

@@ -44,6 +45,27 @@ impl FunctionRegistry {
            .insert(func.name().to_string(), func);
    }

+    pub fn register_async(&self, func: AsyncFunctionRef) {
+        let _ = self
+            .async_functions
+            .write()
+            .unwrap()
+            .insert(func.name().to_string(), func);
+    }
+
+    pub fn get_async_function(&self, name: &str) -> Option<AsyncFunctionRef> {
+        self.async_functions.read().unwrap().get(name).cloned()
+    }
+
+    pub fn async_functions(&self) -> Vec<AsyncFunctionRef> {
+        self.async_functions
+            .read()
+            .unwrap()
+            .values()
+            .cloned()
+            .collect()
+    }
+
    pub fn register_aggregate_function(&self, func: AggregateFunctionMetaRef) {
        let _ = self
            .aggregate_functions
@@ -94,6 +116,10 @@ pub static FUNCTION_REGISTRY: Lazy<Arc<FunctionRegistry>> = Lazy::new(|| {
    SystemFunction::register(&function_registry);
    TableFunction::register(&function_registry);

+    // Geo functions
+    #[cfg(feature = "geo")]
+    crate::scalars::geo::GeoFunctions::register(&function_registry);
+
    Arc::new(function_registry)
 });

--- a/src/common/function/src/scalars.rs
+++ b/src/common/function/src/scalars.rs
@@ -15,6 +15,8 @@
 pub mod aggregate;
 pub(crate) mod date;
 pub mod expression;
+#[cfg(feature = "geo")]
+pub mod geo;
 pub mod matches;
 pub mod math;
 pub mod numpy;
--- a/src/common/function/src/scalars/aggregate/percentile.rs
+++ b/src/common/function/src/scalars/aggregate/percentile.rs
@@ -75,7 +75,7 @@ where
        // to keep the not_greater length == floor+1
        // so to ensure the peek of the not_greater is array[floor]
        // and the peek of the greater is array[floor+1]
-        let p = if let Some(p) = self.p { p } else { 0.0_f64 };
+        let p = self.p.unwrap_or(0.0_f64);
        let floor = (((self.n - 1) as f64) * p / (100_f64)).floor();
        if value <= *self.not_greater.peek().unwrap() {
            self.not_greater.push(value);
--- a/src/common/function/src/scalars/geo.rs
+++ b/src/common/function/src/scalars/geo.rs
@@ -0,0 +1,31 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+mod geohash;
+mod h3;
+
+use geohash::GeohashFunction;
+use h3::H3Function;
+
+use crate::function_registry::FunctionRegistry;
+
+pub(crate) struct GeoFunctions;
+
+impl GeoFunctions {
+    pub fn register(registry: &FunctionRegistry) {
+        registry.register(Arc::new(GeohashFunction));
+        registry.register(Arc::new(H3Function));
+    }
+}
--- a/src/common/function/src/scalars/geo/geohash.rs
+++ b/src/common/function/src/scalars/geo/geohash.rs
@@ -0,0 +1,135 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt;
+
+use common_error::ext::{BoxedError, PlainError};
+use common_error::status_code::StatusCode;
+use common_query::error::{self, InvalidFuncArgsSnafu, Result};
+use common_query::prelude::{Signature, TypeSignature};
+use datafusion::logical_expr::Volatility;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::value::Value;
+use datatypes::vectors::{MutableVector, StringVectorBuilder, VectorRef};
+use geohash::Coord;
+use snafu::{ensure, ResultExt};
+
+use crate::function::{Function, FunctionContext};
+
+/// Function that return geohash string for a given geospatial coordinate.
+#[derive(Clone, Debug, Default)]
+pub struct GeohashFunction;
+
+const NAME: &str = "geohash";
+
+impl Function for GeohashFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        let mut signatures = Vec::new();
+        for coord_type in &[
+            ConcreteDataType::float32_datatype(),
+            ConcreteDataType::float64_datatype(),
+        ] {
+            for resolution_type in &[
+                ConcreteDataType::int8_datatype(),
+                ConcreteDataType::int16_datatype(),
+                ConcreteDataType::int32_datatype(),
+                ConcreteDataType::int64_datatype(),
+                ConcreteDataType::uint8_datatype(),
+                ConcreteDataType::uint16_datatype(),
+                ConcreteDataType::uint32_datatype(),
+                ConcreteDataType::uint64_datatype(),
+            ] {
+                signatures.push(TypeSignature::Exact(vec![
+                    // latitude
+                    coord_type.clone(),
+                    // longitude
+                    coord_type.clone(),
+                    // resolution
+                    resolution_type.clone(),
+                ]));
+            }
+        }
+        Signature::one_of(signatures, Volatility::Stable)
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 3,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 3, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let lat_vec = &columns[0];
+        let lon_vec = &columns[1];
+        let resolution_vec = &columns[2];
+
+        let size = lat_vec.len();
+        let mut results = StringVectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let lat = lat_vec.get(i).as_f64_lossy();
+            let lon = lon_vec.get(i).as_f64_lossy();
+            let r = match resolution_vec.get(i) {
+                Value::Int8(v) => v as usize,
+                Value::Int16(v) => v as usize,
+                Value::Int32(v) => v as usize,
+                Value::Int64(v) => v as usize,
+                Value::UInt8(v) => v as usize,
+                Value::UInt16(v) => v as usize,
+                Value::UInt32(v) => v as usize,
+                Value::UInt64(v) => v as usize,
+                _ => unreachable!(),
+            };
+
+            let result = match (lat, lon) {
+                (Some(lat), Some(lon)) => {
+                    let coord = Coord { x: lon, y: lat };
+                    let encoded = geohash::encode(coord, r)
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("Geohash error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?;
+                    Some(encoded)
+                }
+                _ => None,
+            };
+
+            results.push(result.as_deref());
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+impl fmt::Display for GeohashFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", NAME)
+    }
+}
--- a/src/common/function/src/scalars/geo/h3.rs
+++ b/src/common/function/src/scalars/geo/h3.rs
@@ -0,0 +1,143 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt;
+
+use common_error::ext::{BoxedError, PlainError};
+use common_error::status_code::StatusCode;
+use common_query::error::{self, InvalidFuncArgsSnafu, Result};
+use common_query::prelude::{Signature, TypeSignature};
+use datafusion::logical_expr::Volatility;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::value::Value;
+use datatypes::vectors::{MutableVector, StringVectorBuilder, VectorRef};
+use h3o::{LatLng, Resolution};
+use snafu::{ensure, ResultExt};
+
+use crate::function::{Function, FunctionContext};
+
+/// Function that return h3 encoding string for a given geospatial coordinate.
+#[derive(Clone, Debug, Default)]
+pub struct H3Function;
+
+const NAME: &str = "h3";
+
+impl Function for H3Function {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        let mut signatures = Vec::new();
+        for coord_type in &[
+            ConcreteDataType::float32_datatype(),
+            ConcreteDataType::float64_datatype(),
+        ] {
+            for resolution_type in &[
+                ConcreteDataType::int8_datatype(),
+                ConcreteDataType::int16_datatype(),
+                ConcreteDataType::int32_datatype(),
+                ConcreteDataType::int64_datatype(),
+                ConcreteDataType::uint8_datatype(),
+                ConcreteDataType::uint16_datatype(),
+                ConcreteDataType::uint32_datatype(),
+                ConcreteDataType::uint64_datatype(),
+            ] {
+                signatures.push(TypeSignature::Exact(vec![
+                    // latitude
+                    coord_type.clone(),
+                    // longitude
+                    coord_type.clone(),
+                    // resolution
+                    resolution_type.clone(),
+                ]));
+            }
+        }
+        Signature::one_of(signatures, Volatility::Stable)
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 3,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 3, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let lat_vec = &columns[0];
+        let lon_vec = &columns[1];
+        let resolution_vec = &columns[2];
+
+        let size = lat_vec.len();
+        let mut results = StringVectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let lat = lat_vec.get(i).as_f64_lossy();
+            let lon = lon_vec.get(i).as_f64_lossy();
+            let r = match resolution_vec.get(i) {
+                Value::Int8(v) => v as u8,
+                Value::Int16(v) => v as u8,
+                Value::Int32(v) => v as u8,
+                Value::Int64(v) => v as u8,
+                Value::UInt8(v) => v,
+                Value::UInt16(v) => v as u8,
+                Value::UInt32(v) => v as u8,
+                Value::UInt64(v) => v as u8,
+                _ => unreachable!(),
+            };
+
+            let result = match (lat, lon) {
+                (Some(lat), Some(lon)) => {
+                    let coord = LatLng::new(lat, lon)
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("H3 error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?;
+                    let r = Resolution::try_from(r as u8)
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("H3 error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?;
+                    let encoded = coord.to_cell(r).to_string();
+                    Some(encoded)
+                }
+                _ => None,
+            };
+
+            results.push(result.as_deref());
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+impl fmt::Display for H3Function {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", NAME)
+    }
+}
--- a/src/common/function/src/state.rs
+++ b/src/common/function/src/state.rs
@@ -28,7 +28,7 @@ pub struct FunctionState {

 impl FunctionState {
    /// Create a mock [`FunctionState`] for test.
-    #[cfg(any(test, feature = "testing"))]
+    #[cfg(test)]
    pub fn mock() -> Self {
        use std::sync::Arc;

--- a/src/common/function/src/system.rs
+++ b/src/common/function/src/system.rs
@@ -22,7 +22,7 @@ mod version;
 use std::sync::Arc;

 use build::BuildFunction;
-use database::DatabaseFunction;
+use database::{CurrentSchemaFunction, DatabaseFunction};
 use pg_catalog::PGCatalogFunction;
 use procedure_state::ProcedureStateFunction;
 use timezone::TimezoneFunction;
@@ -37,8 +37,9 @@ impl SystemFunction {
        registry.register(Arc::new(BuildFunction));
        registry.register(Arc::new(VersionFunction));
        registry.register(Arc::new(DatabaseFunction));
+        registry.register(Arc::new(CurrentSchemaFunction));
        registry.register(Arc::new(TimezoneFunction));
-        registry.register(Arc::new(ProcedureStateFunction));
+        registry.register_async(Arc::new(ProcedureStateFunction));
        PGCatalogFunction::register(registry);
    }
 }
--- a/src/common/function/src/system/database.rs
+++ b/src/common/function/src/system/database.rs
@@ -26,11 +26,35 @@ use crate::function::{Function, FunctionContext};
 #[derive(Clone, Debug, Default)]
 pub struct DatabaseFunction;

-const NAME: &str = "database";
+#[derive(Clone, Debug, Default)]
+pub struct CurrentSchemaFunction;
+
+const DATABASE_FUNCTION_NAME: &str = "database";
+const CURRENT_SCHEMA_FUNCTION_NAME: &str = "current_schema";

 impl Function for DatabaseFunction {
    fn name(&self) -> &str {
-        NAME
+        DATABASE_FUNCTION_NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::uniform(0, vec![], Volatility::Immutable)
+    }
+
+    fn eval(&self, func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
+        let db = func_ctx.query_ctx.current_schema();
+
+        Ok(Arc::new(StringVector::from_slice(&[&db])) as _)
+    }
+}
+
+impl Function for CurrentSchemaFunction {
+    fn name(&self) -> &str {
+        CURRENT_SCHEMA_FUNCTION_NAME
    }

    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
@@ -54,6 +78,12 @@ impl fmt::Display for DatabaseFunction {
    }
 }

+impl fmt::Display for CurrentSchemaFunction {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "CURRENT_SCHEMA")
+    }
+}
+
 #[cfg(test)]
 mod tests {
    use std::sync::Arc;
--- a/src/common/function/src/system/pg_catalog.rs
+++ b/src/common/function/src/system/pg_catalog.rs
@@ -14,11 +14,13 @@

 mod pg_get_userbyid;
 mod table_is_visible;
+mod version;

 use std::sync::Arc;

 use pg_get_userbyid::PGGetUserByIdFunction;
 use table_is_visible::PGTableIsVisibleFunction;
+use version::PGVersionFunction;

 use crate::function_registry::FunctionRegistry;

@@ -35,5 +37,6 @@ impl PGCatalogFunction {
    pub fn register(registry: &FunctionRegistry) {
        registry.register(Arc::new(PGTableIsVisibleFunction));
        registry.register(Arc::new(PGGetUserByIdFunction));
+        registry.register(Arc::new(PGVersionFunction));
    }
 }
--- a/src/common/function/src/system/pg_catalog/version.rs
+++ b/src/common/function/src/system/pg_catalog/version.rs
@@ -0,0 +1,54 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+use std::{env, fmt};
+
+use common_query::error::Result;
+use common_query::prelude::{Signature, Volatility};
+use datatypes::data_type::ConcreteDataType;
+use datatypes::vectors::{StringVector, VectorRef};
+
+use crate::function::{Function, FunctionContext};
+
+#[derive(Clone, Debug, Default)]
+pub(crate) struct PGVersionFunction;
+
+impl fmt::Display for PGVersionFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, crate::pg_catalog_func_fullname!("VERSION"))
+    }
+}
+
+impl Function for PGVersionFunction {
+    fn name(&self) -> &str {
+        crate::pg_catalog_func_fullname!("version")
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::exact(vec![], Volatility::Immutable)
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
+        let result = StringVector::from(vec![format!(
+            "PostgreSQL 16.3 GreptimeDB {}",
+            env!("CARGO_PKG_VERSION")
+        )]);
+        Ok(Arc::new(result))
+    }
+}
--- a/src/common/function/src/system/procedure_state.rs
+++ b/src/common/function/src/system/procedure_state.rs
@@ -96,7 +96,7 @@ mod tests {
    use datatypes::vectors::StringVector;

    use super::*;
-    use crate::function::{Function, FunctionContext};
+    use crate::function::{AsyncFunction, FunctionContext};

    #[test]
    fn test_procedure_state_misc() {
@@ -114,8 +114,8 @@ mod tests {
        ));
    }

-    #[test]
-    fn test_missing_procedure_service() {
+    #[tokio::test]
+    async fn test_missing_procedure_service() {
        let f = ProcedureStateFunction;

        let args = vec!["pid"];
@@ -125,15 +125,15 @@ mod tests {
            .map(|arg| Arc::new(StringVector::from_slice(&[arg])) as _)
            .collect::<Vec<_>>();

-        let result = f.eval(FunctionContext::default(), &args).unwrap_err();
+        let result = f.eval(FunctionContext::default(), &args).await.unwrap_err();
        assert_eq!(
            "Missing ProcedureServiceHandler, not expected",
            result.to_string()
        );
    }

-    #[test]
-    fn test_procedure_state() {
+    #[tokio::test]
+    async fn test_procedure_state() {
        let f = ProcedureStateFunction;

        let args = vec!["pid"];
@@ -143,7 +143,7 @@ mod tests {
            .map(|arg| Arc::new(StringVector::from_slice(&[arg])) as _)
            .collect::<Vec<_>>();

-        let result = f.eval(FunctionContext::mock(), &args).unwrap();
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();

        let expect: VectorRef = Arc::new(StringVector::from(vec![
            "{\"status\":\"Done\",\"error\":\"OK\"}",
--- a/src/common/function/src/table.rs
+++ b/src/common/function/src/table.rs
@@ -31,11 +31,11 @@ pub(crate) struct TableFunction;
 impl TableFunction {
    /// Register all table functions to [`FunctionRegistry`].
    pub fn register(registry: &FunctionRegistry) {
-        registry.register(Arc::new(MigrateRegionFunction));
-        registry.register(Arc::new(FlushRegionFunction));
-        registry.register(Arc::new(CompactRegionFunction));
-        registry.register(Arc::new(FlushTableFunction));
-        registry.register(Arc::new(CompactTableFunction));
-        registry.register(Arc::new(FlushFlowFunction));
+        registry.register_async(Arc::new(MigrateRegionFunction));
+        registry.register_async(Arc::new(FlushRegionFunction));
+        registry.register_async(Arc::new(CompactRegionFunction));
+        registry.register_async(Arc::new(FlushTableFunction));
+        registry.register_async(Arc::new(CompactTableFunction));
+        registry.register_async(Arc::new(FlushFlowFunction));
    }
 }
--- a/src/common/function/src/table/flush_compact_region.rs
+++ b/src/common/function/src/table/flush_compact_region.rs
@@ -77,7 +77,7 @@ mod tests {
    use datatypes::vectors::UInt64Vector;

    use super::*;
-    use crate::function::{Function, FunctionContext};
+    use crate::function::{AsyncFunction, FunctionContext};

    macro_rules! define_region_function_test {
        ($name: ident, $func: ident) => {
@@ -97,8 +97,8 @@ mod tests {
                                     } if valid_types == ConcreteDataType::numerics()));
                }

-                #[test]
-                fn [<test_ $name _missing_table_mutation>]() {
+                #[tokio::test]
+                async fn [<test_ $name _missing_table_mutation>]() {
                    let f = $func;

                    let args = vec![99];
@@ -108,15 +108,15 @@ mod tests {
                        .map(|arg| Arc::new(UInt64Vector::from_slice([arg])) as _)
                        .collect::<Vec<_>>();

-                    let result = f.eval(FunctionContext::default(), &args).unwrap_err();
+                    let result = f.eval(FunctionContext::default(), &args).await.unwrap_err();
                    assert_eq!(
                        "Missing TableMutationHandler, not expected",
                        result.to_string()
                    );
                }

-                #[test]
-                fn [<test_ $name>]() {
+                #[tokio::test]
+                async fn [<test_ $name>]() {
                    let f = $func;


@@ -127,7 +127,7 @@ mod tests {
                        .map(|arg| Arc::new(UInt64Vector::from_slice([arg])) as _)
                        .collect::<Vec<_>>();

-                    let result = f.eval(FunctionContext::mock(), &args).unwrap();
+                    let result = f.eval(FunctionContext::mock(), &args).await.unwrap();

                    let expect: VectorRef = Arc::new(UInt64Vector::from_slice([42]));
                    assert_eq!(expect, result);
--- a/src/common/function/src/table/flush_compact_table.rs
+++ b/src/common/function/src/table/flush_compact_table.rs
@@ -210,7 +210,7 @@ mod tests {
    use session::context::QueryContext;

    use super::*;
-    use crate::function::{Function, FunctionContext};
+    use crate::function::{AsyncFunction, FunctionContext};

    macro_rules! define_table_function_test {
        ($name: ident, $func: ident) => {
@@ -230,8 +230,8 @@ mod tests {
                                     } if valid_types == vec![ConcreteDataType::string_datatype()]));
                }

-                #[test]
-                fn [<test_ $name _missing_table_mutation>]() {
+                #[tokio::test]
+                async fn [<test_ $name _missing_table_mutation>]() {
                    let f = $func;

                    let args = vec!["test"];
@@ -241,15 +241,15 @@ mod tests {
                        .map(|arg| Arc::new(StringVector::from(vec![arg])) as _)
                        .collect::<Vec<_>>();

-                    let result = f.eval(FunctionContext::default(), &args).unwrap_err();
+                    let result = f.eval(FunctionContext::default(), &args).await.unwrap_err();
                    assert_eq!(
                        "Missing TableMutationHandler, not expected",
                        result.to_string()
                    );
                }

-                #[test]
-                fn [<test_ $name>]() {
+                #[tokio::test]
+                async fn [<test_ $name>]() {
                    let f = $func;


@@ -260,7 +260,7 @@ mod tests {
                        .map(|arg| Arc::new(StringVector::from(vec![arg])) as _)
                        .collect::<Vec<_>>();

-                    let result = f.eval(FunctionContext::mock(), &args).unwrap();
+                    let result = f.eval(FunctionContext::mock(), &args).await.unwrap();

                    let expect: VectorRef = Arc::new(UInt64Vector::from_slice([42]));
                    assert_eq!(expect, result);
--- a/src/common/function/src/table/migrate_region.rs
+++ b/src/common/function/src/table/migrate_region.rs
@@ -123,7 +123,7 @@ mod tests {
    use datatypes::vectors::{StringVector, UInt64Vector, VectorRef};

    use super::*;
-    use crate::function::{Function, FunctionContext};
+    use crate::function::{AsyncFunction, FunctionContext};

    #[test]
    fn test_migrate_region_misc() {
@@ -140,8 +140,8 @@ mod tests {
                         } if sigs.len() == 2));
    }

-    #[test]
-    fn test_missing_procedure_service() {
+    #[tokio::test]
+    async fn test_missing_procedure_service() {
        let f = MigrateRegionFunction;

        let args = vec![1, 1, 1];
@@ -151,15 +151,15 @@ mod tests {
            .map(|arg| Arc::new(UInt64Vector::from_slice([arg])) as _)
            .collect::<Vec<_>>();

-        let result = f.eval(FunctionContext::default(), &args).unwrap_err();
+        let result = f.eval(FunctionContext::default(), &args).await.unwrap_err();
        assert_eq!(
            "Missing ProcedureServiceHandler, not expected",
            result.to_string()
        );
    }

-    #[test]
-    fn test_migrate_region() {
+    #[tokio::test]
+    async fn test_migrate_region() {
        let f = MigrateRegionFunction;

        let args = vec![1, 1, 1];
@@ -169,7 +169,7 @@ mod tests {
            .map(|arg| Arc::new(UInt64Vector::from_slice([arg])) as _)
            .collect::<Vec<_>>();

-        let result = f.eval(FunctionContext::mock(), &args).unwrap();
+        let result = f.eval(FunctionContext::mock(), &args).await.unwrap();

        let expect: VectorRef = Arc::new(StringVector::from(vec!["test_pid"]));
        assert_eq!(expect, result);
--- a/src/common/macro/src/admin_fn.rs
+++ b/src/common/macro/src/admin_fn.rs
@@ -187,7 +187,8 @@ fn build_struct(
        }


-        impl crate::function::Function for #name {
+        #[async_trait::async_trait]
+        impl crate::function::AsyncFunction for #name {
            fn name(&self) -> &'static str {
                #display_name
            }
@@ -200,7 +201,7 @@ fn build_struct(
                #sig_fn()
            }

-            fn eval(&self, func_ctx: crate::function::FunctionContext, columns: &[datatypes::vectors::VectorRef]) ->  common_query::error::Result<datatypes::vectors::VectorRef> {
+            async fn eval(&self, func_ctx: crate::function::FunctionContext, columns: &[datatypes::vectors::VectorRef]) ->  common_query::error::Result<datatypes::vectors::VectorRef> {
                // Ensure under the `greptime` catalog for security
                crate::ensure_greptime!(func_ctx);

@@ -212,51 +213,36 @@ fn build_struct(
                };
                let columns = Vec::from(columns);

-                // TODO(dennis): DataFusion doesn't support async UDF currently
-                std::thread::spawn(move || {
-                    use snafu::OptionExt;
-                    use datatypes::data_type::DataType;
+                use snafu::OptionExt;
+                use datatypes::data_type::DataType;

-                    let query_ctx = &func_ctx.query_ctx;
-                    let handler = func_ctx
-                        .state
-                        .#handler
-                        .as_ref()
-                        .context(#snafu_type)?;
+                let query_ctx = &func_ctx.query_ctx;
+                let handler = func_ctx
+                    .state
+                    .#handler
+                    .as_ref()
+                    .context(#snafu_type)?;

-                    let mut builder = store_api::storage::ConcreteDataType::#ret()
-                        .create_mutable_vector(rows_num);
+                let mut builder = store_api::storage::ConcreteDataType::#ret()
+                    .create_mutable_vector(rows_num);

-                    if columns_num == 0 {
-                        let result = common_runtime::block_on_global(async move {
-                            #fn_name(handler, query_ctx, &[]).await
-                        })?;
+                if columns_num == 0 {
+                    let result = #fn_name(handler, query_ctx, &[]).await?;
+
+                    builder.push_value_ref(result.as_value_ref());
+                } else {
+                    for i in 0..rows_num {
+                        let args: Vec<_> = columns.iter()
+                            .map(|vector| vector.get_ref(i))
+                            .collect();
+
+                        let result = #fn_name(handler, query_ctx, &args).await?;

                        builder.push_value_ref(result.as_value_ref());
-                    } else {
-                        for i in 0..rows_num {
-                            let args: Vec<_> = columns.iter()
-                                .map(|vector| vector.get_ref(i))
-                                .collect();
-
-                            let result = common_runtime::block_on_global(async move {
-                                #fn_name(handler, query_ctx, &args).await
-                            })?;
-
-                            builder.push_value_ref(result.as_value_ref());
-                        }
                    }
+                }

-                    Ok(builder.to_vector())
-                })
-                    .join()
-                    .map_err(|e| {
-                        common_telemetry::error!(e; "Join thread error");
-                        common_query::error::Error::ThreadJoin {
-                            location: snafu::Location::default(),
-                        }
-                    })?
-
+                Ok(builder.to_vector())
            }

        }
--- a/src/common/macro/src/lib.rs
+++ b/src/common/macro/src/lib.rs
@@ -76,6 +76,7 @@ pub fn range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
 ///    - `&ProcedureServiceHandlerRef` or `&TableMutationHandlerRef` or `FlowServiceHandlerRef` as the first argument,
 ///    - `&QueryContextRef` as the second argument, and
 ///    - `&[ValueRef<'_>]` as the third argument which is SQL function input values in each row.
+///
 /// Return type must be `common_query::error::Result<Value>`.
 ///
 /// # Example see `common/function/src/system/procedure_state.rs`.
--- a/src/common/mem-prof/Cargo.toml
+++ b/src/common/mem-prof/Cargo.toml
@@ -15,8 +15,8 @@ tempfile = "3.4"
 tokio.workspace = true

 [target.'cfg(not(windows))'.dependencies]
-tikv-jemalloc-ctl = { version = "0.5", features = ["use_std"] }
+tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats"] }

 [target.'cfg(not(windows))'.dependencies.tikv-jemalloc-sys]
 features = ["stats", "profiling", "unprefixed_malloc_on_supported_platforms"]
-version = "0.5"
+version = "0.6"
--- a/src/common/meta/src/instruction.rs
+++ b/src/common/meta/src/instruction.rs
@@ -153,6 +153,9 @@ pub struct UpgradeRegion {
    /// it's helpful to verify whether the leader region is ready.
    #[serde(with = "humantime_serde")]
    pub wait_for_replay_timeout: Option<Duration>,
+    /// The hint for replaying memtable.
+    #[serde(default)]
+    pub location_id: Option<u64>,
 }

 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
--- a/src/common/meta/src/lock_key.rs
+++ b/src/common/meta/src/lock_key.rs
@@ -172,8 +172,8 @@ impl From<TableLock> for StringKey {
 ///
 /// Note:
 /// - Allows modification the corresponding region's [TableRouteValue](crate::key::table_route::TableRouteValue),
-/// [TableDatanodeValue](crate::key::datanode_table::DatanodeTableValue) even if
-/// it acquires the [RegionLock::Write] only without acquiring the [TableLock::Write].
+///   [TableDatanodeValue](crate::key::datanode_table::DatanodeTableValue) even if
+///   it acquires the [RegionLock::Write] only without acquiring the [TableLock::Write].
 ///
 /// - Should acquire [TableLock] of the table at same procedure.
 ///
--- a/src/common/meta/src/wal_options_allocator/kafka/topic_manager.rs
+++ b/src/common/meta/src/wal_options_allocator/kafka/topic_manager.rs
@@ -76,6 +76,10 @@ impl TopicManager {
    /// The initializer first tries to restore persisted topics from the kv backend.
    /// If not enough topics retrieved, the initializer will try to contact the Kafka cluster and request creating more topics.
    pub async fn start(&self) -> Result<()> {
+        // Skip creating topics.
+        if !self.config.auto_create_topics {
+            return Ok(());
+        }
        let num_topics = self.config.kafka_topic.num_topics;
        ensure!(num_topics > 0, InvalidNumTopicsSnafu { num_topics });

--- a/src/common/procedure/Cargo.toml
+++ b/src/common/procedure/Cargo.toml
@@ -13,7 +13,7 @@ workspace = true
 [dependencies]
 async-stream.workspace = true
 async-trait.workspace = true
-backon = "0.4"
+backon = "1"
 common-base.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
--- a/src/common/procedure/src/local.rs
+++ b/src/common/procedure/src/local.rs
@@ -51,7 +51,7 @@ const META_TTL: Duration = Duration::from_secs(60 * 10);
 /// [Notify] is not a condition variable, we can't guarantee the waiters are notified
 /// if they didn't call `notified()` before we signal the notify. So we
 /// 1. use dedicated notify for each condition, such as waiting for a lock, waiting
-/// for children;
+///    for children;
 /// 2. always use `notify_one` and ensure there are only one waiter.
 #[derive(Debug)]
 pub(crate) struct ProcedureMeta {
--- a/src/common/procedure/src/local/runner.rs
+++ b/src/common/procedure/src/local/runner.rs
@@ -373,7 +373,7 @@ impl Runner {
            procedure,
            manager_ctx: self.manager_ctx.clone(),
            step,
-            exponential_builder: self.exponential_builder.clone(),
+            exponential_builder: self.exponential_builder,
            store: self.store.clone(),
            rolling_back: false,
        };
--- a/src/common/recordbatch/src/error.rs
+++ b/src/common/recordbatch/src/error.rs
@@ -172,12 +172,13 @@ impl ErrorExt for Error {

            Error::DataTypes { .. }
            | Error::CreateRecordBatches { .. }
-            | Error::PollStream { .. }
            | Error::Format { .. }
            | Error::ToArrowScalar { .. }
            | Error::ProjectArrowRecordBatch { .. }
            | Error::PhysicalExpr { .. } => StatusCode::Internal,

+            Error::PollStream { .. } => StatusCode::EngineExecuteQuery,
+
            Error::ArrowCompute { .. } => StatusCode::IllegalState,

            Error::ColumnNotExists { .. } => StatusCode::TableColumnNotFound,
--- a/src/common/test-util/src/lib.rs
+++ b/src/common/test-util/src/lib.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-#![feature(lazy_cell)]
-
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::sync::LazyLock;
--- a/src/common/version/build.rs
+++ b/src/common/version/build.rs
@@ -19,7 +19,11 @@ use build_data::{format_timestamp, get_source_time};
 use shadow_rs::{CARGO_METADATA, CARGO_TREE};

 fn main() -> shadow_rs::SdResult<()> {
-    println!("cargo:rerun-if-changed=.git/refs/heads");
+    println!(
+        "cargo:rerun-if-changed={}/.git/refs/heads",
+        env!("CARGO_RUSTC_CURRENT_DIR")
+    );
+
    println!(
        "cargo:rustc-env=SOURCE_TIMESTAMP={}",
        if let Ok(t) = get_source_time() {
--- a/src/common/wal/src/config.rs
+++ b/src/common/wal/src/config.rs
@@ -53,6 +53,7 @@ impl From<DatanodeWalConfig> for MetasrvWalConfig {
                connection: config.connection,
                backoff: config.backoff,
                kafka_topic: config.kafka_topic,
+                auto_create_topics: config.auto_create_topics,
            }),
        }
    }
@@ -188,6 +189,7 @@ mod tests {
                replication_factor: 1,
                create_topic_timeout: Duration::from_secs(30),
            },
+            auto_create_topics: true,
        };
        assert_eq!(metasrv_wal_config, MetasrvWalConfig::Kafka(expected));

--- a/src/common/wal/src/config/kafka/common.rs
+++ b/src/common/wal/src/config/kafka/common.rs
@@ -187,7 +187,7 @@ impl Default for KafkaConnectionConfig {
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct KafkaTopicConfig {
-    /// Number of topics to be created upon start.
+    /// Number of topics.
    pub num_topics: usize,
    /// Number of partitions per topic.
    pub num_partitions: i32,
--- a/src/common/wal/src/config/kafka/datanode.rs
+++ b/src/common/wal/src/config/kafka/datanode.rs
@@ -40,6 +40,9 @@ pub struct DatanodeKafkaConfig {
    /// The kafka topic config.
    #[serde(flatten)]
    pub kafka_topic: KafkaTopicConfig,
+    // Automatically create topics for WAL.
+    pub auto_create_topics: bool,
+    // Create index for WAL.
    pub create_index: bool,
    #[serde(with = "humantime_serde")]
    pub dump_index_interval: Duration,
@@ -54,6 +57,7 @@ impl Default for DatanodeKafkaConfig {
            consumer_wait_timeout: Duration::from_millis(100),
            backoff: BackoffConfig::default(),
            kafka_topic: KafkaTopicConfig::default(),
+            auto_create_topics: true,
            create_index: true,
            dump_index_interval: Duration::from_secs(60),
        }
--- a/src/common/wal/src/config/kafka/metasrv.rs
+++ b/src/common/wal/src/config/kafka/metasrv.rs
@@ -18,7 +18,7 @@ use super::common::KafkaConnectionConfig;
 use crate::config::kafka::common::{backoff_prefix, BackoffConfig, KafkaTopicConfig};

 /// Kafka wal configurations for metasrv.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(default)]
 pub struct MetasrvKafkaConfig {
    /// The kafka connection config.
@@ -30,4 +30,17 @@ pub struct MetasrvKafkaConfig {
    /// The kafka config.
    #[serde(flatten)]
    pub kafka_topic: KafkaTopicConfig,
+    // Automatically create topics for WAL.
+    pub auto_create_topics: bool,
+}
+
+impl Default for MetasrvKafkaConfig {
+    fn default() -> Self {
+        Self {
+            connection: Default::default(),
+            backoff: Default::default(),
+            kafka_topic: Default::default(),
+            auto_create_topics: true,
+        }
+    }
 }
--- a/src/datanode/src/heartbeat/handler.rs
+++ b/src/datanode/src/heartbeat/handler.rs
@@ -206,6 +206,7 @@ mod tests {
            region_id,
            last_entry_id: None,
            wait_for_replay_timeout: None,
+            location_id: None,
        });
        assert!(
            heartbeat_handler.is_acceptable(&heartbeat_env.create_handler_ctx((meta, instruction)))
--- a/src/datanode/src/heartbeat/handler/upgrade_region.rs
+++ b/src/datanode/src/heartbeat/handler/upgrade_region.rs
@@ -27,6 +27,7 @@ impl HandlerContext {
            region_id,
            last_entry_id,
            wait_for_replay_timeout,
+            location_id,
        }: UpgradeRegion,
    ) -> BoxFuture<'static, InstructionReply> {
        Box::pin(async move {
@@ -62,6 +63,7 @@ impl HandlerContext {
                                RegionRequest::Catchup(RegionCatchupRequest {
                                    set_writable: true,
                                    entry_id: last_entry_id,
+                                    location_id,
                                }),
                            )
                            .await?;
@@ -151,6 +153,7 @@ mod tests {
                    region_id,
                    last_entry_id: None,
                    wait_for_replay_timeout,
+                    location_id: None,
                })
                .await;
            assert_matches!(reply, InstructionReply::UpgradeRegion(_));
@@ -191,6 +194,7 @@ mod tests {
                    region_id,
                    last_entry_id: None,
                    wait_for_replay_timeout,
+                    location_id: None,
                })
                .await;
            assert_matches!(reply, InstructionReply::UpgradeRegion(_));
@@ -232,6 +236,7 @@ mod tests {
                    region_id,
                    last_entry_id: None,
                    wait_for_replay_timeout,
+                    location_id: None,
                })
                .await;
            assert_matches!(reply, InstructionReply::UpgradeRegion(_));
@@ -274,8 +279,9 @@ mod tests {
                .clone()
                .handle_upgrade_region_instruction(UpgradeRegion {
                    region_id,
-                    last_entry_id: None,
                    wait_for_replay_timeout,
+                    last_entry_id: None,
+                    location_id: None,
                })
                .await;
            assert_matches!(reply, InstructionReply::UpgradeRegion(_));
@@ -293,6 +299,7 @@ mod tests {
                region_id,
                last_entry_id: None,
                wait_for_replay_timeout: Some(Duration::from_millis(500)),
+                location_id: None,
            })
            .await;
        assert_matches!(reply, InstructionReply::UpgradeRegion(_));
@@ -337,6 +344,7 @@ mod tests {
                region_id,
                last_entry_id: None,
                wait_for_replay_timeout: None,
+                location_id: None,
            })
            .await;
        assert_matches!(reply, InstructionReply::UpgradeRegion(_));
@@ -354,6 +362,7 @@ mod tests {
                region_id,
                last_entry_id: None,
                wait_for_replay_timeout: Some(Duration::from_millis(200)),
+                location_id: None,
            })
            .await;
        assert_matches!(reply, InstructionReply::UpgradeRegion(_));
--- a/src/datanode/src/tests.rs
+++ b/src/datanode/src/tests.rs
@@ -27,7 +27,7 @@ use common_runtime::Runtime;
 use query::dataframe::DataFrame;
 use query::plan::LogicalPlan;
 use query::planner::LogicalPlanner;
-use query::query_engine::DescribeResult;
+use query::query_engine::{DescribeResult, QueryEngineState};
 use query::{QueryEngine, QueryEngineContext};
 use session::context::QueryContextRef;
 use store_api::metadata::RegionMetadataRef;
@@ -86,6 +86,9 @@ impl QueryEngine for MockQueryEngine {
    fn engine_context(&self, _query_ctx: QueryContextRef) -> QueryEngineContext {
        unimplemented!()
    }
+    fn engine_state(&self) -> &QueryEngineState {
+        unimplemented!()
+    }
 }

 /// Create a region server without any engine
--- a/src/datatypes/src/value.rs
+++ b/src/datatypes/src/value.rs
@@ -268,6 +268,23 @@ impl Value {
        }
    }

+    /// Cast Value to f32. Return None if it's not castable;
+    pub fn as_f64_lossy(&self) -> Option<f64> {
+        match self {
+            Value::Float32(v) => Some(v.0 as _),
+            Value::Float64(v) => Some(v.0),
+            Value::Int8(v) => Some(*v as _),
+            Value::Int16(v) => Some(*v as _),
+            Value::Int32(v) => Some(*v as _),
+            Value::Int64(v) => Some(*v as _),
+            Value::UInt8(v) => Some(*v as _),
+            Value::UInt16(v) => Some(*v as _),
+            Value::UInt32(v) => Some(*v as _),
+            Value::UInt64(v) => Some(*v as _),
+            _ => None,
+        }
+    }
+
    /// Returns the logical type of the value.
    pub fn logical_type_id(&self) -> LogicalTypeId {
        match self {
--- a/src/flow/src/lib.rs
+++ b/src/flow/src/lib.rs
@@ -17,7 +17,6 @@
 //! It also contains definition of expression, adapter and plan, and internal state management.

 #![feature(let_chains)]
-#![feature(duration_abs_diff)]
 #![allow(dead_code)]
 #![warn(clippy::missing_docs_in_private_items)]
 #![warn(clippy::too_many_lines)]
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -441,6 +441,9 @@ pub fn check_permission(
    }

    match stmt {
+        // Will be checked in execution.
+        // TODO(dennis): add a hook for admin commands.
+        Statement::Admin(_) => {}
        // These are executed by query engine, and will be checked there.
        Statement::Query(_) | Statement::Explain(_) | Statement::Tql(_) | Statement::Delete(_) => {}
        // database ops won't be checked
--- a/src/log-store/Cargo.toml
+++ b/src/log-store/Cargo.toml
@@ -26,6 +26,7 @@ common-telemetry.workspace = true
 common-time.workspace = true
 common-wal.workspace = true
 delta-encoding = "0.4"
+derive_builder.workspace = true
 futures.workspace = true
 futures-util.workspace = true
 itertools.workspace = true
--- a/src/log-store/src/error.rs
+++ b/src/log-store/src/error.rs
@@ -304,6 +304,15 @@ pub enum Error {
        error: object_store::Error,
    },

+    #[snafu(display("Failed to read index, path: {path}"))]
+    ReadIndex {
+        #[snafu(implicit)]
+        location: Location,
+        #[snafu(source)]
+        error: object_store::Error,
+        path: String,
+    },
+
    #[snafu(display(
        "The length of meta if exceeded the limit: {}, actual: {}",
        limit,
--- a/src/log-store/src/kafka.rs
+++ b/src/log-store/src/kafka.rs
@@ -14,15 +14,10 @@

 pub(crate) mod client_manager;
 pub(crate) mod consumer;
-/// TODO(weny): remove it.
-#[allow(dead_code)]
-#[allow(unused_imports)]
 pub(crate) mod index;
 pub mod log_store;
 pub(crate) mod producer;
 pub(crate) mod util;
-/// TODO(weny): remove it.
-#[allow(dead_code)]
 pub(crate) mod worker;

 pub use index::{default_index_file, GlobalIndexCollector};
--- a/src/log-store/src/kafka/consumer.rs
+++ b/src/log-store/src/kafka/consumer.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
 use std::task::{Context, Poll};

 use common_telemetry::debug;
+use derive_builder::Builder;
 use futures::future::{BoxFuture, Fuse, FusedFuture};
 use futures::{FutureExt, Stream};
 use pin_project::pin_project;
@@ -60,40 +61,61 @@ struct FetchResult {
    used_offset: i64,
 }

+const MAX_BATCH_SIZE: usize = 52428800;
+const AVG_RECORD_SIZE: usize = 256 * 1024;
+
 /// The [`Consumer`] struct represents a Kafka consumer that fetches messages from
 /// a Kafka cluster. Yielding records respecting the [`RegionWalIndexIterator`].
 #[pin_project]
+#[derive(Builder)]
+#[builder(pattern = "owned")]
 pub struct Consumer {
+    #[builder(default = "-1")]
    last_high_watermark: i64,

    /// The client is used to fetch records from kafka topic.
    client: Arc<dyn FetchClient>,

    /// The max batch size in a single fetch request.
+    #[builder(default = "MAX_BATCH_SIZE")]
    max_batch_size: usize,

    /// The max wait milliseconds.
+    #[builder(default = "500")]
    max_wait_ms: u32,

    /// The avg record size
+    #[builder(default = "AVG_RECORD_SIZE")]
    avg_record_size: usize,

    /// Termination flag
+    #[builder(default = "false")]
    terminated: bool,

    /// The buffer of records.
    buffer: RecordsBuffer,

    /// The fetch future.
+    #[builder(default = "Fuse::terminated()")]
    fetch_fut: Fuse<BoxFuture<'static, rskafka::client::error::Result<FetchResult>>>,
 }

-struct RecordsBuffer {
+pub(crate) struct RecordsBuffer {
    buffer: VecDeque<RecordAndOffset>,

    index: Box<dyn RegionWalIndexIterator>,
 }

+impl RecordsBuffer {
+    /// Creates an empty [`RecordsBuffer`]
+    pub fn new(index: Box<dyn RegionWalIndexIterator>) -> Self {
+        RecordsBuffer {
+            buffer: VecDeque::new(),
+            index,
+        }
+    }
+}
+
 impl RecordsBuffer {
    fn pop_front(&mut self) -> Option<RecordAndOffset> {
        while let Some(index) = self.index.peek() {
--- a/src/log-store/src/kafka/index.rs
+++ b/src/log-store/src/kafka/index.rs
@@ -20,10 +20,11 @@ pub use collector::GlobalIndexCollector;
 pub(crate) use collector::{IndexCollector, NoopCollector};
 pub(crate) use encoder::{IndexEncoder, JsonIndexEncoder};
 pub(crate) use iterator::{
-    MultipleRegionWalIndexIterator, NextBatchHint, RegionWalIndexIterator, RegionWalRange,
-    RegionWalVecIndex,
+    build_region_wal_index_iterator, NextBatchHint, RegionWalIndexIterator, MIN_BATCH_WINDOW_SIZE,
 };
+#[cfg(test)]
+pub(crate) use iterator::{MultipleRegionWalIndexIterator, RegionWalRange, RegionWalVecIndex};

-pub fn default_index_file(datanode_id: u64) -> String {
-    format!("__datanode/{datanode_id}/index.json")
+pub fn default_index_file(location_id: u64) -> String {
+    format!("__wal/{location_id}/index.json")
 }
--- a/src/log-store/src/kafka/index/collector.rs
+++ b/src/log-store/src/kafka/index/collector.rs
@@ -19,6 +19,7 @@ use std::time::Duration;

 use common_telemetry::{error, info};
 use futures::future::try_join_all;
+use object_store::ErrorKind;
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 use store_api::logstore::provider::KafkaProvider;
@@ -28,8 +29,9 @@ use tokio::select;
 use tokio::sync::mpsc::Sender;
 use tokio::sync::Mutex as TokioMutex;

+use super::default_index_file;
 use crate::error::{self, Result};
-use crate::kafka::index::encoder::IndexEncoder;
+use crate::kafka::index::encoder::{DatanodeWalIndexes, IndexEncoder};
 use crate::kafka::index::JsonIndexEncoder;
 use crate::kafka::worker::{DumpIndexRequest, TruncateIndexRequest, WorkerRequest};

@@ -52,10 +54,11 @@ pub trait IndexCollector: Send + Sync {

 /// The [`GlobalIndexCollector`] struct is responsible for managing index entries
 /// across multiple providers.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct GlobalIndexCollector {
    providers: Arc<TokioMutex<HashMap<Arc<KafkaProvider>, Sender<WorkerRequest>>>>,
-    task: CollectionTask,
+    operator: object_store::ObjectStore,
+    _handle: CollectionTaskHandle,
 }

 #[derive(Debug, Clone)]
@@ -103,7 +106,7 @@ impl CollectionTask {
    /// The background task performs two main operations:
    /// - Persists the WAL index to the specified `path` at every `dump_index_interval`.
    /// - Updates the latest index ID for each WAL provider at every `checkpoint_interval`.
-    fn run(&self) {
+    fn run(self) -> CollectionTaskHandle {
        let mut dump_index_interval = tokio::time::interval(self.dump_index_interval);
        let running = self.running.clone();
        let moved_self = self.clone();
@@ -122,15 +125,23 @@ impl CollectionTask {
                }
            }
        });
+        CollectionTaskHandle {
+            running: self.running.clone(),
+        }
    }
 }

-impl Drop for CollectionTask {
+impl Drop for CollectionTaskHandle {
    fn drop(&mut self) {
        self.running.store(false, Ordering::Relaxed);
    }
 }

+#[derive(Debug, Default)]
+struct CollectionTaskHandle {
+    running: Arc<AtomicBool>,
+}
+
 impl GlobalIndexCollector {
    /// Constructs a [`GlobalIndexCollector`].
    ///
@@ -148,16 +159,65 @@ impl GlobalIndexCollector {
        let task = CollectionTask {
            providers: providers.clone(),
            dump_index_interval,
-            operator,
+            operator: operator.clone(),
            path,
            running: Arc::new(AtomicBool::new(true)),
        };
-        task.run();
-        Self { providers, task }
+        let handle = task.run();
+        Self {
+            providers,
+            operator,
+            _handle: handle,
+        }
+    }
+
+    #[cfg(test)]
+    pub fn new_for_test(operator: object_store::ObjectStore) -> Self {
+        Self {
+            providers: Default::default(),
+            operator,
+            _handle: Default::default(),
+        }
    }
 }

 impl GlobalIndexCollector {
+    /// Retrieve [`EntryId`]s for a specified `region_id` in `datanode_id`
+    /// that are greater than or equal to a given `entry_id`.
+    pub(crate) async fn read_remote_region_index(
+        &self,
+        location_id: u64,
+        provider: &KafkaProvider,
+        region_id: RegionId,
+        entry_id: EntryId,
+    ) -> Result<Option<(BTreeSet<EntryId>, EntryId)>> {
+        let path = default_index_file(location_id);
+
+        let bytes = match self.operator.read(&path).await {
+            Ok(bytes) => bytes.to_vec(),
+            Err(err) => {
+                if err.kind() == ErrorKind::NotFound {
+                    return Ok(None);
+                } else {
+                    return Err(err).context(error::ReadIndexSnafu { path });
+                }
+            }
+        };
+
+        match DatanodeWalIndexes::decode(&bytes)?.provider(provider) {
+            Some(indexes) => {
+                let last_index = indexes.last_index();
+                let indexes = indexes
+                    .region(region_id)
+                    .unwrap_or_default()
+                    .split_off(&entry_id);
+
+                Ok(Some((indexes, last_index)))
+            }
+            None => Ok(None),
+        }
+    }
+
    /// Creates a new [`ProviderLevelIndexCollector`] for a specified provider.
    pub(crate) async fn provider_level_index_collector(
        &self,
@@ -266,3 +326,92 @@ impl IndexCollector for NoopCollector {

    fn dump(&mut self, _encoder: &dyn IndexEncoder) {}
 }
+
+#[cfg(test)]
+mod tests {
+    use std::collections::{BTreeSet, HashMap};
+
+    use store_api::logstore::provider::KafkaProvider;
+    use store_api::storage::RegionId;
+
+    use crate::kafka::index::collector::RegionIndexes;
+    use crate::kafka::index::encoder::IndexEncoder;
+    use crate::kafka::index::JsonIndexEncoder;
+    use crate::kafka::{default_index_file, GlobalIndexCollector};
+
+    #[tokio::test]
+    async fn test_read_remote_region_index() {
+        let operator = object_store::ObjectStore::new(object_store::services::Memory::default())
+            .unwrap()
+            .finish();
+
+        let path = default_index_file(0);
+        let encoder = JsonIndexEncoder::default();
+        encoder.encode(
+            &KafkaProvider::new("my_topic_0".to_string()),
+            &RegionIndexes {
+                regions: HashMap::from([(RegionId::new(1, 1), BTreeSet::from([1, 5, 15]))]),
+                latest_entry_id: 20,
+            },
+        );
+        let bytes = encoder.finish().unwrap();
+        let mut writer = operator.writer(&path).await.unwrap();
+        writer.write(bytes).await.unwrap();
+        writer.close().await.unwrap();
+
+        let collector = GlobalIndexCollector::new_for_test(operator.clone());
+        // Index file doesn't exist
+        let result = collector
+            .read_remote_region_index(
+                1,
+                &KafkaProvider::new("my_topic_0".to_string()),
+                RegionId::new(1, 1),
+                1,
+            )
+            .await
+            .unwrap();
+        assert!(result.is_none());
+
+        // RegionId doesn't exist
+        let (indexes, last_index) = collector
+            .read_remote_region_index(
+                0,
+                &KafkaProvider::new("my_topic_0".to_string()),
+                RegionId::new(1, 2),
+                5,
+            )
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(indexes, BTreeSet::new());
+        assert_eq!(last_index, 20);
+
+        // RegionId(1, 1), Start EntryId: 5
+        let (indexes, last_index) = collector
+            .read_remote_region_index(
+                0,
+                &KafkaProvider::new("my_topic_0".to_string()),
+                RegionId::new(1, 1),
+                5,
+            )
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(indexes, BTreeSet::from([5, 15]));
+        assert_eq!(last_index, 20);
+
+        // RegionId(1, 1), Start EntryId: 20
+        let (indexes, last_index) = collector
+            .read_remote_region_index(
+                0,
+                &KafkaProvider::new("my_topic_0".to_string()),
+                RegionId::new(1, 1),
+                20,
+            )
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(indexes, BTreeSet::new());
+        assert_eq!(last_index, 20);
+    }
+}
--- a/src/log-store/src/kafka/index/encoder.rs
+++ b/src/log-store/src/kafka/index/encoder.rs
@@ -50,7 +50,7 @@ pub struct DeltaEncodedRegionIndexes {

 impl DeltaEncodedRegionIndexes {
    /// Retrieves the original (decoded) index values for a given region.
-    fn region(&self, region_id: RegionId) -> Option<BTreeSet<u64>> {
+    pub(crate) fn region(&self, region_id: RegionId) -> Option<BTreeSet<u64>> {
        let decoded = self
            .regions
            .get(&region_id)
@@ -60,7 +60,7 @@ impl DeltaEncodedRegionIndexes {
    }

    /// Retrieves the last index.
-    fn last_index(&self) -> u64 {
+    pub(crate) fn last_index(&self) -> u64 {
        self.last_index
    }
 }
@@ -86,7 +86,7 @@ impl DatanodeWalIndexes {
        value
    }

-    fn decode(byte: &[u8]) -> Result<Self> {
+    pub(crate) fn decode(byte: &[u8]) -> Result<Self> {
        serde_json::from_slice(byte).context(error::DecodeJsonSnafu)
    }

@@ -118,7 +118,7 @@ impl IndexEncoder for JsonIndexEncoder {

 #[cfg(test)]
 mod tests {
-    use std::collections::{BTreeSet, HashMap, HashSet};
+    use std::collections::{BTreeSet, HashMap};

    use store_api::logstore::provider::KafkaProvider;
    use store_api::storage::RegionId;
--- a/src/log-store/src/kafka/index/iterator.rs
+++ b/src/log-store/src/kafka/index/iterator.rs
@@ -12,8 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::cmp::min;
-use std::collections::VecDeque;
+use std::cmp::{max, min};
+use std::collections::{BTreeSet, VecDeque};
+use std::fmt::Debug;
 use std::ops::Range;

 use store_api::logstore::EntryId;
@@ -27,7 +28,7 @@ pub(crate) struct NextBatchHint {
 }

 /// An iterator over WAL (Write-Ahead Log) entries index for a region.
-pub trait RegionWalIndexIterator: Send + Sync {
+pub trait RegionWalIndexIterator: Send + Sync + Debug {
    /// Returns next batch hint.
    fn next_batch_hint(&self, avg_size: usize) -> Option<NextBatchHint>;

@@ -36,9 +37,13 @@ pub trait RegionWalIndexIterator: Send + Sync {

    // Advances the iterator and returns the next EntryId.
    fn next(&mut self) -> Option<EntryId>;
+
+    #[cfg(test)]
+    fn as_any(&self) -> &dyn std::any::Any;
 }

 /// Represents a range [next_entry_id, end_entry_id) of WAL entries for a region.
+#[derive(Debug)]
 pub struct RegionWalRange {
    current_entry_id: EntryId,
    end_entry_id: EntryId,
@@ -96,10 +101,18 @@ impl RegionWalIndexIterator for RegionWalRange {
            None
        }
    }
+
+    #[cfg(test)]
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
 }

+pub const MIN_BATCH_WINDOW_SIZE: usize = 4 * 1024 * 1024;
+
 /// Represents an index of Write-Ahead Log entries for a region,
 /// stored as a vector of [EntryId]s.
+#[derive(Debug)]
 pub struct RegionWalVecIndex {
    index: VecDeque<EntryId>,
    min_batch_window_size: usize,
@@ -134,11 +147,17 @@ impl RegionWalIndexIterator for RegionWalVecIndex {
    fn next(&mut self) -> Option<EntryId> {
        self.index.pop_front()
    }
+
+    #[cfg(test)]
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
 }

 /// Represents an iterator over multiple region WAL indexes.
 ///
 /// Allowing iteration through multiple WAL indexes.
+#[derive(Debug)]
 pub struct MultipleRegionWalIndexIterator {
    iterator: VecDeque<Box<dyn RegionWalIndexIterator>>,
 }
@@ -185,6 +204,53 @@ impl RegionWalIndexIterator for MultipleRegionWalIndexIterator {

        self.iterator.front_mut().and_then(|iter| iter.next())
    }
+
+    #[cfg(test)]
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+}
+
+/// Builds [`RegionWalIndexIterator`].
+///
+/// Returns None means there are no entries to replay.
+pub fn build_region_wal_index_iterator(
+    start_entry_id: EntryId,
+    end_entry_id: EntryId,
+    region_indexes: Option<(BTreeSet<EntryId>, EntryId)>,
+    max_batch_bytes: usize,
+    min_window_size: usize,
+) -> Option<Box<dyn RegionWalIndexIterator>> {
+    if (start_entry_id..end_entry_id).is_empty() {
+        return None;
+    }
+
+    match region_indexes {
+        Some((region_indexes, last_index)) => {
+            if region_indexes.is_empty() && last_index >= end_entry_id {
+                return None;
+            }
+
+            let mut iterator: Vec<Box<dyn RegionWalIndexIterator>> = Vec::with_capacity(2);
+            if !region_indexes.is_empty() {
+                let index = RegionWalVecIndex::new(region_indexes, min_window_size);
+                iterator.push(Box::new(index));
+            }
+            let known_last_index = max(last_index, start_entry_id);
+            if known_last_index < end_entry_id {
+                let range = known_last_index..end_entry_id;
+                let index = RegionWalRange::new(range, max_batch_bytes);
+                iterator.push(Box::new(index));
+            }
+
+            Some(Box::new(MultipleRegionWalIndexIterator::new(iterator)))
+        }
+        None => {
+            let range = start_entry_id..end_entry_id;
+
+            Some(Box::new(RegionWalRange::new(range, max_batch_bytes)))
+        }
+    }
 }

 #[cfg(test)]
@@ -353,4 +419,69 @@ mod tests {
        assert_eq!(iter.peek(), None);
        assert_eq!(iter.next(), None);
    }
+
+    #[test]
+    fn test_build_region_wal_index_iterator() {
+        let iterator = build_region_wal_index_iterator(1024, 1024, None, 5, 5);
+        assert!(iterator.is_none());
+
+        let iterator = build_region_wal_index_iterator(1024, 1023, None, 5, 5);
+        assert!(iterator.is_none());
+
+        let iterator =
+            build_region_wal_index_iterator(1024, 1024, Some((BTreeSet::new(), 1024)), 5, 5);
+        assert!(iterator.is_none());
+
+        let iterator =
+            build_region_wal_index_iterator(1, 1024, Some((BTreeSet::new(), 1024)), 5, 5);
+        assert!(iterator.is_none());
+
+        let iterator =
+            build_region_wal_index_iterator(1, 1024, Some((BTreeSet::new(), 1025)), 5, 5);
+        assert!(iterator.is_none());
+
+        let iterator = build_region_wal_index_iterator(
+            1,
+            1024,
+            Some((BTreeSet::from([512, 756]), 1024)),
+            5,
+            5,
+        )
+        .unwrap();
+        let iter = iterator
+            .as_any()
+            .downcast_ref::<MultipleRegionWalIndexIterator>()
+            .unwrap();
+        assert_eq!(iter.iterator.len(), 1);
+        let vec_index = iter.iterator[0]
+            .as_any()
+            .downcast_ref::<RegionWalVecIndex>()
+            .unwrap();
+        assert_eq!(vec_index.index, VecDeque::from([512, 756]));
+
+        let iterator = build_region_wal_index_iterator(
+            1,
+            1024,
+            Some((BTreeSet::from([512, 756]), 1023)),
+            5,
+            5,
+        )
+        .unwrap();
+        let iter = iterator
+            .as_any()
+            .downcast_ref::<MultipleRegionWalIndexIterator>()
+            .unwrap();
+        assert_eq!(iter.iterator.len(), 2);
+        let vec_index = iter.iterator[0]
+            .as_any()
+            .downcast_ref::<RegionWalVecIndex>()
+            .unwrap();
+        assert_eq!(vec_index.index, VecDeque::from([512, 756]));
+        let wal_range = iter.iterator[1]
+            .as_any()
+            .downcast_ref::<RegionWalRange>()
+            .unwrap();
+        assert_eq!(wal_range.current_entry_id, 1023);
+        assert_eq!(wal_range.end_entry_id, 1024);
+    }
 }
--- a/src/log-store/src/kafka/log_store.rs
+++ b/src/log-store/src/kafka/log_store.rs
@@ -20,19 +20,20 @@ use common_telemetry::{debug, warn};
 use common_wal::config::kafka::DatanodeKafkaConfig;
 use futures::future::try_join_all;
 use futures_util::StreamExt;
-use rskafka::client::consumer::{StartOffset, StreamConsumerBuilder};
 use rskafka::client::partition::OffsetAt;
 use snafu::{OptionExt, ResultExt};
 use store_api::logstore::entry::{
    Entry, Id as EntryId, MultiplePartEntry, MultiplePartHeader, NaiveEntry,
 };
 use store_api::logstore::provider::{KafkaProvider, Provider};
-use store_api::logstore::{AppendBatchResponse, LogStore, SendableEntryStream};
+use store_api::logstore::{AppendBatchResponse, LogStore, SendableEntryStream, WalIndex};
 use store_api::storage::RegionId;

+use super::index::build_region_wal_index_iterator;
 use crate::error::{self, ConsumeRecordSnafu, Error, GetOffsetSnafu, InvalidProviderSnafu, Result};
 use crate::kafka::client_manager::{ClientManager, ClientManagerRef};
-use crate::kafka::index::GlobalIndexCollector;
+use crate::kafka::consumer::{ConsumerBuilder, RecordsBuffer};
+use crate::kafka::index::{GlobalIndexCollector, MIN_BATCH_WINDOW_SIZE};
 use crate::kafka::producer::OrderedBatchProducerRef;
 use crate::kafka::util::record::{
    convert_to_kafka_records, maybe_emit_entry, remaining_entries, Record, ESTIMATED_META_SIZE,
@@ -205,6 +206,7 @@ impl LogStore for KafkaLogStore {
        &self,
        provider: &Provider,
        entry_id: EntryId,
+        index: Option<WalIndex>,
    ) -> Result<SendableEntryStream<'static, Entry, Self::Error>> {
        let provider = provider
            .as_kafka_provider()
@@ -232,35 +234,41 @@ impl LogStore for KafkaLogStore {
            .await
            .context(GetOffsetSnafu {
                topic: &provider.topic,
-            })?
-            - 1;
-        // Reads entries with offsets in the range [start_offset, end_offset].
-        let start_offset = entry_id as i64;
+            })?;

-        debug!(
-            "Start reading entries in range [{}, {}] for ns {}",
-            start_offset, end_offset, provider
-        );
+        let region_indexes = if let (Some(index), Some(collector)) =
+            (index, self.client_manager.global_index_collector())
+        {
+            collector
+                .read_remote_region_index(index.location_id, provider, index.region_id, entry_id)
+                .await?
+        } else {
+            None
+        };

-        // Abort if there're no new entries.
-        // FIXME(niebayes): how come this case happens?
-        if start_offset > end_offset {
-            warn!(
-                "No new entries for ns {} in range [{}, {}]",
-                provider, start_offset, end_offset
-            );
+        let Some(iterator) = build_region_wal_index_iterator(
+            entry_id,
+            end_offset as u64,
+            region_indexes,
+            self.max_batch_bytes,
+            MIN_BATCH_WINDOW_SIZE,
+        ) else {
+            let range = entry_id..end_offset as u64;
+            warn!("No new entries in range {:?} of ns {}", range, provider);
            return Ok(futures_util::stream::empty().boxed());
-        }
+        };

-        let mut stream_consumer = StreamConsumerBuilder::new(client, StartOffset::At(start_offset))
-            .with_max_batch_size(self.max_batch_bytes as i32)
-            .with_max_wait_ms(self.consumer_wait_timeout.as_millis() as i32)
-            .build();
+        debug!("Reading entries with {:?} of ns {}", iterator, provider);

-        debug!(
-            "Built a stream consumer for ns {} to consume entries in range [{}, {}]",
-            provider, start_offset, end_offset
-        );
+        // Safety: Must be ok.
+        let mut stream_consumer = ConsumerBuilder::default()
+            .client(client)
+            // Safety: checked before.
+            .buffer(RecordsBuffer::new(iterator))
+            .max_batch_size(self.max_batch_bytes)
+            .max_wait_ms(self.consumer_wait_timeout.as_millis() as u32)
+            .build()
+            .unwrap();

        // A buffer is used to collect records to construct a complete entry.
        let mut entry_records: HashMap<RegionId, Vec<Record>> = HashMap::new();
@@ -511,7 +519,7 @@ mod tests {
        // 5 region
        assert_eq!(response.last_entry_ids.len(), 5);
        let got_entries = logstore
-            .read(&provider, 0)
+            .read(&provider, 0, None)
            .await
            .unwrap()
            .try_collect::<Vec<_>>()
@@ -584,7 +592,7 @@ mod tests {
        // 5 region
        assert_eq!(response.last_entry_ids.len(), 5);
        let got_entries = logstore
-            .read(&provider, 0)
+            .read(&provider, 0, None)
            .await
            .unwrap()
            .try_collect::<Vec<_>>()
--- a/src/log-store/src/kafka/util/record.rs
+++ b/src/log-store/src/kafka/util/record.rs
@@ -40,8 +40,9 @@ pub(crate) const ESTIMATED_META_SIZE: usize = 256;
 /// - If the entry is able to fit into a Kafka record, it's converted into a Full record.
 ///
 /// - If the entry is too large to fit into a Kafka record, it's converted into a collection of records.
+///
 /// Those records must contain exactly one First record and one Last record, and potentially several
-/// Middle records. There may be no Middle record.
+///   Middle records. There may be no Middle record.
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
 pub enum RecordType {
    /// The record is self-contained, i.e. an entry's data is fully stored into this record.
--- a/src/log-store/src/raft_engine.rs
+++ b/src/log-store/src/raft_engine.rs
@@ -23,6 +23,7 @@ use store_api::logstore::entry::{Entry, NaiveEntry};
 use store_api::logstore::provider::Provider;
 use store_api::storage::RegionId;

+#[allow(renamed_and_removed_lints)]
 pub mod protos {
    include!(concat!(env!("OUT_DIR"), concat!("/", "protos/", "mod.rs")));
 }
--- a/src/log-store/src/raft_engine/log_store.rs
+++ b/src/log-store/src/raft_engine/log_store.rs
@@ -25,7 +25,7 @@ use raft_engine::{Config, Engine, LogBatch, MessageExt, ReadableSize, RecoveryMo
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::logstore::entry::{Entry, Id as EntryId, NaiveEntry};
 use store_api::logstore::provider::{Provider, RaftEngineProvider};
-use store_api::logstore::{AppendBatchResponse, LogStore, SendableEntryStream};
+use store_api::logstore::{AppendBatchResponse, LogStore, SendableEntryStream, WalIndex};
 use store_api::storage::RegionId;

 use crate::error::{
@@ -252,6 +252,7 @@ impl LogStore for RaftEngineLogStore {
        &self,
        provider: &Provider,
        entry_id: EntryId,
+        _index: Option<WalIndex>,
    ) -> Result<SendableEntryStream<'static, Entry, Self::Error>> {
        let ns = provider
            .as_raft_engine_provider()
@@ -545,7 +546,7 @@ mod tests {
        }
        let mut entries = HashSet::with_capacity(1024);
        let mut s = logstore
-            .read(&Provider::raft_engine_provider(1), 0)
+            .read(&Provider::raft_engine_provider(1), 0, None)
            .await
            .unwrap();
        while let Some(r) = s.next().await {
@@ -578,7 +579,7 @@ mod tests {
                .await
                .is_ok());
            let entries = logstore
-                .read(&Provider::raft_engine_provider(1), 1)
+                .read(&Provider::raft_engine_provider(1), 1, None)
                .await
                .unwrap()
                .collect::<Vec<_>>()
@@ -596,7 +597,7 @@ mod tests {

        let entries = collect_entries(
            logstore
-                .read(&Provider::raft_engine_provider(1), 1)
+                .read(&Provider::raft_engine_provider(1), 1, None)
                .await
                .unwrap(),
        )
@@ -682,7 +683,7 @@ mod tests {
        logstore.obsolete(&namespace, region_id, 100).await.unwrap();
        assert_eq!(101, logstore.engine.first_index(namespace_id).unwrap());

-        let res = logstore.read(&namespace, 100).await.unwrap();
+        let res = logstore.read(&namespace, 100, None).await.unwrap();
        let mut vec = collect_entries(res).await;
        vec.sort_by(|a, b| a.entry_id().partial_cmp(&b.entry_id()).unwrap());
        assert_eq!(101, vec.first().unwrap().entry_id());
--- a/src/meta-srv/src/lib.rs
+++ b/src/meta-srv/src/lib.rs
@@ -15,8 +15,8 @@
 #![feature(async_closure)]
 #![feature(result_flattening)]
 #![feature(assert_matches)]
-#![feature(option_take_if)]
 #![feature(extract_if)]
+#![feature(option_take_if)]

 pub mod bootstrap;
 mod cache_invalidator;
--- a/src/meta-srv/src/procedure/region_migration/update_metadata/downgrade_leader_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/update_metadata/downgrade_leader_region.rs
@@ -33,8 +33,8 @@ impl UpdateMetadata {
    /// About the failure of updating the [TableRouteValue](common_meta::key::table_region::TableRegionValue):
    ///
    /// - There may be another [RegionMigrationProcedure](crate::procedure::region_migration::RegionMigrationProcedure)
-    /// that is executed concurrently for **other region**.
-    /// It will only update **other region** info. Therefore, It's safe to retry after failure.
+    ///   that is executed concurrently for **other region**.
+    ///   It will only update **other region** info. Therefore, It's safe to retry after failure.
    ///
    /// - There is no other DDL procedure executed concurrently for the current table.
    pub async fn downgrade_leader_region(&self, ctx: &mut Context) -> Result<()> {
--- a/src/meta-srv/src/procedure/region_migration/update_metadata/upgrade_candidate_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/update_metadata/upgrade_candidate_region.rs
@@ -122,7 +122,7 @@ impl UpdateMetadata {
    ///
    /// Abort(non-retry):
    /// - TableRoute or RegionRoute is not found.
-    /// Typically, it's impossible, there is no other DDL procedure executed concurrently for the current table.
+    ///   Typically, it's impossible, there is no other DDL procedure executed concurrently for the current table.
    ///
    /// Retry:
    /// - Failed to update [TableRouteValue](common_meta::key::table_region::TableRegionValue).
--- a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
@@ -90,6 +90,7 @@ impl UpgradeCandidateRegion {
            region_id,
            last_entry_id,
            wait_for_replay_timeout: Some(self.replay_timeout),
+            location_id: Some(ctx.persistent_ctx.from_peer.id),
        })
    }

--- a/src/meta-srv/src/selector/weighted_choose.rs
+++ b/src/meta-srv/src/selector/weighted_choose.rs
@@ -27,6 +27,7 @@ pub trait WeightedChoose<Item>: Send + Sync {
    /// Note:
    /// 1. make sure weight_array is not empty.
    /// 2. the total weight is greater than 0.
+    ///
    /// Otherwise an error will be returned.
    fn set_weight_array(&mut self, weight_array: Vec<WeightedItem<Item>>) -> Result<()>;

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Ning Sun	0bff41e038	refactor: address review comments	2024-09-02 11:43:33 -07:00
Ning Sun	e56709b545	fix: toml format	2024-08-31 10:32:25 -07:00
Ning Sun	a64eb0a5bc	fix: lint error	2024-08-31 10:31:45 -07:00
Ning Sun	2e4ab6dd91	doc: correct h3 comment	2024-08-31 10:28:20 -07:00
Ning Sun	f761798f93	tests: add sqlness tests for geo functions	2024-08-31 10:25:19 -07:00
Ning Sun	420446f19f	feat: add built-in functions h3 and geohash	2024-08-31 10:14:27 -07:00
dennis zhuang	841e66c810	fix: config api and export metrics default database (#4633 )	2024-08-28 14:28:49 +00:00
shuiyisong	d1c635085c	chore: modify grafana config to accord with version 9 (#4634 ) chore: update grafana config to accord with version 9	2024-08-28 12:53:35 +00:00
Weny Xu	47657ebbc8	feat: replay WAL entries respect index (#4565 ) * feat(log_store): use new `Consumer` * feat: add `from_peer_id` * feat: read WAL entries respect index * test: add test for `build_region_wal_index_iterator` * fix: keep the handle * fix: incorrect last index * fix: replay last entry id may be greater than expected * chore: remove unused code * chore: apply suggestions from CR * chore: rename `datanode_id` to `location_id` * chore: rename `from_peer_id` to `location_id` * chore: rename `from_peer_id` to `location_id` * chore: apply suggestions from CR	2024-08-28 11:37:18 +00:00
Ruihang Xia	64ae32def0	feat: remove some redundent clone/conversion on constructing MergeScan stream (#4632 ) Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-08-28 08:52:09 +00:00
Weny Xu	744946957e	fix: set `selector_result_cache_size` in unit test (#4631 )	2024-08-28 07:24:17 +00:00
Ruihang Xia	d5455db2d5	fix: update properties on updating partitions (#4627 ) * fix: update properties on updating partitions Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * add unit test and handle insufficient ranges Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-08-28 07:06:01 +00:00
Yingwen	28bf549907	fix: fallback to window size in manifest (#4629 )	2024-08-28 06:43:56 +00:00
zyy17	4ea412249a	ci: add check-builder-rust-version job in release and change release-dev-builder-images trigger condition (#4615 )	2024-08-27 16:59:01 +00:00
zyy17	eacc7bc471	refactor: add `app` in `greptime_app_version` metric (#4626 ) refactor: add app in greptime_app_version metric	2024-08-27 11:19:01 +00:00
Ruihang Xia	b72d3bc71d	build(deps): bump backon to 1.0 (#4625 ) * build(deps): bump backon to 1.0 Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * fix clippy Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-08-27 09:38:12 +00:00
Ning Sun	0b102ef846	ci: improve toolchain resolution in ci (#4614 ) * ci: improve toolchain resolution in ci * fix: yaml format	2024-08-27 07:46:51 +00:00
liyang	e404e9dafc	chore: setting docker authentication in dev-build image (#4623 )	2024-08-27 03:49:53 +00:00
liyang	63a442632e	fix: failed to get version (#4622 )	2024-08-26 15:33:30 +00:00
liyang	d39bafcfbd	fix: change toolchain file name (#4621 )	2024-08-26 13:04:06 +00:00
liyang	1717445ebe	fix: failed to get github sha (#4620 )	2024-08-26 11:42:07 +00:00
liyang	55d65da24d	ci: add push dev-build images to aws ecr (#4618 ) * ci: add push dev-build images to aws ecr * chore: use toolchain file generation dev-build image tag * chore: change dev-build version * Update .github/workflows/release-dev-builder-images.yaml Co-authored-by: zyy17 <zyylsxm@gmail.com> --------- Co-authored-by: zyy17 <zyylsxm@gmail.com>	2024-08-26 09:36:55 +00:00
Weny Xu	3297d5f657	feat: allow skipping topic creation (#4616 ) * feat: introduce `create_topics` opt * feat: allow skipping topic creation * chore: refine docs * chore: apply suggestions from CR	2024-08-26 08:34:27 +00:00
Ning Sun	d6865911ee	feat: add postgres response for trasaction related statements (#4562 ) * feat: add postgres fixtures WIP * feat: implement more postgres fixtures * feat: add compatibility for transaction/set transaction/show transaction * fix: improve regex for set transaction	2024-08-26 08:09:21 +00:00
dennis zhuang	63f2463273	feat!: impl admin command (#4600 ) * feat: impl admin statement parser * feat: introduce AsyncFunction and implements it for admin functions * feat: execute admin functions * fix: license header * fix: panic in test * chore: fixed by code review	2024-08-26 07:53:40 +00:00
Ruihang Xia	da337a9635	perf: acclerate scatter query (#4607 ) Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-08-26 03:03:30 +00:00
fys	3973d6b01f	chore: optimize common_version build (#4611 )	2024-08-23 12:36:28 +00:00
discord9	2c731c76ad	chore: add `stats` feature for jemalloc-ctl (#4610 )	2024-08-23 11:18:30 +00:00
ozewr	40e7b58c80	feat: refactoring LruCacheLayer with list_with_metakey and concurrent_stat_in_list (#4596 ) * use list_with_metakey and concurrent_stat_in_list * change concurrent in recover_cache like before * remove stat funcation * use 8 concurrent * use const value * fmt code * Apply suggestions from code review --------- Co-authored-by: ozewr <l19ht@google.com> Co-authored-by: Weny Xu <wenymedia@gmail.com>	2024-08-23 03:22:00 +00:00
zyy17	5177717f71	refactor: add `fallback_to_local` region option (#4578 ) * refactor: add 'fallback_to_local_compaction' region option * refactor: use 'fallback_to_local'	2024-08-23 03:09:43 +00:00
Weny Xu	8d61e6fe49	chore: bump rskafka to `75535b` (#4608 )	2024-08-23 03:05:52 +00:00
Ruihang Xia	a3b8d2fe8f	chore: bump rust toolchain to 2024-08-21 (#4606 ) * chore: bump rust toolchain to 2024-08-22 Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * fix clippy Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * update workflow Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * try 20240606 Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-08-22 15:38:10 +00:00
Ning Sun	863ee073a9	chore: add commerial support section (#4601 ) doc: add commerial support section	2024-08-22 12:03:20 +00:00
Weny Xu	25cd61b310	chore: upgrade toolchain to nightly-2024-08-07 (#4549 ) * chore: upgrade toolchain to `nightly-2024-08-07` * chore(ci): upgrade toolchain * fix: fix unit test	2024-08-22 11:02:18 +00:00
fys	3517c13192	fix: incremental compilation always compile the common-version crate (#4605 ) fix: wrong cargo:rerun	2024-08-22 11:00:33 +00:00
Ruihang Xia	b9cedf2c1a	perf: optimize series divide algo (#4603 ) * perf: optimize series divide algo Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * remove dead code Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-08-22 09:16:36 +00:00
LFC	883c5bc5b0	refactor: skip checking the existence of the SST files (#4602 ) refactor: skip checking the existence of the SST files when region is directly edited	2024-08-22 08:32:27 +00:00
Yingwen	d628079f4c	feat: collect filters metrics for scanners (#4591 ) * feat: collect filter metrics * refactor: reuse ReaderFilterMetrics * feat: record read rows from parquet by type * feat: unordered scan observe rows also fix read type * chore: rename label	2024-08-22 03:22:05 +00:00
Weny Xu	0025fa6ec7	chore: bump opendal version to 0.49 (#4587 ) * chore: bump opendal version to 0.49 * chore: apply suggestions from CR * Update src/object-store/src/util.rs Co-authored-by: Yingwen <realevenyag@gmail.com> --------- Co-authored-by: Yingwen <realevenyag@gmail.com>	2024-08-22 03:05:36 +00:00
Lanqing Yang	ff04109ee6	docs: add example configs introduced by pg_kvbackend (#4573 ) chore: add example configs that introduced after pg_kvbackend	2024-08-22 01:52:02 +00:00
Yingwen	9c1704d4cb	docs: move v0.9.1 benchmark report to tsbs dir (#4598 ) * docs: move v0.9.1 benchmark report to tsbs dir * docs: add newlines	2024-08-21 09:31:05 +00:00
Yingwen	a12a905578	chore: disable ttl for write cache by default (#4595 ) * chore: remove default write cache ttl * docs: update example config * chore: fix ci	2024-08-21 08:38:38 +00:00
shuiyisong	449236360d	docs: log benchmark (#4597 ) * chore: add log benchmark stuff * chore: minor update	2024-08-21 07:12:32 +00:00
localhost	bf16422cee	fix: pipeline prepare loop break detects a conditional error (#4593 )	2024-08-21 06:20:09 +00:00
Ran Joe	9db08dbbe0	refactor(mito2): reduce duplicate IndexOutput struct (#4592 ) * refactor(mito2): reduce duplicate IndexOutput struct * docs(mito2): add index output note	2024-08-20 12:30:17 +00:00
fys	9d885fa0c2	chore: bump tikv-jemalloc* to "0.6" (#4590 ) chore: bump tikv-jemalloc* ti "0.6"	2024-08-20 09:08:21 +00:00
Ruihang Xia	b25a2b117e	feat: remove sql in error desc (#4589 ) Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-08-20 06:37:30 +00:00
fys	6fccff4810	chore: keep symbol table in nightly profile (#4588 ) chore: keep symbol table in nighly profile	2024-08-20 02:27:31 +00:00
ozewr	30af78700f	feat: Implement the Buf to avoid extra memory allocation (#4585 ) * feat: Implement the Buf to avoid extra memory allocation * fmt toml * fmt code * mv entry.into_buffer to raw_entry_buffer * less reuse opendal * remove todo #4065 * Update src/mito2/src/wal/entry_reader.rs Co-authored-by: Weny Xu <wenymedia@gmail.com> * fmt code --------- Co-authored-by: ozewr <l19ht@google.com> Co-authored-by: Weny Xu <wenymedia@gmail.com>	2024-08-19 12:11:08 +00:00
Ruihang Xia	8de11a0e34	perf: set simple filter on primary key columns to exact filter (#4564 ) * perf: set simple filter on primary key columns to exact filter Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * add sqlness test Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * fix typo Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * fix sqlness Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>	2024-08-19 09:07:35 +00:00