chore: update greptime-proto to main (#2672)

* chore: use https

* chore: use tls

.github/actions/build-dev-builder-image/action.yml

@@ -1,103 +0,0 @@
-name: Build and push dev-builder image
-description: Build and push dev-builder image to DockerHub and ACR
-inputs:
-  dockerhub-image-registry:
-    description: The dockerhub image registry to store the images
-    required: false
-    default: docker.io
-  dockerhub-image-registry-username:
-    description: The dockerhub username to login to the image registry
-    required: true
-  dockerhub-image-registry-token:
-    description: The dockerhub token to login to the image registry
-    required: true
-  dockerhub-image-namespace:
-    description: The dockerhub namespace of the image registry to store the images
-    required: false
-    default: greptime
-  acr-image-registry:
-    description: The ACR image registry to store the images
-    required: true
-  acr-image-registry-username:
-    description: The ACR username to login to the image registry
-    required: true
-  acr-image-registry-password:
-    description: The ACR password to login to the image registry
-    required: true
-  acr-image-namespace:
-    description: The ACR namespace of the image registry to store the images
-    required: false
-    default: greptime
-  version:
-    description: Version of the dev-builder
-    required: false
-    default: latest
-runs:
-  using: composite
-  steps:
-    - name: Login to Dockerhub
-      uses: docker/login-action@v2
-      with:
-        registry: ${{ inputs.dockerhub-image-registry }}
-        username: ${{ inputs.dockerhub-image-registry-username }}
-        password: ${{ inputs.dockerhub-image-registry-token }}
-
-    - name: Build and push ubuntu dev builder image to dockerhub
-      shell: bash
-      run:
-        make dev-builder \
-        BASE_IMAGE=ubuntu \
-        BUILDX_MULTI_PLATFORM_BUILD=true \
-        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
-        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-        IMAGE_TAG=${{ inputs.version }}
-
-    - name: Build and push centos dev builder image to dockerhub
-      shell: bash
-      run:
-        make dev-builder \
-        BASE_IMAGE=centos \
-        BUILDX_MULTI_PLATFORM_BUILD=true \
-        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
-        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-        IMAGE_TAG=${{ inputs.version }}
-
-    - name: Build and push android dev builder image to dockerhub
-      shell: bash
-      run:
-        make dev-builder \
-        BASE_IMAGE=android \
-        BUILDX_MULTI_PLATFORM_BUILD=true \
-        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
-        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-        IMAGE_TAG=${{ inputs.version }}
-
-    - name: Login to ACR
-      uses: docker/login-action@v2
-      continue-on-error: true
-      with:
-        registry: ${{ inputs.acr-image-registry }}
-        username: ${{ inputs.acr-image-registry-username }}
-        password: ${{ inputs.acr-image-registry-password }}
-
-    - name: Build and push ubuntu dev builder image to ACR
-      shell: bash
-      continue-on-error: true
-      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
-        make dev-builder \
-        BASE_IMAGE=ubuntu \
-        BUILDX_MULTI_PLATFORM_BUILD=true \
-        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
-        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
-        IMAGE_TAG=${{ inputs.version }}
-
-    - name: Build and push centos dev builder image to ACR
-      shell: bash
-      continue-on-error: true
-      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
-        make dev-builder \
-        BASE_IMAGE=centos \
-        BUILDX_MULTI_PLATFORM_BUILD=true \
-        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
-        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
-        IMAGE_TAG=${{ inputs.version }}

.github/actions/build-dev-builder-images/action.yml

@@ -0,0 +1,76 @@
+name: Build and push dev-builder images
+description: Build and push dev-builder images to DockerHub and ACR
+inputs:
+  dockerhub-image-registry:
+    description: The dockerhub image registry to store the images
+    required: false
+    default: docker.io
+  dockerhub-image-registry-username:
+    description: The dockerhub username to login to the image registry
+    required: true
+  dockerhub-image-registry-token:
+    description: The dockerhub token to login to the image registry
+    required: true
+  dockerhub-image-namespace:
+    description: The dockerhub namespace of the image registry to store the images
+    required: false
+    default: greptime
+  version:
+    description: Version of the dev-builder
+    required: false
+    default: latest
+  build-dev-builder-ubuntu:
+    description: Build dev-builder-ubuntu image
+    required: false
+    default: 'true'
+  build-dev-builder-centos:
+    description: Build dev-builder-centos image
+    required: false
+    default: 'true'
+  build-dev-builder-android:
+    description: Build dev-builder-android image
+    required: false
+    default: 'true'
+runs:
+  using: composite
+  steps:
+    - name: Login to Dockerhub
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.dockerhub-image-registry }}
+        username: ${{ inputs.dockerhub-image-registry-username }}
+        password: ${{ inputs.dockerhub-image-registry-token }}
+
+    - name: Build and push dev-builder-ubuntu image
+      shell: bash
+      if: ${{ inputs.build-dev-builder-ubuntu == 'true' }}
+      run: |
+        make dev-builder \
+          BASE_IMAGE=ubuntu \
+          BUILDX_MULTI_PLATFORM_BUILD=true \
+          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+          IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push dev-builder-centos image
+      shell: bash
+      if: ${{ inputs.build-dev-builder-centos == 'true' }}
+      run: |
+        make dev-builder \
+          BASE_IMAGE=centos \
+          BUILDX_MULTI_PLATFORM_BUILD=true \
+          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+          IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push dev-builder-android image # Only build image for amd64 platform.
+      shell: bash
+      if: ${{ inputs.build-dev-builder-android == 'true' }}
+      run: |
+        make dev-builder \
+          BASE_IMAGE=android \
+          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+          IMAGE_TAG=${{ inputs.version }} && \
+        
+        docker push ${{ inputs.dockerhub-image-registry }}/${{ inputs.dockerhub-image-namespace }}/dev-builder-android:${{ inputs.version }}

.github/actions/build-greptime-binary/action.yml

@@ -16,35 +16,20 @@ inputs:
   version:
     description: Version of the artifact
     required: true
-  release-to-s3-bucket:
-    description: S3 bucket to store released artifacts
-    required: true
-  aws-access-key-id:
-    description: AWS access key id
-    required: true
-  aws-secret-access-key:
-    description: AWS secret access key
-    required: true
-  aws-region:
-    description: AWS region
-    required: true
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
-  upload-latest-artifacts:
-    description: Upload the latest artifacts to S3
-    required: false
-    default: 'true'
   working-dir:
     description: Working directory to build the artifacts
     required: false
     default: .
+  build-android-artifacts:
+    description: Build android artifacts
+    required: false
+    default: 'false'
 runs:
   using: composite
   steps:
     - name: Build greptime binary
       shell: bash
+      if: ${{ inputs.build-android-artifacts == 'false' }}
       run: |
         cd ${{ inputs.working-dir }} && \
         make build-by-dev-builder \
@@ -54,14 +39,25 @@ runs:
 
     - name: Upload artifacts
       uses: ./.github/actions/upload-artifacts
+      if: ${{ inputs.build-android-artifacts == 'false' }}
       with:
         artifacts-dir: ${{ inputs.artifacts-dir }}
         target-file: ./target/${{ inputs.cargo-profile }}/greptime
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}
-        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}
+
+    # TODO(zyy17): We can remove build-android-artifacts flag in the future.
+    - name: Build greptime binary
+      shell: bash
+      if: ${{ inputs.build-android-artifacts == 'true' }}
+      run: |
+        cd ${{ inputs.working-dir }} && make strip-android-bin
+
+    - name: Upload android artifacts
+      uses: ./.github/actions/upload-artifacts
+      if: ${{ inputs.build-android-artifacts == 'true' }}
+      with:
+        artifacts-dir: ${{ inputs.artifacts-dir }}
+        target-file: ./target/aarch64-linux-android/release/greptime
+        version: ${{ inputs.version }}
         working-dir: ${{ inputs.working-dir }}

.github/actions/build-linux-artifacts/action.yml

@@ -13,30 +13,10 @@ inputs:
   disable-run-tests:
     description: Disable running integration tests
     required: true
-  release-to-s3-bucket:
-    description: S3 bucket to store released artifacts
-    required: true
-  aws-access-key-id:
-    description: AWS access key id
-    required: true
-  aws-secret-access-key:
-    description: AWS secret access key
-    required: true
-  aws-region:
-    description: AWS region
-    required: true
   dev-mode:
     description: Enable dev mode, only build standard greptime
     required: false
     default: 'false'
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
-  upload-latest-artifacts:
-    description: Upload the latest artifacts to S3
-    required: false
-    default: 'true'
   working-dir:
     description: Working directory to build the artifacts
     required: false
@@ -68,12 +48,6 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }}
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}
-        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
         working-dir: ${{ inputs.working-dir }}
 
     - name: Build greptime without pyo3
@@ -85,12 +59,6 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}
-        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
         working-dir: ${{ inputs.working-dir }}
 
     - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build.
@@ -107,10 +75,14 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}
-        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
         working-dir: ${{ inputs.working-dir }}
+
+    - name: Build greptime on android base image
+      uses: ./.github/actions/build-greptime-binary
+      if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Only build android base image on amd64.
+      with:
+        base-image: android
+        artifacts-dir: greptime-android-arm64-${{ inputs.version }}
+        version: ${{ inputs.version }}
+        working-dir: ${{ inputs.working-dir }}
+        build-android-artifacts: true

.github/actions/build-macos-artifacts/action.yml

@@ -19,25 +19,9 @@ inputs:
   disable-run-tests:
     description: Disable running integration tests
     required: true
-  release-to-s3-bucket:
-    description: S3 bucket to store released artifacts
-    required: true
   artifacts-dir:
     description: Directory to store artifacts
     required: true
-  aws-access-key-id:
-    description: AWS access key id
-    required: true
-  aws-secret-access-key:
-    description: AWS secret access key
-    required: true
-  aws-region:
-    description: AWS region
-    required: true
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
 runs:
   using: composite
   steps:
@@ -103,8 +87,3 @@ runs:
         artifacts-dir: ${{ inputs.artifacts-dir }}
         target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}

.github/actions/build-windows-artifacts/action.yml

@@ -0,0 +1,80 @@
+name: Build Windows artifacts
+description: Build Windows artifacts
+inputs:
+  arch:
+    description: Architecture to build
+    required: true
+  rust-toolchain:
+    description: Rust toolchain to use
+    required: true
+  cargo-profile:
+    description: Cargo profile to build
+    required: true
+  features:
+    description: Cargo features to build
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  disable-run-tests:
+    description: Disable running integration tests
+    required: true
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: true
+runs:
+  using: composite
+  steps:
+    - uses: arduino/setup-protoc@v1
+
+    - name: Install rust toolchain
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ inputs.rust-toolchain }}
+        targets: ${{ inputs.arch }}
+        components: llvm-tools-preview
+
+    - name: Rust Cache
+      uses: Swatinem/rust-cache@v2
+
+    - name: Install Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Install PyArrow Package
+      shell: pwsh
+      run: pip install pyarrow
+
+    - name: Install WSL distribution
+      uses: Vampire/setup-wsl@v2
+      with:
+        distribution: Ubuntu-22.04
+
+    - name: Install latest nextest release # For integration tests.
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      uses: taiki-e/install-action@nextest
+
+    - name: Run integration tests
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      shell: pwsh
+      run: make test sqlness-test
+
+    - name: Upload sqlness logs
+      if: ${{ failure() }} # Only upload logs when the integration tests failed.
+      uses: actions/upload-artifact@v3
+      with:
+        name: sqlness-logs
+        path: ${{ runner.temp }}/greptime-*.log
+        retention-days: 3
+
+    - name: Build greptime binary
+      shell: pwsh
+      run: cargo build --profile ${{ inputs.cargo-profile }} --features ${{ inputs.features }} --target ${{ inputs.arch }}
+
+    - name: Upload artifacts
+      uses: ./.github/actions/upload-artifacts
+      with:
+        artifacts-dir: ${{ inputs.artifacts-dir }}
+        target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
+        version: ${{ inputs.version }}

.github/actions/deploy-greptimedb/action.yml

@@ -0,0 +1,31 @@
+name: Deploy GreptimeDB cluster
+description: Deploy GreptimeDB cluster on Kubernetes
+inputs:
+  aws-ci-test-bucket:
+    description: 'AWS S3 bucket name for testing'
+    required: true
+  aws-region:
+    description: 'AWS region for testing'
+    required: true
+  data-root:
+    description: 'Data root for testing'
+    required: true
+  aws-access-key-id:
+    description: 'AWS access key id for testing'
+    required: true
+  aws-secret-access-key:
+    description: 'AWS secret access key for testing'
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Deploy GreptimeDB by Helm
+      shell: bash
+      env:
+        DATA_ROOT: ${{ inputs.data-root }}
+        AWS_CI_TEST_BUCKET: ${{ inputs.aws-ci-test-bucket }}
+        AWS_REGION: ${{ inputs.aws-region }}
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+      run: |
+        ./.github/scripts/deploy-greptimedb.sh

.github/actions/publish-github-release/action.yml

@@ -1,5 +1,5 @@
-name: Release artifacts
-description: Release artifacts
+name: Publish GitHub release
+description: Publish GitHub release
 inputs:
   version:
     description: Version to release

.github/actions/release-cn-artifacts/action.yaml

@@ -0,0 +1,138 @@
+name: Release CN artifacts
+description: Release artifacts to CN region
+inputs:
+  src-image-registry:
+    description: The source image registry to store the images
+    required: true
+    default: docker.io
+  src-image-namespace:
+    description: The namespace of the source image registry to store the images
+    required: true
+    default: greptime
+  src-image-name:
+    description: The name of the source image
+    required: false
+    default: greptimedb
+  dst-image-registry:
+    description: The destination image registry to store the images
+    required: true
+  dst-image-namespace:
+    description: The namespace of the destination image registry to store the images
+    required: true
+    default: greptime
+  dst-image-registry-username:
+    description: The username to login to the image registry
+    required: true
+  dst-image-registry-password:
+    description: The password to login to the image registry
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  dev-mode:
+    description: Enable dev mode, only push standard greptime
+    required: false
+    default: 'false'
+  push-latest-tag:
+    description: Whether to push the latest tag of the image
+    required: false
+    default: 'true'
+  aws-cn-s3-bucket:
+    description: S3 bucket to store released artifacts in CN region
+    required: true
+  aws-cn-access-key-id:
+    description: AWS access key id in CN region
+    required: true
+  aws-cn-secret-access-key:
+    description: AWS secret access key in CN region
+    required: true
+  aws-cn-region:
+    description: AWS region in CN
+    required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: false
+    default: 'artifacts'
+  update-version-info:
+    description: Update the version info in S3
+    required: false
+    default: 'true'
+  upload-max-retry-times:
+    description: Max retry times for uploading artifacts to S3
+    required: false
+    default: "20"
+  upload-retry-timeout:
+    description: Timeout for uploading artifacts to S3
+    required: false
+    default: "30" # minutes
+runs:
+  using: composite
+  steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v3
+      with:
+        path: ${{ inputs.artifacts-dir }}
+
+    - name: Release artifacts to cn region
+      uses: nick-invision/retry@v2
+      if: ${{ inputs.upload-to-s3 == 'true' }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-cn-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-cn-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-cn-region }}
+        UPDATE_VERSION_INFO: ${{ inputs.update-version-info }}
+      with:
+        max_attempts: ${{ inputs.upload-max-retry-times }}
+        timeout_minutes: ${{ inputs.upload-retry-timeout }}
+        command: |
+          ./.github/scripts/upload-artifacts-to-s3.sh \
+            ${{ inputs.artifacts-dir }} \
+            ${{ inputs.version }} \
+            ${{ inputs.aws-cn-s3-bucket }}
+
+    - name: Push greptimedb image from Dockerhub to ACR
+      shell: bash
+      env:
+        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
+        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
+      run: |
+        ./.github/scripts/copy-image.sh \
+         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}:${{ inputs.version }} \
+         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
+
+    - name: Push latest greptimedb image from Dockerhub to ACR
+      shell: bash
+      if: ${{ inputs.push-latest-tag == 'true' }}
+      env:
+        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
+        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
+      run: |
+        ./.github/scripts/copy-image.sh \
+         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}:latest \
+         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
+
+    - name: Push greptimedb-centos image from DockerHub to ACR
+      shell: bash
+      if: ${{ inputs.dev-mode == 'false' }}
+      env:
+        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
+        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
+      run: |
+        ./.github/scripts/copy-image.sh \
+         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}-centos:latest \
+         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
+
+    - name: Push greptimedb-centos image from DockerHub to ACR
+      shell: bash
+      if: ${{ inputs.dev-mode == 'false' && inputs.push-latest-tag == 'true' }}
+      env:
+        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
+        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
+      run: |
+        ./.github/scripts/copy-image.sh \
+         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}-centos:latest \
+         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}

.github/actions/sqlness-test/action.yml

@@ -0,0 +1,59 @@
+name: Run sqlness test
+description: Run sqlness test on GreptimeDB
+
+inputs:
+  aws-ci-test-bucket:
+    description: 'AWS S3 bucket name for testing'
+    required: true
+  aws-region:
+    description: 'AWS region for testing'
+    required: true
+  data-root:
+    description: 'Data root for testing'
+    required: true
+  aws-access-key-id:
+    description: 'AWS access key id for testing'
+    required: true
+  aws-secret-access-key:
+    description: 'AWS secret access key for testing'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Deploy GreptimeDB cluster by Helm
+      uses: ./.github/actions/deploy-greptimedb
+      with:
+        data-root: ${{ inputs.data-root }}
+        aws-ci-test-bucket: ${{ inputs.aws-ci-test-bucket }}
+        aws-region: ${{ inputs.aws-region }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+
+    # TODO(zyy17): The following tests will be replaced by the real sqlness test.
+    - name: Run tests on greptimedb cluster
+      shell: bash
+      run: |
+        mysql -h 127.0.0.1 -P 14002 -e "CREATE TABLE IF NOT EXISTS system_metrics (host VARCHAR(255), idc VARCHAR(255), cpu_util DOUBLE, memory_util DOUBLE, disk_util DOUBLE, ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(host, idc), TIME INDEX(ts));" && \
+        mysql -h 127.0.0.1 -P 14002 -e "SHOW TABLES;"
+
+    - name: Run tests on greptimedb cluster that uses S3
+      shell: bash
+      run: |
+        mysql -h 127.0.0.1 -P 24002 -e "CREATE TABLE IF NOT EXISTS system_metrics (host VARCHAR(255), idc VARCHAR(255), cpu_util DOUBLE, memory_util DOUBLE, disk_util DOUBLE, ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(host, idc), TIME INDEX(ts));" && \
+        mysql -h 127.0.0.1 -P 24002 -e "SHOW TABLES;"
+
+    - name: Run tests on standalone greptimedb
+      shell: bash
+      run: |
+        mysql -h 127.0.0.1 -P 34002 -e "CREATE TABLE IF NOT EXISTS system_metrics (host VARCHAR(255), idc VARCHAR(255), cpu_util DOUBLE, memory_util DOUBLE, disk_util DOUBLE, ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(host, idc), TIME INDEX(ts));" && \
+        mysql -h 127.0.0.1 -P 34002 -e "SHOW TABLES;"
+
+    - name: Clean S3 data
+      shell: bash
+      env:
+        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+      run: |
+        aws s3 rm s3://${{ inputs.aws-ci-test-bucket }}/${{ inputs.data-root }} --recursive

.github/actions/upload-artifacts/action.yml

@@ -10,34 +10,6 @@ inputs:
   version:
     description: Version of the artifact
     required: true
-  release-to-s3-bucket:
-    description: S3 bucket to store released artifacts
-    required: true
-  aws-access-key-id:
-    description: AWS access key id
-    required: true
-  aws-secret-access-key:
-    description: AWS secret access key
-    required: true
-  aws-region:
-    description: AWS region
-    required: true
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
-  upload-latest-artifacts:
-    description: Upload the latest artifacts to S3
-    required: false
-    default: 'true'
-  upload-max-retry-times:
-    description: Max retry times for uploading artifacts to S3
-    required: false
-    default: "20"
-  upload-retry-timeout:
-    description: Timeout for uploading artifacts to S3
-    required: false
-    default: "30" # minutes
   working-dir:
     description: Working directory to upload the artifacts
     required: false
@@ -61,9 +33,21 @@ runs:
       working-directory: ${{ inputs.working-dir }}
       shell: bash
       run: |
-        tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }} && \
+        tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }}
+
+    - name: Calculate checksum
+      if: runner.os != 'Windows'
+      working-directory: ${{ inputs.working-dir }}
+      shell: bash
+      run: |
         echo $(shasum -a 256 ${{ inputs.artifacts-dir }}.tar.gz | cut -f1 -d' ') > ${{ inputs.artifacts-dir }}.sha256sum
 
+    - name: Calculate checksum on Windows
+      if: runner.os == 'Windows'
+      working-directory: ${{ inputs.working-dir }}
+      shell: pwsh
+      run: Get-FileHash ${{ inputs.artifacts-dir }}.tar.gz -Algorithm SHA256 | select -ExpandProperty Hash > ${{ inputs.artifacts-dir }}.sha256sum
+
     # Note: The artifacts will be double zip compressed(related issue: https://github.com/actions/upload-artifact/issues/39).
     # However, when we use 'actions/download-artifact@v3' to download the artifacts, it will be automatically unzipped.
     - name: Upload artifacts
@@ -77,49 +61,3 @@ runs:
       with:
         name: ${{ inputs.artifacts-dir }}.sha256sum
         path: ${{ inputs.working-dir }}/${{ inputs.artifacts-dir }}.sha256sum
-
-    - name: Upload artifacts to S3
-      if: ${{ inputs.upload-to-s3 == 'true' }}
-      uses: nick-invision/retry@v2
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
-        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
-      with:
-        max_attempts: ${{ inputs.upload-max-retry-times }}
-        timeout_minutes: ${{ inputs.upload-retry-timeout }}
-        # The bucket layout will be:
-        # releases/greptimedb
-        # ├── v0.1.0
-        # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
-        # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
-        # └── v0.2.0
-        #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
-        #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
-        command: |
-          cd ${{ inputs.working-dir }} && \
-          aws s3 cp \
-            ${{ inputs.artifacts-dir }}.tar.gz \
-            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.tar.gz && \
-          aws s3 cp \
-            ${{ inputs.artifacts-dir }}.sha256sum \
-            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.sha256sum
-
-    - name: Upload latest artifacts to S3
-      if: ${{ inputs.upload-to-s3 == 'true' && inputs.upload-latest-artifacts == 'true' }} # We'll also upload the latest artifacts to S3 in the scheduled and formal release.
-      uses: nick-invision/retry@v2
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
-        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
-      with:
-        max_attempts: ${{ inputs.upload-max-retry-times }}
-        timeout_minutes: ${{ inputs.upload-retry-timeout }}
-        command: |
-          cd ${{ inputs.working-dir }} && \
-          aws s3 cp \
-            ${{ inputs.artifacts-dir }}.tar.gz \
-            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/latest/${{ inputs.artifacts-dir }}.tar.gz && \
-          aws s3 cp \
-            ${{ inputs.artifacts-dir }}.sha256sum \
-            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/latest/${{ inputs.artifacts-dir }}.sha256sum

.github/scripts/copy-image.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+SRC_IMAGE=$1
+DST_REGISTRY=$2
+SKOPEO_STABLE_IMAGE="quay.io/skopeo/stable:latest"
+
+# Check if necessary variables are set.
+function check_vars() {
+  for var in DST_REGISTRY_USERNAME DST_REGISTRY_PASSWORD DST_REGISTRY SRC_IMAGE; do
+    if [ -z "${!var}" ]; then
+      echo "$var is not set or empty."
+      echo "Usage: DST_REGISTRY_USERNAME=<your-dst-registry-username> DST_REGISTRY_PASSWORD=<your-dst-registry-password> $0 <dst-registry> <src-image>"
+      exit 1
+    fi
+  done
+}
+
+# Copies images from DockerHub to the destination registry.
+function copy_images_from_dockerhub() {
+  # Check if docker is installed.
+  if ! command -v docker &> /dev/null; then
+    echo "docker is not installed. Please install docker to continue."
+    exit 1
+  fi
+
+  # Extract the name and tag of the source image.
+  IMAGE_NAME=$(echo "$SRC_IMAGE" | sed "s/.*\///")
+
+  echo "Copying $SRC_IMAGE to $DST_REGISTRY/$IMAGE_NAME"
+
+  docker run "$SKOPEO_STABLE_IMAGE" copy -a docker://"$SRC_IMAGE" \
+    --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+    docker://"$DST_REGISTRY/$IMAGE_NAME"
+}
+
+function main() {
+  check_vars
+  copy_images_from_dockerhub
+}
+
+# Usage example:
+# DST_REGISTRY_USERNAME=123 DST_REGISTRY_PASSWORD=456 \
+#   ./copy-image.sh greptime/greptimedb:v0.4.0 greptime-registry.cn-hangzhou.cr.aliyuncs.com
+main

.github/scripts/deploy-greptimedb.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.24.0}"
+ENABLE_STANDALONE_MODE="${ENABLE_STANDALONE_MODE:-true}"
+DEFAULT_INSTALL_NAMESPACE=${DEFAULT_INSTALL_NAMESPACE:-default}
+GREPTIMEDB_IMAGE_TAG=${GREPTIMEDB_IMAGE_TAG:-latest}
+ETCD_CHART="oci://registry-1.docker.io/bitnamicharts/etcd"
+GREPTIME_CHART="https://greptimeteam.github.io/helm-charts/"
+
+# Ceate a cluster with 1 control-plane node and 5 workers.
+function create_kind_cluster() {
+  cat <<EOF | kind create cluster --name "${CLUSTER}" --image kindest/node:"$KUBERNETES_VERSION" --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+- role: worker
+- role: worker
+- role: worker
+EOF
+}
+
+# Add greptime Helm chart repo.
+function add_greptime_chart() {
+  helm repo add greptime "$GREPTIME_CHART"
+  helm repo update
+}
+
+# Deploy a etcd cluster with 3 members.
+function deploy_etcd_cluster() {
+  local namespace="$1"
+
+  helm install etcd "$ETCD_CHART" \
+    --set replicaCount=3 \
+    --set auth.rbac.create=false \
+    --set auth.rbac.token.enabled=false \
+    -n "$namespace"
+
+  # Wait for etcd cluster to be ready.
+  kubectl rollout status statefulset/etcd -n "$namespace"
+}
+
+# Deploy greptimedb-operator.
+function deploy_greptimedb_operator() {
+  # Use the latest chart and image.
+  helm install greptimedb-operator greptime/greptimedb-operator \
+    --set image.tag=latest \
+    -n "$DEFAULT_INSTALL_NAMESPACE"
+
+  # Wait for greptimedb-operator to be ready.
+  kubectl rollout status deployment/greptimedb-operator -n "$DEFAULT_INSTALL_NAMESPACE"
+}
+
+# Deploy greptimedb cluster by using local storage.
+# It will expose cluster service ports as '14000', '14001', '14002', '14003' to local access.
+function deploy_greptimedb_cluster() {
+  local cluster_name=$1
+  local install_namespace=$2
+
+  kubectl create ns "$install_namespace"
+
+  deploy_etcd_cluster "$install_namespace"
+
+  helm install "$cluster_name" greptime/greptimedb-cluster \
+    --set image.tag="$GREPTIMEDB_IMAGE_TAG" \
+    --set meta.etcdEndpoints="etcd.$install_namespace:2379" \
+    -n "$install_namespace"
+
+  # Wait for greptimedb cluster to be ready.
+  while true; do
+    PHASE=$(kubectl -n "$install_namespace" get gtc "$cluster_name" -o jsonpath='{.status.clusterPhase}')
+    if [ "$PHASE" == "Running" ]; then
+      echo "Cluster is ready"
+      break
+    else
+      echo "Cluster is not ready yet: Current phase: $PHASE"
+      sleep 5 # wait for 5 seconds before check again.
+    fi
+  done
+
+  # Expose greptimedb cluster to local access.
+  kubectl -n "$install_namespace" port-forward svc/"$cluster_name"-frontend \
+    14000:4000 \
+    14001:4001 \
+    14002:4002 \
+    14003:4003 > /tmp/connections.out &
+}
+
+# Deploy greptimedb cluster by using S3.
+# It will expose cluster service ports as '24000', '24001', '24002', '24003' to local access.
+function deploy_greptimedb_cluster_with_s3_storage() {
+  local cluster_name=$1
+  local install_namespace=$2
+
+  kubectl create ns "$install_namespace"
+
+  deploy_etcd_cluster "$install_namespace"
+
+  helm install "$cluster_name" greptime/greptimedb-cluster -n "$install_namespace" \
+    --set image.tag="$GREPTIMEDB_IMAGE_TAG" \
+    --set meta.etcdEndpoints="etcd.$install_namespace:2379" \
+    --set storage.s3.bucket="$AWS_CI_TEST_BUCKET" \
+    --set storage.s3.region="$AWS_REGION" \
+    --set storage.s3.root="$DATA_ROOT" \
+    --set storage.s3.secretName=s3-credentials \
+    --set storage.credentials.secretName=s3-credentials \
+    --set storage.credentials.secretCreation.enabled=true \
+    --set storage.credentials.secretCreation.enableEncryption=false \
+    --set storage.credentials.secretCreation.data.access-key-id="$AWS_ACCESS_KEY_ID" \
+    --set storage.credentials.secretCreation.data.secret-access-key="$AWS_SECRET_ACCESS_KEY"
+
+  # Wait for greptimedb cluster to be ready.
+  while true; do
+    PHASE=$(kubectl -n "$install_namespace" get gtc "$cluster_name" -o jsonpath='{.status.clusterPhase}')
+    if [ "$PHASE" == "Running" ]; then
+      echo "Cluster is ready"
+      break
+    else
+      echo "Cluster is not ready yet: Current phase: $PHASE"
+      sleep 5 # wait for 5 seconds before check again.
+    fi
+  done
+
+  # Expose greptimedb cluster to local access.
+  kubectl -n "$install_namespace" port-forward svc/"$cluster_name"-frontend \
+    24000:4000 \
+    24001:4001 \
+    24002:4002 \
+    24003:4003 > /tmp/connections.out &
+}
+
+# Deploy standalone greptimedb.
+# It will expose cluster service ports as '34000', '34001', '34002', '34003' to local access.
+function deploy_standalone_greptimedb() {
+  helm install greptimedb-standalone greptime/greptimedb-standalone \
+    --set image.tag="$GREPTIMEDB_IMAGE_TAG" \
+    -n "$DEFAULT_INSTALL_NAMESPACE"
+
+  # Wait for etcd cluster to be ready.
+  kubectl rollout status statefulset/greptimedb-standalone -n "$DEFAULT_INSTALL_NAMESPACE"
+
+  # Expose greptimedb to local access.
+  kubectl -n "$DEFAULT_INSTALL_NAMESPACE" port-forward svc/greptimedb-standalone \
+    34000:4000 \
+    34001:4001 \
+    34002:4002 \
+    34003:4003 > /tmp/connections.out &
+}
+
+# Entrypoint of the script.
+function main() {
+  create_kind_cluster
+  add_greptime_chart
+
+  # Deploy standalone greptimedb in the same K8s.
+  if [ "$ENABLE_STANDALONE_MODE" == "true" ]; then
+    deploy_standalone_greptimedb
+  fi
+
+  deploy_greptimedb_operator
+  deploy_greptimedb_cluster testcluster testcluster
+  deploy_greptimedb_cluster_with_s3_storage testcluster-s3 testcluster-s3
+}
+
+# Usages:
+# - Deploy greptimedb cluster: ./deploy-greptimedb.sh
+main

.github/scripts/upload-artifacts-to-s3.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+ARTIFACTS_DIR=$1
+VERSION=$2
+AWS_S3_BUCKET=$3
+RELEASE_DIRS="releases/greptimedb"
+GREPTIMEDB_REPO="GreptimeTeam/greptimedb"
+
+# Check if necessary variables are set.
+function check_vars() {
+  for var in AWS_S3_BUCKET VERSION ARTIFACTS_DIR; do
+    if [ -z "${!var}" ]; then
+      echo "$var is not set or empty."
+      echo "Usage: $0 <artifacts-dir> <version> <aws-s3-bucket>"
+      exit 1
+    fi
+  done
+}
+
+# Uploads artifacts to AWS S3 bucket.
+function upload_artifacts() {
+  # The bucket layout will be:
+  # releases/greptimedb
+  # ├── latest-version.txt
+  # ├── latest-nightly-version.txt
+  # ├── v0.1.0
+  # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
+  # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
+  # └── v0.2.0
+  #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
+  #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
+  find "$ARTIFACTS_DIR" -type f \( -name "*.tar.gz" -o -name "*.sha256sum" \) | while IFS= read -r file; do
+    aws s3 cp \
+      "$file" "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/$VERSION/$(basename "$file")"
+  done
+}
+
+# Updates the latest version information in AWS S3 if UPDATE_VERSION_INFO is true.
+function update_version_info() {
+  if [ "$UPDATE_VERSION_INFO" == "true" ]; then
+    # If it's the officail release(like v1.0.0, v1.0.1, v1.0.2, etc.), update latest-version.txt.
+    if [[ "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+      echo "Updating latest-version.txt"
+      echo "$VERSION" > latest-version.txt
+      aws s3 cp \
+        latest-version.txt "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/latest-version.txt"
+    fi
+
+    # If it's the nightly release, update latest-nightly-version.txt.
+    if [[ "$VERSION" == *"nightly"* ]]; then
+      echo "Updating latest-nightly-version.txt"
+      echo "$VERSION" > latest-nightly-version.txt
+      aws s3 cp \
+        latest-nightly-version.txt "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/latest-nightly-version.txt"
+    fi
+  fi
+}
+
+# Downloads artifacts from Github if DOWNLOAD_ARTIFACTS_FROM_GITHUB is true.
+function download_artifacts_from_github() {
+  if [ "$DOWNLOAD_ARTIFACTS_FROM_GITHUB" == "true" ]; then
+    # Check if jq is installed.
+    if ! command -v jq &> /dev/null; then
+      echo "jq is not installed. Please install jq to continue."
+      exit 1
+    fi
+
+    # Get the latest release API response.
+    RELEASES_API_RESPONSE=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/$GREPTIMEDB_REPO/releases/latest")
+
+    # Extract download URLs for the artifacts.
+    # Exclude source code archives which are typically named as 'greptimedb-<version>.zip' or 'greptimedb-<version>.tar.gz'.
+    ASSET_URLS=$(echo "$RELEASES_API_RESPONSE" | jq -r '.assets[] | select(.name | test("greptimedb-.*\\.(zip|tar\\.gz)$") | not) | .browser_download_url')
+
+    # Download each asset.
+    while IFS= read -r url; do
+      if [ -n "$url" ]; then
+        curl -LJO "$url"
+        echo "Downloaded: $url"
+      fi
+    done <<< "$ASSET_URLS"
+  fi
+}
+
+function main() {
+  check_vars
+  download_artifacts_from_github
+  upload_artifacts
+  update_version_info
+}
+
+# Usage example:
+#   AWS_ACCESS_KEY_ID=<your_access_key_id> \
+#   AWS_SECRET_ACCESS_KEY=<your_secret_access_key> \
+#   AWS_DEFAULT_REGION=<your_region> \
+#   UPDATE_VERSION_INFO=true \
+#   DOWNLOAD_ARTIFACTS_FROM_GITHUB=false \
+#     ./upload-artifacts-to-s3.sh <artifacts-dir> <version> <aws-s3-bucket>
+main

.github/workflows/apidoc.yml

@@ -17,7 +17,7 @@ env:
 
 jobs:
   apidoc:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v3
     - uses: arduino/setup-protoc@v1

.github/workflows/dev-build.yml

@@ -16,11 +16,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-latest
-          - ubuntu-latest-8-cores
-          - ubuntu-latest-16-cores
-          - ubuntu-latest-32-cores
-          - ubuntu-latest-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -78,7 +78,7 @@ jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -164,12 +164,7 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
           dev-mode: true # Only build the standard greptime binary.
-          upload-to-s3: false # No need to upload to S3.
           working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
 
   build-linux-arm64-artifacts:
@@ -198,12 +193,7 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
           dev-mode: true # Only build the standard greptime binary.
-          upload-to-s3: false # No need to upload to S3.
           working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
 
   release-images-to-dockerhub:
@@ -214,7 +204,7 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       build-result: ${{ steps.set-build-result.outputs.build-result }}
     steps:
@@ -239,41 +229,44 @@ jobs:
         run: |
           echo "build-result=success" >> $GITHUB_OUTPUT
 
-  release-images-to-acr:
-    name: Build and push images to ACR
+  release-cn-artifacts:
+    name: Release artifacts to CN region
     if: ${{ inputs.release_images || github.event_name == 'schedule' }}
     needs: [
       allocate-runners,
-      build-linux-amd64-artifacts,
-      build-linux-arm64-artifacts,
+      release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
-    # When we push to ACR, it's easy to fail due to some unknown network issues.
-    # However, we don't want to fail the whole workflow because of this.
-    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
+    runs-on: ubuntu-20.04
     continue-on-error: true
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Build and push images to ACR
-        uses: ./.github/actions/build-images
+      - name: Release artifacts to CN region
+        uses: ./.github/actions/release-cn-artifacts
         with:
-          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-name: ${{ env.IMAGE_NAME }}
-          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          src-image-registry: docker.io
+          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          src-image-name: ${{ env.IMAGE_NAME }}
+          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          push-latest-tag: false # Don't push the latest tag to registry.
-          dev-mode: true # Only build the standard images.
+          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: true                     # Only build the standard images(exclude centos images).
+          push-latest-tag: false             # Don't push the latest tag to registry.
+          update-version-info: false         # Don't update the version info in S3.
 
   stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -298,7 +291,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -325,7 +318,7 @@ jobs:
     needs: [
       release-images-to-dockerhub
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:

.github/workflows/develop.yml

@@ -34,7 +34,7 @@ env:
 jobs:
   typos:
     name: Spell Check with Typos
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - uses: crate-ci/typos@v1.13.10
@@ -42,7 +42,7 @@ jobs:
   check:
     name: Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -60,7 +60,7 @@ jobs:
   toml:
     name: Toml Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -80,7 +80,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest-8-cores, windows-latest-8-cores ]
+        os: [ ubuntu-20.04-8-cores ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -105,7 +105,7 @@ jobs:
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -124,7 +124,7 @@ jobs:
   clippy:
     name: Clippy
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -142,7 +142,7 @@ jobs:
 
   coverage:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-20.04-8-cores
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -188,43 +188,3 @@ jobs:
           flags: rust
           fail_ci_if_error: false
           verbose: true
-
-  test-on-windows:
-    if: github.event.pull_request.draft == false
-    runs-on: windows-latest-8-cores
-    timeout-minutes: 60
-    steps:
-      - run: git config --global core.autocrlf false
-      - uses: actions/checkout@v3
-      - uses: arduino/setup-protoc@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          components: llvm-tools-preview
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-      - name: Install Cargo Nextest
-        uses: taiki-e/install-action@nextest
-      - name: Install Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Install PyArrow Package
-        run: pip install pyarrow
-      - name: Install WSL distribution
-        uses: Vampire/setup-wsl@v2
-        with:
-         distribution: Ubuntu-22.04
-      - name: Running tests
-        run: cargo nextest run -F pyo3_backend,dashboard
-        env:
-          RUST_BACKTRACE: 1
-          CARGO_INCREMENTAL: 0
-          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-          GT_S3_REGION: ${{ secrets.S3_REGION }}
-          UNITTEST_LOG_DIR: "__unittest_logs"

.github/workflows/doc-issue.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   doc_issue:
     if: github.event.label.name == 'doc update required'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: create an issue in doc repo
         uses: dacbd/create-issue-action@main
@@ -25,7 +25,7 @@ jobs:
             ${{ github.event.issue.html_url || github.event.pull_request.html_url }}
   cloud_issue:
     if: github.event.label.name == 'cloud followup required'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: create an issue in cloud repo
         uses: dacbd/create-issue-action@main

.github/workflows/docs.yml

@@ -30,7 +30,7 @@ name: CI
 jobs:
   typos:
     name: Spell Check with Typos
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - uses: crate-ci/typos@v1.13.10
@@ -38,33 +38,33 @@ jobs:
   check:
     name: Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   clippy:
     name: Clippy
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   coverage:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   sqlness:
     name: Sqlness Test
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'

.github/workflows/license.yaml

@@ -8,7 +8,7 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
 jobs:
   license-header-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     name: license-header-check
     steps:
     - uses: actions/checkout@v2

.github/workflows/nightly-build.yml

@@ -14,11 +14,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.2xlarge-amd64
         options:
-          - ubuntu-latest
-          - ubuntu-latest-8-cores
-          - ubuntu-latest-16-cores
-          - ubuntu-latest-32-cores
-          - ubuntu-latest-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -70,7 +70,7 @@ jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -147,11 +147,6 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-latest-artifacts: false
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -171,11 +166,6 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-latest-artifacts: false
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub
@@ -185,7 +175,7 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       nightly-build-result: ${{ steps.set-nightly-build-result.outputs.nightly-build-result }}
     steps:
@@ -208,15 +198,14 @@ jobs:
         run: |
           echo "nightly-build-result=success" >> $GITHUB_OUTPUT
 
-  release-images-to-acr:
-    name: Build and push images to ACR
+  release-cn-artifacts:
+    name: Release artifacts to CN region
     if: ${{ inputs.release_images || github.event_name == 'schedule' }}
     needs: [
       allocate-runners,
-      build-linux-amd64-artifacts,
-      build-linux-arm64-artifacts,
+      release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -226,21 +215,30 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Build and push images to ACR
-        uses: ./.github/actions/build-images
+      - name: Release artifacts to CN region
+        uses: ./.github/actions/release-cn-artifacts
         with:
-          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          src-image-registry: docker.io
+          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          src-image-name: greptimedb
+          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          push-latest-tag: false # Don't push the latest tag to registry.
+          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: false
+          update-version-info: false  # Don't update version info in S3.
+          push-latest-tag: false      # Don't push the latest tag to registry.
 
   stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -265,7 +263,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -292,7 +290,7 @@ jobs:
     needs: [
       release-images-to-dockerhub
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:

.github/workflows/nightly-ci.yml

@@ -0,0 +1,98 @@
+# Nightly CI: runs tests every night for our second tier plaforms (Windows)
+
+on:
+  schedule:
+    - cron: '0 23 * * 1-5'
+  workflow_dispatch:
+
+name: Nightly CI
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  RUST_TOOLCHAIN: nightly-2023-08-07
+
+jobs:
+  sqlness:
+    name: Sqlness Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ windows-latest-8-cores ]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4.1.0
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Run sqlness
+        run: cargo sqlness
+      - name: Notify slack if failed
+        if: failure()
+        uses: slackapi/slack-github-action@v1.23.0
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
+        with:
+          payload: |
+            {"text": "Nightly CI failed for sqlness tests"}
+      - name: Upload sqlness logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: sqlness-logs
+          path: ${{ runner.temp }}/greptime-*.log
+          retention-days: 3
+
+  test-on-windows:
+    runs-on: windows-latest-8-cores
+    timeout-minutes: 60
+    steps:
+      - run: git config --global core.autocrlf false
+      - uses: actions/checkout@v4.1.0
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
+      - name: Install WSL distribution
+        uses: Vampire/setup-wsl@v2
+        with:
+          distribution: Ubuntu-22.04
+      - name: Running tests
+        run: cargo nextest run -F pyo3_backend,dashboard
+        env:
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          GT_S3_REGION: ${{ secrets.S3_REGION }}
+          UNITTEST_LOG_DIR: "__unittest_logs"
+      - name: Notify slack if failed
+        if: failure()
+        uses: slackapi/slack-github-action@v1.23.0
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
+        with:
+          payload: |
+            {"text": "Nightly CI failed for cargo test"}

.github/workflows/nightly-funtional-tests.yml

@@ -0,0 +1,26 @@
+name: Nightly functional tests
+
+on:
+  schedule:
+    # At 00:00 on Tuesday.
+    - cron: '0 0 * * 2'
+  workflow_dispatch:
+
+jobs:
+  sqlness-test:
+    name: Run sqlness test
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Run sqlness test
+        uses: ./.github/actions/sqlness-test
+        with:
+          data-root: sqlness-test
+          aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
+          aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}

.github/workflows/pr-title-checker.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 10
     steps:
       - uses: thehanimo/pr-title-checker@v1.3.4
@@ -19,7 +19,7 @@ jobs:
           pass_on_octokit_error: false
           configuration_path: ".github/pr-title-checker-config.json"
   breaking:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 10
     steps:
       - uses: thehanimo/pr-title-checker@v1.3.4

.github/workflows/release-dev-builder-images.yaml

@@ -0,0 +1,85 @@
+name: Release dev-builder images
+
+on:
+  workflow_dispatch: # Allows you to run this workflow manually.
+    inputs:
+      version:
+        description: Version of the dev-builder
+        required: false
+        default: latest
+      release_dev_builder_ubuntu_image:
+        type: boolean
+        description: Release dev-builder-ubuntu image
+        required: false
+        default: false
+      release_dev_builder_centos_image:
+        type: boolean
+        description: Release dev-builder-centos image
+        required: false
+        default: false
+      release_dev_builder_android_image:
+        type: boolean
+        description: Release dev-builder-android image
+        required: false
+        default: false
+
+jobs:
+  release-dev-builder-images:
+    name: Release dev builder images
+    if: ${{ inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }} # Only manually trigger this job.
+    runs-on: ubuntu-20.04-16-cores
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push dev builder images
+        uses: ./.github/actions/build-dev-builder-images
+        with:
+          version: ${{ inputs.version }}
+          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          build-dev-builder-ubuntu: ${{ inputs.release_dev_builder_ubuntu_image }}
+          build-dev-builder-centos: ${{ inputs.release_dev_builder_centos_image }}
+          build-dev-builder-android: ${{ inputs.release_dev_builder_android_image }}
+
+  release-dev-builder-images-cn: # Note: Be careful issue: https://github.com/containers/skopeo/issues/1874 and we decide to use the latest stable skopeo container.
+    name: Release dev builder images to CN region
+    runs-on: ubuntu-20.04
+    needs: [
+      release-dev-builder-images
+    ]
+    steps:
+      - name: Push dev-builder-ubuntu image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_ubuntu_image }}
+        env:
+          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
+          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }} \
+            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }}
+
+      - name: Push dev-builder-centos image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_centos_image }}
+        env:
+          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
+          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }} \
+            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }}
+
+      - name: Push dev-builder-android image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_android_image }}
+        env:
+          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
+          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }} \
+            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }}

.github/workflows/release.yml

@@ -18,11 +18,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-latest
-          - ubuntu-latest-8-cores
-          - ubuntu-latest-16-cores
-          - ubuntu-latest-32-cores
-          - ubuntu-latest-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -63,7 +63,12 @@ on:
         description: Build macos artifacts
         required: false
         default: false
-      release_artifacts:
+      build_windows_artifacts:
+        type: boolean
+        description: Build Windows artifacts
+        required: false
+        default: false
+      publish_github_release:
         type: boolean
         description: Create GitHub release and upload artifacts
         required: false
@@ -73,11 +78,6 @@ on:
         description: Build and push images to DockerHub and ACR
         required: false
         default: false
-      release_dev_builder_image:
-        type: boolean
-        description: Release dev-builder image
-        required: false
-        default: false
 
 # Use env variables to control all the release process.
 env:
@@ -97,11 +97,12 @@ jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
       macos-runner: ${{ inputs.macos_runner || vars.DEFAULT_MACOS_RUNNER }}
+      windows-runner: windows-latest-8-cores
 
       # The following EC2 resource id will be used for resource releasing.
       linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
@@ -177,11 +178,6 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -201,11 +197,6 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
 
   build-macos-artifacts:
     name: Build macOS artifacts
@@ -247,12 +238,43 @@ jobs:
           features: ${{ matrix.features }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
           artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
+
+  build-windows-artifacts:
+    name: Build Windows artifacts
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ${{ needs.allocate-runners.outputs.windows-runner }}
+            arch: x86_64-pc-windows-msvc
+            features: servers/dashboard
+            artifacts-dir-prefix: greptime-windows-amd64
+          - os: ${{ needs.allocate-runners.outputs.windows-runner }}
+            arch: x86_64-pc-windows-msvc
+            features: pyo3_backend,servers/dashboard
+            artifacts-dir-prefix: greptime-windows-amd64-pyo3
+    runs-on: ${{ matrix.os }}
+    needs: [
+      allocate-runners,
+    ]
+    if: ${{ inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    steps:
+      - run: git config --global core.autocrlf false
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-windows-artifacts
+        with:
+          arch: ${{ matrix.arch }}
+          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          features: ${{ matrix.features }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub
@@ -277,15 +299,18 @@ jobs:
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
 
-  release-images-to-acr:
-    name: Build and push images to ACR
+  release-cn-artifacts:
+    name: Release artifacts to CN region
     if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
-    needs: [
+    needs: [ # The job have to wait for all the artifacts are built.
       allocate-runners,
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
+      build-macos-artifacts,
+      build-windows-artifacts,
+      release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-2004-16-cores
+    runs-on: ubuntu-20.04
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -295,55 +320,47 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Build and push images to ACR
-        uses: ./.github/actions/build-images
+      - name: Release artifacts to CN region
+        uses: ./.github/actions/release-cn-artifacts
         with:
-          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          src-image-registry: docker.io
+          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          src-image-name: greptimedb
+          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
           version: ${{ needs.allocate-runners.outputs.version }}
+          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: false
+          update-version-info: true
+          push-latest-tag: true
 
-  release-artifacts:
+  publish-github-release:
     name: Create GitHub release and upload artifacts
-    if: ${{ inputs.release_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
-    needs: [
+    if: ${{ inputs.publish_github_release || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [ # The job have to wait for all the artifacts are built.
       allocate-runners,
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
       build-macos-artifacts,
+      build-windows-artifacts,
       release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Release artifacts
-        uses: ./.github/actions/release-artifacts
+      - name: Publish GitHub release
+        uses: ./.github/actions/publish-github-release
         with:
           version: ${{ needs.allocate-runners.outputs.version }}
 
-  release-dev-builder-image:
-    name: Release dev builder image
-    if: ${{ inputs.release_dev_builder_image }} # Only manually trigger this job.
-    runs-on: ubuntu-latest-16-cores
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Build and push dev builder image
-        uses: ./.github/actions/build-dev-builder-image
-        with:
-          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
-          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
-          acr-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          acr-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          acr-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
-
   ### Stop runners ###
   # It's very necessary to split the job of releasing runners into 'stop-linux-amd64-runner' and 'stop-linux-arm64-runner'.
   # Because we can terminate the specified EC2 instance immediately after the job is finished without uncessary waiting.
@@ -351,7 +368,7 @@ jobs:
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -376,7 +393,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,

.github/workflows/size-label.yml

@@ -0,0 +1,26 @@
+name: size-labeler
+
+on: [pull_request]
+
+jobs:
+  labeler:
+    runs-on: ubuntu-latest
+    name: Label the PR size
+    steps:
+      - uses: codelytv/pr-size-labeler@v1
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          s_label: 'Size: S'
+          s_max_size: '100'
+          m_label: 'Size: M'
+          m_max_size: '500'
+          l_label: 'Size: L'
+          l_max_size: '1000'
+          xl_label: 'Size: XL'
+          fail_if_xl: 'false'
+          message_if_xl: >
+            This PR exceeds the recommended size of 1000 lines.
+            Please make sure you are NOT addressing multiple issues with one PR.
+            Note this PR might be rejected due to its size.
+          github_api_url: 'api.github.com'
+          files_to_ignore: 'Cargo.lock'

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 Thanks a lot for considering contributing to GreptimeDB. We believe people like you would make GreptimeDB a great product. We intend to build a community where individuals can have open talks, show respect for one another, and speak with true ❤️. Meanwhile, we are to keep transparency and make your effort count here.
 
-Read the guidelines, and they can help you get started. Communicate with respect to developers maintaining and developing the project. In return, they should reciprocate that respect by addressing your issue, reviewing changes, as well as helping finalize and merge your pull requests.
+Please read the guidelines, and they can help you get started. Communicate with respect to developers maintaining and developing the project. In return, they should reciprocate that respect by addressing your issue, reviewing changes, as well as helping finalize and merge your pull requests.
 
 Follow our [README](https://github.com/GreptimeTeam/greptimedb#readme) to get the whole picture of the project. To learn about the design of GreptimeDB, please refer to the [design docs](https://github.com/GrepTimeTeam/docs).
 
@@ -21,7 +21,7 @@ Pull requests are great, but we accept all kinds of other help if you like. Such
 - Write tutorials or blog posts. Blog, speak about, or create tutorials about one of GreptimeDB's many features. Mention [@greptime](https://twitter.com/greptime) on Twitter and email info@greptime.com so we can give pointers and tips and help you spread the word by promoting your content on Greptime communication channels.
 - Improve the documentation. [Submit documentation](http://github.com/greptimeTeam/docs/) updates, enhancements, designs, or bug fixes, and fixing any spelling or grammar errors will be very much appreciated.
 - Present at meetups and conferences about your GreptimeDB projects. Your unique challenges and successes in building things with GreptimeDB can provide great speaking material. We'd love to review your talk abstract, so get in touch with us if you'd like some help!
-- Submit bug reports. To report a bug or a security issue, you can [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
+- Submitting bug reports. To report a bug or a security issue, you can [open a new GitHub issue](https://github.com/GrepTimeTeam/greptimedb/issues/new).
 - Speak up feature requests. Send feedback is a great way for us to understand your different use cases of GreptimeDB better. If you want to share your experience with GreptimeDB, or if you want to discuss any ideas, you can start a discussion on [GitHub discussions](https://github.com/GreptimeTeam/greptimedb/discussions), chat with the Greptime team on [Slack](https://greptime.com/slack), or you can tweet [@greptime](https://twitter.com/greptime) on Twitter.
 
 ## Code of Conduct
@@ -81,7 +81,7 @@ Now, `pre-commit` will run automatically on `git commit`.
 ### Title
 
 The titles of pull requests should be prefixed with category names listed in [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0)
-like `feat`/`fix`/`docs`, with a concise summary of code change following. DO NOT use last commit message as pull request title.
+like `feat`/`fix`/`docs`, with a concise summary of code change following. AVOID using the last commit message as pull request title.
 
 ### Description
 
@@ -100,7 +100,7 @@ of what you were trying to do and what went wrong. You can also reach for help i
 
 ## Community
 
-The core team will be thrilled if you participate in any way you like. When you are stuck, try ask for help by filing an issue, with a detailed description of what you were trying to do and what went wrong. If you have any questions or if you would like to get involved in our community, please check out:
+The core team will be thrilled if you would like to participate in any way you like. When you are stuck, try to ask for help by filing an issue, with a detailed description of what you were trying to do and what went wrong. If you have any questions or if you would like to get involved in our community, please check out:
 
 - [GreptimeDB Community Slack](https://greptime.com/slack)
 - [GreptimeDB Github Discussions](https://github.com/GreptimeTeam/greptimedb/discussions)

Cargo.toml

@@ -55,42 +55,46 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.4.0"
+version = "0.4.2"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
 aquamarine = "0.3"
-arrow = { version = "43.0" }
-arrow-array = "43.0"
-arrow-flight = "43.0"
-arrow-schema = { version = "43.0", features = ["serde"] }
+arrow = { version = "47.0" }
+arrow-array = "47.0"
+arrow-flight = "47.0"
+arrow-schema = { version = "47.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
+datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" }
 derive_builder = "0.12"
 etcd-client = "0.11"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "1f1dd532a111e3834cc3019c5605e2993ffb9dc3" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "5da72f1cae6b24315e5afc87520aaf7b4d6bb872" }
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
-metrics = "0.20"
 moka = "0.12"
 once_cell = "1.18"
-opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
-parquet = "43.0"
+opentelemetry-proto = { git = "https://github.com/waynexia/opentelemetry-rust.git", rev = "33841b38dda79b15f2024952be5f32533325ca02", features = [
+    "gen-tonic",
+    "metrics",
+    "trace",
+] }
+parquet = "47.0"
 paste = "1.0"
-prost = "0.11"
+prometheus = { version = "0.13.3", features = ["process"] }
+prost = "0.12"
 raft-engine = { git = "https://github.com/tikv/raft-engine.git", rev = "22dfb426cd994602b57725ef080287d3e53db479" }
 rand = "0.8"
 regex = "1.8"
@@ -103,7 +107,8 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 smallvec = "1"
 snafu = { version = "0.7", features = ["backtraces"] }
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4", features = [
+# on branch v0.38.x
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "602d7878c9949e48512251c7f18695a50936e51c", features = [
     "visitor",
 ] }
 strum = { version = "0.25", features = ["derive"] }
@@ -111,7 +116,7 @@ tempfile = "3"
 tokio = { version = "1.28", features = ["full"] }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.7"
-tonic = { version = "0.9", features = ["tls"] }
+tonic = { version = "0.10", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
 ## workspaces members
 api = { path = "src/api" }

Makefile

@@ -94,7 +94,7 @@ build-android-bin: ## Build greptime binary for android.
 	CARGO_BUILD_EXTRA_OPTS="--bin greptime --no-default-features"
 
 .PHONY: strip-android-bin
-strip-android-bin: ## Strip greptime binary for android.
+strip-android-bin: build-android-bin ## Strip greptime binary for android.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb \
 	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \

README.md

@@ -104,11 +104,11 @@ Or if you built from docker:
 docker run -p 4002:4002 -v "$(pwd):/tmp/greptimedb" greptime/greptimedb standalone start
 ```
 
-Please see [the online document site](https://docs.greptime.com/getting-started/overview#install-greptimedb) for more installation options and [operations info](https://docs.greptime.com/user-guide/operations/overview).
+Please see the online document site for more installation options and [operations info](https://docs.greptime.com/user-guide/operations/overview).
 
 ### Get started
 
-Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview#connect) on our [official document site](https://docs.greptime.com/).
+Read the [complete getting started guide](https://docs.greptime.com/getting-started/try-out-greptimedb) on our [official document site](https://docs.greptime.com/).
 
 To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients/overview).
 
@@ -184,6 +184,6 @@ Please refer to [contribution guidelines](CONTRIBUTING.md) for more information.
 ## Acknowledgement
 - GreptimeDB uses [Apache Arrow](https://arrow.apache.org/) as the memory model and [Apache Parquet](https://parquet.apache.org/) as the persistent file format.
 - GreptimeDB's query engine is powered by [Apache Arrow DataFusion](https://github.com/apache/arrow-datafusion).
-- [OpenDAL](https://github.com/datafuselabs/opendal) from [Datafuse Labs](https://github.com/datafuselabs) gives GreptimeDB a very general and elegant data access abstraction layer.
-- GreptimeDB’s meta service is based on [etcd](https://etcd.io/).
+- [Apache OpenDAL (incubating)](https://opendal.apache.org) gives GreptimeDB a very general and elegant data access abstraction layer.
+- GreptimeDB's meta service is based on [etcd](https://etcd.io/).
 - GreptimeDB uses [RustPython](https://github.com/RustPython/RustPython) for experimental embedded python scripting.

benchmarks/Cargo.toml

@@ -6,8 +6,10 @@ license.workspace = true
 
 [dependencies]
 arrow.workspace = true
+chrono.workspace = true
 clap = { version = "4.0", features = ["derive"] }
 client = { workspace = true }
+futures-util.workspace = true
 indicatif = "0.17.1"
 itertools.workspace = true
 parquet.workspace = true

benchmarks/src/bin/nyc-taxi.rs

@@ -29,14 +29,14 @@ use client::api::v1::column::Values;
 use client::api::v1::{
     Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests, SemanticType,
 };
-use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use client::{Client, Database, Output, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use futures_util::TryStreamExt;
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
 use tokio::task::JoinSet;
 
 const CATALOG_NAME: &str = "greptime";
 const SCHEMA_NAME: &str = "public";
-const TABLE_NAME: &str = "nyc_taxi";
 
 #[derive(Parser)]
 #[command(name = "NYC benchmark runner")]
@@ -74,7 +74,12 @@ fn get_file_list<P: AsRef<Path>>(path: P) -> Vec<PathBuf> {
         .collect()
 }
 
+fn new_table_name() -> String {
+    format!("nyc_taxi_{}", chrono::Utc::now().timestamp())
+}
+
 async fn write_data(
+    table_name: &str,
     batch_size: usize,
     db: &Database,
     path: PathBuf,
@@ -104,7 +109,7 @@ async fn write_data(
         }
         let (columns, row_count) = convert_record_batch(record_batch);
         let request = InsertRequest {
-            table_name: TABLE_NAME.to_string(),
+            table_name: table_name.to_string(),
             columns,
             row_count,
         };
@@ -113,7 +118,7 @@ async fn write_data(
         };
 
         let now = Instant::now();
-        let _ = db.insert(requests).await.unwrap();
+        db.insert(requests).await.unwrap();
         let elapsed = now.elapsed();
         total_rpc_elapsed_ms += elapsed.as_millis();
         progress_bar.inc(row_count as _);
@@ -131,6 +136,11 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
 
     for (array, field) in record_batch.columns().iter().zip(fields.iter()) {
         let (values, datatype) = build_values(array);
+        let semantic_type = match field.name().as_str() {
+            "VendorID" => SemanticType::Tag,
+            "tpep_pickup_datetime" => SemanticType::Timestamp,
+            _ => SemanticType::Field,
+        };
 
         let column = Column {
             column_name: field.name().clone(),
@@ -141,8 +151,7 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
                 .map(|bitmap| bitmap.buffer().as_slice().to_vec())
                 .unwrap_or_default(),
             datatype: datatype.into(),
-            // datatype and semantic_type are set to default
-            ..Default::default()
+            semantic_type: semantic_type as i32,
         };
         columns.push(column);
     }
@@ -243,11 +252,11 @@ fn is_record_batch_full(batch: &RecordBatch) -> bool {
     batch.columns().iter().all(|col| col.null_count() == 0)
 }
 
-fn create_table_expr() -> CreateTableExpr {
+fn create_table_expr(table_name: &str) -> CreateTableExpr {
     CreateTableExpr {
         catalog_name: CATALOG_NAME.to_string(),
         schema_name: SCHEMA_NAME.to_string(),
-        table_name: TABLE_NAME.to_string(),
+        table_name: table_name.to_string(),
         desc: "".to_string(),
         column_defs: vec![
             ColumnDef {
@@ -261,7 +270,7 @@ fn create_table_expr() -> CreateTableExpr {
             ColumnDef {
                 name: "tpep_pickup_datetime".to_string(),
                 data_type: ColumnDataType::TimestampMicrosecond as i32,
-                is_nullable: true,
+                is_nullable: false,
                 default_constraint: vec![],
                 semantic_type: SemanticType::Timestamp as i32,
                 comment: String::new(),
@@ -405,31 +414,31 @@ fn create_table_expr() -> CreateTableExpr {
         ],
         time_index: "tpep_pickup_datetime".to_string(),
         primary_keys: vec!["VendorID".to_string()],
-        create_if_not_exists: false,
+        create_if_not_exists: true,
         table_options: Default::default(),
         table_id: None,
         engine: "mito".to_string(),
     }
 }
 
-fn query_set() -> HashMap<String, String> {
+fn query_set(table_name: &str) -> HashMap<String, String> {
     HashMap::from([
         (
             "count_all".to_string(), 
-            format!("SELECT COUNT(*) FROM {TABLE_NAME};"),
+            format!("SELECT COUNT(*) FROM {table_name};"),
         ),
         (
             "fare_amt_by_passenger".to_string(),
-            format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {TABLE_NAME} GROUP BY passenger_count"),
+            format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {table_name} GROUP BY passenger_count"),
         )
     ])
 }
 
-async fn do_write(args: &Args, db: &Database) {
+async fn do_write(args: &Args, db: &Database, table_name: &str) {
     let mut file_list = get_file_list(args.path.clone().expect("Specify data path in argument"));
     let mut write_jobs = JoinSet::new();
 
-    let create_table_result = db.create(create_table_expr()).await;
+    let create_table_result = db.create(create_table_expr(table_name)).await;
     println!("Create table result: {create_table_result:?}");
 
     let progress_bar_style = ProgressStyle::with_template(
@@ -447,8 +456,10 @@ async fn do_write(args: &Args, db: &Database) {
             let db = db.clone();
             let mpb = multi_progress_bar.clone();
             let pb_style = progress_bar_style.clone();
-            let _ = write_jobs
-                .spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+            let table_name = table_name.to_string();
+            let _ = write_jobs.spawn(async move {
+                write_data(&table_name, batch_size, &db, path, mpb, pb_style).await
+            });
         }
     }
     while write_jobs.join_next().await.is_some() {
@@ -457,24 +468,32 @@ async fn do_write(args: &Args, db: &Database) {
             let db = db.clone();
             let mpb = multi_progress_bar.clone();
             let pb_style = progress_bar_style.clone();
-            let _ = write_jobs
-                .spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+            let table_name = table_name.to_string();
+            let _ = write_jobs.spawn(async move {
+                write_data(&table_name, batch_size, &db, path, mpb, pb_style).await
+            });
         }
     }
 }
 
-async fn do_query(num_iter: usize, db: &Database) {
-    for (query_name, query) in query_set() {
+async fn do_query(num_iter: usize, db: &Database, table_name: &str) {
+    for (query_name, query) in query_set(table_name) {
         println!("Running query: {query}");
         for i in 0..num_iter {
             let now = Instant::now();
-            let _res = db.sql(&query).await.unwrap();
+            let res = db.sql(&query).await.unwrap();
+            match res {
+                Output::AffectedRows(_) | Output::RecordBatches(_) => (),
+                Output::Stream(stream) => {
+                    stream.try_collect::<Vec<_>>().await.unwrap();
+                }
+            }
             let elapsed = now.elapsed();
             println!(
                 "query {}, iteration {}: {}ms",
                 query_name,
                 i,
-                elapsed.as_millis()
+                elapsed.as_millis(),
             );
         }
     }
@@ -491,13 +510,14 @@ fn main() {
         .block_on(async {
             let client = Client::with_urls(vec![&args.endpoint]);
             let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+            let table_name = new_table_name();
 
             if !args.skip_write {
-                do_write(&args, &db).await;
+                do_write(&args, &db, &table_name).await;
             }
 
             if !args.skip_read {
-                do_query(args.iter_num, &db).await;
+                do_query(args.iter_num, &db, &table_name).await;
             }
         })
 }

config/datanode.example.toml

@@ -13,19 +13,19 @@ rpc_runtime_size = 8
 require_lease_before_startup = false
 
 [heartbeat]
-# Interval for sending heartbeat messages to the Metasrv in milliseconds, 3000 by default.
-interval_millis = 3000
+# Interval for sending heartbeat messages to the Metasrv, 3 seconds by default.
+interval = "3s"
 
 # Metasrv client options.
 [meta_client]
 # Metasrv address list.
 metasrv_addrs = ["127.0.0.1:3002"]
-# Heartbeat timeout in milliseconds, 500 by default.
-heartbeat_timeout_millis = 500
-# Operation timeout in milliseconds, 3000 by default.
-timeout_millis = 3000
-# Connect server timeout in milliseconds, 5000 by default.
-connect_timeout_millis = 1000
+# Heartbeat timeout, 500 milliseconds by default.
+heartbeat_timeout = "500ms"
+# Operation timeout, 3 seconds by default.
+timeout = "3s"
+# Connect server timeout, 1 second by default.
+connect_timeout = "1s"
 # `TCP_NODELAY` option for accepted connections, true by default.
 tcp_nodelay = true
 
@@ -51,7 +51,7 @@ type = "File"
 # The local file cache directory
 # cache_path = "/path/local_cache"
 # The local file cache capacity in bytes.
-# cache_capacity = "256Mib"
+# cache_capacity = "256MB"
 
 # Compaction options, see `standalone.example.toml`.
 [storage.compaction]
@@ -101,6 +101,10 @@ auto_flush_interval = "1h"
 global_write_buffer_size = "1GB"
 # Global write buffer size threshold to reject write requests (default 2G).
 global_write_buffer_reject_size = "2GB"
+# Cache size for SST metadata (default 128MB). Setting it to 0 to disable the cache.
+sst_meta_cache_size = "128MB"
+# Cache size for vectors and arrow arrays (default 512MB). Setting it to 0 to disable the cache.
+vector_cache_size = "512MB"
 
 # Log options
 # [logging]

config/frontend.example.toml

@@ -2,10 +2,10 @@
 mode = "distributed"
 
 [heartbeat]
-# Interval for sending heartbeat task to the Metasrv in milliseconds, 5000 by default.
-interval_millis = 5000
-# Interval for retry sending heartbeat task in milliseconds, 5000 by default.
-retry_interval_millis = 5000
+# Interval for sending heartbeat task to the Metasrv, 5 seconds by default.
+interval = "5s"
+# Interval for retry sending heartbeat task, 5 seconds by default.
+retry_interval = "5s"
 
 # HTTP server options, see `standalone.example.toml`.
 [http]
@@ -59,10 +59,10 @@ enable = true
 # Metasrv client options, see `datanode.example.toml`.
 [meta_client]
 metasrv_addrs = ["127.0.0.1:3002"]
-timeout_millis = 3000
+timeout = "3s"
 # DDL timeouts options.
-ddl_timeout_millis = 10000
-connect_timeout_millis = 1000
+ddl_timeout = "10s"
+connect_timeout = "1s"
 tcp_nodelay = true
 
 # Log options, see `standalone.example.toml`

config/metasrv.example.toml

@@ -32,6 +32,6 @@ retry_delay = "500ms"
 # [datanode]
 # # Datanode client options.
 # [datanode.client_options]
-# timeout_millis = 10000
-# connect_timeout_millis = 10000
+# timeout = "10s"
+# connect_timeout = "10s"
 # tcp_nodelay = true

config/standalone.example.toml

@@ -82,6 +82,8 @@ enable = true
 
 # WAL options.
 [wal]
+# WAL data directory
+# dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
 file_size = "256MB"
 # WAL purge threshold.
@@ -93,8 +95,8 @@ read_batch_size = 128
 # Whether to sync log file after every write.
 sync_write = false
 
-# Kv options.
-[kv_store]
+# Metadata storage options.
+[metadata_store]
 # Kv file size in bytes.
 file_size = "256MB"
 # Kv purge threshold.
@@ -118,7 +120,7 @@ type = "File"
 # Cache configuration for object storage such as 'S3' etc.
 # cache_path = "/path/local_cache"
 # The local file cache capacity in bytes.
-# cache_capacity = "256Mib"
+# cache_capacity = "256MB"
 
 # Compaction options.
 [storage.compaction]

docker/buildx/ubuntu/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:20.04 as builder
 
 ARG CARGO_PROFILE
 ARG FEATURES
@@ -7,6 +7,11 @@ ARG OUTPUT_DIR
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
+# Add PPA for Python 3.10.
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa -y
+
 # Install dependencies.
 RUN --mount=type=cache,target=/var/cache/apt \
     apt-get update && apt-get install -y \

docker/dev-builder/ubuntu/Dockerfile

@@ -1,8 +1,13 @@
-FROM ubuntu:22.04
+FROM ubuntu:20.04
 
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
+# Add PPA for Python 3.10.
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa -y
+
 # Install dependencies.
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     libssl-dev \
@@ -14,8 +19,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     build-essential \
     pkg-config \
     python3.10 \
-    python3.10-dev \
-    python3-pip
+    python3.10-dev
+
+# Remove Python 3.8 and install pip.
+RUN apt-get -y purge python3.8 && \
+    apt-get -y autoremove && \
+    ln -s /usr/bin/python3.10 /usr/bin/python3 && \
+    curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
 
 RUN git config --global --add safe.directory /greptimedb
 

docs/benchmarks/tsbs/v0.4.0.md

@@ -0,0 +1,61 @@
+# TSBS benchmark - v0.4.0
+
+## Environment
+
+### Local
+|        |                                    |
+| ------ | ---------------------------------- |
+| CPU    | AMD Ryzen 7 7735HS (8 core 3.2GHz) |
+| Memory | 32GB                               |
+| Disk   | SOLIDIGM SSDPFKNU010TZ             |
+| OS     | Ubuntu 22.04.2 LTS                 |
+
+### Aliyun amd64
+
+|         |                |
+| ------- | -------------- |
+| Machine | ecs.g7.4xlarge |
+| CPU     | 16 core        |
+| Memory  | 64GB           |
+| Disk    | 100G           |
+| OS      | Ubuntu  22.04  |
+
+### Aliyun arm64
+
+|         |                   |
+| ------- | ----------------- |
+| Machine | ecs.g8y.4xlarge   |
+| CPU     | 16 core           |
+| Memory  | 64GB              |
+| Disk    | 100G              |
+| OS      | Ubuntu  22.04 ARM |
+
+
+## Write performance
+
+| Environment        | Ingest rate（rows/s） |
+| ------------------ | --------------------- |
+| Local              | 365280.60             |
+| Aliyun g7.4xlarge  | 341368.72             |
+| Aliyun g8y.4xlarge | 320907.29             |
+
+
+## Query performance
+
+| Query type            | Local (ms) | Aliyun g7.4xlarge (ms) | Aliyun g8y.4xlarge (ms) |
+| --------------------- | ---------- | ---------------------- | ----------------------- |
+| cpu-max-all-1         | 50.70      | 31.46                  | 47.61                   |
+| cpu-max-all-8         | 262.16     | 129.26                 | 152.43                  |
+| double-groupby-1      | 2512.71    | 1408.19                | 1586.10                 |
+| double-groupby-5      | 3896.15    | 2304.29                | 2585.29                 |
+| double-groupby-all    | 5404.67    | 3337.61                | 3773.91                 |
+| groupby-orderby-limit | 3786.98    | 2065.72                | 2312.57                 |
+| high-cpu-1            | 71.96      | 37.29                  | 54.01                   |
+| high-cpu-all          | 9468.75    | 7595.69                | 8467.46                 |
+| lastpoint             | 13379.43   | 11253.76               | 12949.40                |
+| single-groupby-1-1-1  | 20.72      | 12.16                  | 13.35                   |
+| single-groupby-1-1-12 | 28.53      | 15.67                  | 21.62                   |
+| single-groupby-1-8-1  | 72.23      | 37.90                  | 43.52                   |
+| single-groupby-5-1-1  | 26.75      | 15.59                  | 17.48                   |
+| single-groupby-5-1-12 | 45.41      | 22.90                  | 31.96                   |
+| single-groupby-5-8-1  | 107.96     | 59.76                  | 69.58                   |

src/api/src/error.rs

@@ -28,7 +28,12 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unknown proto column datatype: {}", datatype))]
-    UnknownColumnDataType { datatype: i32, location: Location },
+    UnknownColumnDataType {
+        datatype: i32,
+        location: Location,
+        #[snafu(source)]
+        error: prost::DecodeError,
+    },
 
     #[snafu(display("Failed to create column datatype from {:?}", from))]
     IntoColumnDataType {

src/api/src/helper.rs

@@ -50,7 +50,7 @@ pub struct ColumnDataTypeWrapper(ColumnDataType);
 
 impl ColumnDataTypeWrapper {
     pub fn try_new(datatype: i32) -> Result<Self> {
-        let datatype = ColumnDataType::from_i32(datatype)
+        let datatype = ColumnDataType::try_from(datatype)
             .context(error::UnknownColumnDataTypeSnafu { datatype })?;
         Ok(Self(datatype))
     }
@@ -705,7 +705,7 @@ pub fn is_semantic_type_eq(type_value: i32, semantic_type: SemanticType) -> bool
 
 /// Returns true if the pb type value is valid.
 pub fn is_column_type_value_eq(type_value: i32, expect_type: &ConcreteDataType) -> bool {
-    let Some(column_type) = ColumnDataType::from_i32(type_value) else {
+    let Ok(column_type) = ColumnDataType::try_from(type_value) else {
         return false;
     };
 

src/catalog/Cargo.toml

@@ -30,10 +30,10 @@ futures = "0.3"
 futures-util.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
-metrics.workspace = true
 moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 partition.workspace = true
+prometheus.workspace = true
 regex.workspace = true
 serde.workspace = true
 serde_json = "1.0"

src/catalog/src/kvbackend/client.rs

@@ -25,11 +25,10 @@ use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
 use common_meta::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
     BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
-    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
-    RangeRequest, RangeResponse,
+    DeleteRangeResponse, PutRequest, PutResponse, RangeRequest, RangeResponse,
 };
 use common_meta::rpc::KeyValue;
-use common_telemetry::{debug, timer};
+use common_telemetry::debug;
 use meta_client::client::MetaClient;
 use moka::future::{Cache, CacheBuilder};
 use snafu::{OptionExt, ResultExt};
@@ -152,25 +151,11 @@ impl KvBackend for CachedMetaKvBackend {
         }
     }
 
-    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
-        let from_key = &req.from_key.clone();
-        let to_key = &req.to_key.clone();
-
-        let ret = self.kv_backend.move_value(req).await;
-
-        if ret.is_ok() {
-            self.invalidate_key(from_key).await;
-            self.invalidate_key(to_key).await;
-        }
-
-        ret
-    }
-
     async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
-        let _timer = timer!(METRIC_CATALOG_KV_GET);
+        let _timer = METRIC_CATALOG_KV_GET.start_timer();
 
         let init = async {
-            let _timer = timer!(METRIC_CATALOG_KV_REMOTE_GET);
+            let _timer = METRIC_CATALOG_KV_REMOTE_GET.start_timer();
             self.kv_backend.get(key).await.map(|val| {
                 val.with_context(|| CacheNotGetSnafu {
                     key: String::from_utf8_lossy(key),
@@ -319,14 +304,6 @@ impl KvBackend for MetaKvBackend {
             .context(ExternalSnafu)
     }
 
-    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
-        self.client
-            .move_value(req)
-            .await
-            .map_err(BoxedError::new)
-            .context(ExternalSnafu)
-    }
-
     fn as_any(&self) -> &dyn Any {
         self
     }

src/catalog/src/memory/manager.rs

@@ -17,8 +17,8 @@ use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock, Weak};
 
+use common_catalog::build_db_string;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
-use metrics::{decrement_gauge, increment_gauge};
 use snafu::OptionExt;
 use table::TableRef;
 
@@ -166,7 +166,7 @@ impl MemoryCatalogManager {
                 let arc_self = Arc::new(self.clone());
                 let catalog = arc_self.create_catalog_entry(name);
                 e.insert(catalog);
-                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
+                crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT.inc();
                 Ok(true)
             }
             Entry::Occupied(_) => Ok(false),
@@ -187,11 +187,9 @@ impl MemoryCatalogManager {
             })?;
         let result = schema.remove(&request.table_name);
         if result.is_some() {
-            decrement_gauge!(
-                crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-                1.0,
-                &[crate::metrics::db_label(&request.catalog, &request.schema)],
-            );
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT
+                .with_label_values(&[build_db_string(&request.catalog, &request.schema).as_str()])
+                .dec();
         }
         Ok(())
     }
@@ -210,7 +208,7 @@ impl MemoryCatalogManager {
         match catalog.entry(request.schema) {
             Entry::Vacant(e) => {
                 e.insert(HashMap::new());
-                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
+                crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT.inc();
                 Ok(true)
             }
             Entry::Occupied(_) => Ok(false),
@@ -238,11 +236,9 @@ impl MemoryCatalogManager {
             .fail();
         }
         schema.insert(request.table_name, request.table);
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
+        crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT
+            .with_label_values(&[build_db_string(&request.catalog, &request.schema).as_str()])
+            .inc();
         Ok(true)
     }
 

src/catalog/src/metrics.rs

@@ -12,18 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_catalog::build_db_string;
-
 pub(crate) const METRIC_DB_LABEL: &str = "db";
 
-pub(crate) const METRIC_CATALOG_MANAGER_CATALOG_COUNT: &str = "catalog.catalog_count";
-pub(crate) const METRIC_CATALOG_MANAGER_SCHEMA_COUNT: &str = "catalog.schema_count";
-pub(crate) const METRIC_CATALOG_MANAGER_TABLE_COUNT: &str = "catalog.table_count";
+use lazy_static::lazy_static;
+use prometheus::*;
 
-pub(crate) const METRIC_CATALOG_KV_REMOTE_GET: &str = "catalog.kv.get.remote";
-pub(crate) const METRIC_CATALOG_KV_GET: &str = "catalog.kv.get";
-
-#[inline]
-pub(crate) fn db_label(catalog: &str, schema: &str) -> (&'static str, String) {
-    (METRIC_DB_LABEL, build_db_string(catalog, schema))
+lazy_static! {
+    pub static ref METRIC_CATALOG_MANAGER_CATALOG_COUNT: IntGauge =
+        register_int_gauge!("catalog_catalog_count", "catalog catalog count").unwrap();
+    pub static ref METRIC_CATALOG_MANAGER_SCHEMA_COUNT: IntGauge =
+        register_int_gauge!("catalog_schema_count", "catalog schema count").unwrap();
+    pub static ref METRIC_CATALOG_MANAGER_TABLE_COUNT: IntGaugeVec = register_int_gauge_vec!(
+        "catalog_table_count",
+        "catalog table count",
+        &[METRIC_DB_LABEL]
+    )
+    .unwrap();
+    pub static ref METRIC_CATALOG_KV_REMOTE_GET: Histogram =
+        register_histogram!("catalog_kv_get_remote", "catalog kv get remote").unwrap();
+    pub static ref METRIC_CATALOG_KV_GET: Histogram =
+        register_histogram!("catalog_kv_get", "catalog kv get").unwrap();
 }

src/client/Cargo.toml

@@ -27,8 +27,10 @@ datatypes = { workspace = true }
 derive_builder.workspace = true
 enum_dispatch = "0.3"
 futures-util.workspace = true
+lazy_static.workspace = true
 moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
+prometheus.workspace = true
 prost.workspace = true
 rand.workspace = true
 session = { workspace = true }
@@ -41,11 +43,10 @@ tonic.workspace = true
 common-grpc-expr = { workspace = true }
 datanode = { workspace = true }
 derive-new = "0.5"
-prost.workspace = true
 substrait = { workspace = true }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [dev-dependencies.substrait_proto]
 package = "substrait"
-version = "0.7"
+version = "0.17"

src/client/src/client.rs

@@ -139,11 +139,19 @@ impl Client {
     }
 
     fn max_grpc_recv_message_size(&self) -> usize {
-        self.inner.channel_manager.config().max_recv_message_size
+        self.inner
+            .channel_manager
+            .config()
+            .max_recv_message_size
+            .as_bytes() as usize
     }
 
     fn max_grpc_send_message_size(&self) -> usize {
-        self.inner.channel_manager.config().max_send_message_size
+        self.inner
+            .channel_manager
+            .config()
+            .max_send_message_size
+            .as_bytes() as usize
     }
 
     pub(crate) fn make_flight_client(&self) -> Result<FlightClient> {

src/client/src/database.rs

@@ -28,7 +28,7 @@ use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_query::Output;
 use common_recordbatch::error::ExternalSnafu;
 use common_recordbatch::RecordBatchStreamAdaptor;
-use common_telemetry::{logging, timer};
+use common_telemetry::logging;
 use futures_util::StreamExt;
 use prost::Message;
 use snafu::{ensure, ResultExt};
@@ -111,12 +111,12 @@ impl Database {
     }
 
     pub async fn insert(&self, requests: InsertRequests) -> Result<u32> {
-        let _timer = timer!(metrics::METRIC_GRPC_INSERT);
+        let _timer = metrics::METRIC_GRPC_INSERT.start_timer();
         self.handle(Request::Inserts(requests)).await
     }
 
     pub async fn row_insert(&self, requests: RowInsertRequests) -> Result<u32> {
-        let _timer = timer!(metrics::METRIC_GRPC_INSERT);
+        let _timer = metrics::METRIC_GRPC_INSERT.start_timer();
         self.handle(Request::RowInserts(requests)).await
     }
 
@@ -141,7 +141,7 @@ impl Database {
     }
 
     pub async fn delete(&self, request: DeleteRequests) -> Result<u32> {
-        let _timer = timer!(metrics::METRIC_GRPC_DELETE);
+        let _timer = metrics::METRIC_GRPC_DELETE.start_timer();
         self.handle(Request::Deletes(request)).await
     }
 
@@ -167,11 +167,14 @@ impl Database {
         }
     }
 
-    pub async fn sql(&self, sql: &str) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_SQL);
+    pub async fn sql<S>(&self, sql: S) -> Result<Output>
+    where
+        S: AsRef<str>,
+    {
+        let _timer = metrics::METRIC_GRPC_SQL.start_timer();
         self.do_get(
             Request::Query(QueryRequest {
-                query: Some(Query::Sql(sql.to_string())),
+                query: Some(Query::Sql(sql.as_ref().to_string())),
             }),
             0,
         )
@@ -179,7 +182,7 @@ impl Database {
     }
 
     pub async fn logical_plan(&self, logical_plan: Vec<u8>, trace_id: u64) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_LOGICAL_PLAN);
+        let _timer = metrics::METRIC_GRPC_LOGICAL_PLAN.start_timer();
         self.do_get(
             Request::Query(QueryRequest {
                 query: Some(Query::LogicalPlan(logical_plan)),
@@ -196,7 +199,7 @@ impl Database {
         end: &str,
         step: &str,
     ) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_PROMQL_RANGE_QUERY);
+        let _timer = metrics::METRIC_GRPC_PROMQL_RANGE_QUERY.start_timer();
         self.do_get(
             Request::Query(QueryRequest {
                 query: Some(Query::PromRangeQuery(PromRangeQuery {
@@ -212,7 +215,7 @@ impl Database {
     }
 
     pub async fn create(&self, expr: CreateTableExpr) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_CREATE_TABLE);
+        let _timer = metrics::METRIC_GRPC_CREATE_TABLE.start_timer();
         self.do_get(
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::CreateTable(expr)),
@@ -223,7 +226,7 @@ impl Database {
     }
 
     pub async fn alter(&self, expr: AlterExpr) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_ALTER);
+        let _timer = metrics::METRIC_GRPC_ALTER.start_timer();
         self.do_get(
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::Alter(expr)),
@@ -234,7 +237,7 @@ impl Database {
     }
 
     pub async fn drop_table(&self, expr: DropTableExpr) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_DROP_TABLE);
+        let _timer = metrics::METRIC_GRPC_DROP_TABLE.start_timer();
         self.do_get(
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::DropTable(expr)),
@@ -245,7 +248,7 @@ impl Database {
     }
 
     pub async fn truncate_table(&self, expr: TruncateTableExpr) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_TRUNCATE_TABLE);
+        let _timer = metrics::METRIC_GRPC_TRUNCATE_TABLE.start_timer();
         self.do_get(
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::TruncateTable(expr)),
@@ -257,7 +260,7 @@ impl Database {
 
     async fn do_get(&self, request: Request, trace_id: u64) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
-        let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
+        let _timer = metrics::METRIC_GRPC_DO_GET.start_timer();
         let request = self.to_rpc_request(request, trace_id);
         let request = Ticket {
             ticket: request.encode_to_vec().into(),

src/client/src/lib.rs

@@ -26,6 +26,8 @@ use api::v1::greptime_response::Response;
 use api::v1::{AffectedRows, GreptimeResponse};
 pub use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_error::status_code::StatusCode;
+pub use common_query::Output;
+pub use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
 use snafu::OptionExt;
 
 pub use self::client::Client;

src/client/src/metrics.rs

@@ -12,15 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! client metrics
-pub const METRIC_GRPC_CREATE_TABLE: &str = "grpc.create_table";
-pub const METRIC_GRPC_PROMQL_RANGE_QUERY: &str = "grpc.promql.range_query";
-pub const METRIC_GRPC_INSERT: &str = "grpc.insert";
-pub const METRIC_GRPC_DELETE: &str = "grpc.delete";
-pub const METRIC_GRPC_SQL: &str = "grpc.sql";
-pub const METRIC_GRPC_LOGICAL_PLAN: &str = "grpc.logical_plan";
-pub const METRIC_GRPC_ALTER: &str = "grpc.alter";
-pub const METRIC_GRPC_DROP_TABLE: &str = "grpc.drop_table";
-pub const METRIC_GRPC_TRUNCATE_TABLE: &str = "grpc.truncate_table";
-pub const METRIC_GRPC_DO_GET: &str = "grpc.do_get";
-pub(crate) const METRIC_REGION_REQUEST_GRPC: &str = "grpc.region_request";
+use lazy_static::lazy_static;
+use prometheus::*;
+
+lazy_static! {
+    pub static ref METRIC_GRPC_CREATE_TABLE: Histogram =
+        register_histogram!("grpc_create_table", "grpc create table").unwrap();
+    pub static ref METRIC_GRPC_PROMQL_RANGE_QUERY: Histogram =
+        register_histogram!("grpc_promql_range_query", "grpc promql range query").unwrap();
+    pub static ref METRIC_GRPC_INSERT: Histogram =
+        register_histogram!("grpc_insert", "grpc insert").unwrap();
+    pub static ref METRIC_GRPC_DELETE: Histogram =
+        register_histogram!("grpc_delete", "grpc delete").unwrap();
+    pub static ref METRIC_GRPC_SQL: Histogram =
+        register_histogram!("grpc_sql", "grpc sql").unwrap();
+    pub static ref METRIC_GRPC_LOGICAL_PLAN: Histogram =
+        register_histogram!("grpc_logical_plan", "grpc logical plan").unwrap();
+    pub static ref METRIC_GRPC_ALTER: Histogram =
+        register_histogram!("grpc_alter", "grpc alter").unwrap();
+    pub static ref METRIC_GRPC_DROP_TABLE: Histogram =
+        register_histogram!("grpc_drop_table", "grpc drop table").unwrap();
+    pub static ref METRIC_GRPC_TRUNCATE_TABLE: Histogram =
+        register_histogram!("grpc_truncate_table", "grpc truncate table").unwrap();
+    pub static ref METRIC_GRPC_DO_GET: Histogram =
+        register_histogram!("grpc_do_get", "grpc do get").unwrap();
+    pub static ref METRIC_REGION_REQUEST_GRPC: HistogramVec = register_histogram_vec!(
+        "grpc_region_request",
+        "grpc region request",
+        &["request_type"]
+    )
+    .unwrap();
+}

src/client/src/region.rs

@@ -24,7 +24,7 @@ use common_meta::datanode_manager::{AffectedRows, Datanode};
 use common_meta::error::{self as meta_error, Result as MetaResult};
 use common_recordbatch::error::ExternalSnafu;
 use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
-use common_telemetry::{error, timer};
+use common_telemetry::error;
 use prost::Message;
 use snafu::{location, Location, OptionExt, ResultExt};
 use tokio_stream::StreamExt;
@@ -152,11 +152,9 @@ impl RegionRequester {
             .with_context(|| MissingFieldSnafu { field: "body" })?
             .as_ref()
             .to_string();
-
-        let _timer = timer!(
-            metrics::METRIC_REGION_REQUEST_GRPC,
-            &[("request_type", request_type)]
-        );
+        let _timer = metrics::METRIC_REGION_REQUEST_GRPC
+            .with_label_values(&[request_type.as_str()])
+            .start_timer();
 
         let mut client = self.client.raw_region_client()?;
 

src/cmd/Cargo.toml

@@ -10,9 +10,7 @@ name = "greptime"
 path = "src/bin/greptime.rs"
 
 [features]
-default = ["metrics-process"]
 tokio-console = ["common-telemetry/tokio-console"]
-metrics-process = ["servers/metrics-process"]
 
 [dependencies]
 anymap = "1.0.0-beta.2"
@@ -45,11 +43,11 @@ futures.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 meta-srv = { workspace = true }
-metrics.workspace = true
 mito2 = { workspace = true }
 nu-ansi-term = "0.46"
 partition = { workspace = true }
 plugins.workspace = true
+prometheus.workspace = true
 prost.workspace = true
 query = { workspace = true }
 rand.workspace = true

src/cmd/src/bin/greptime.rs

@@ -21,7 +21,11 @@ use cmd::error::Result;
 use cmd::options::{Options, TopLevelOptions};
 use cmd::{cli, datanode, frontend, metasrv, standalone};
 use common_telemetry::logging::{error, info, TracingOptions};
-use metrics::gauge;
+
+lazy_static::lazy_static! {
+    static ref APP_VERSION: prometheus::IntGaugeVec =
+        prometheus::register_int_gauge_vec!("app_version", "app version", &["short_version", "version"]).unwrap();
+}
 
 #[derive(Parser)]
 #[clap(name = "greptimedb", version = print_version())]
@@ -204,11 +208,12 @@ async fn main() -> Result<()> {
     };
 
     common_telemetry::set_panic_hook();
-    common_telemetry::init_default_metrics_recorder();
     let _guard = common_telemetry::init_global_logging(app_name, logging_opts, tracing_opts);
 
     // Report app version as gauge.
-    gauge!("app_version", 1.0, "short_version" => short_version(), "version" => full_version());
+    APP_VERSION
+        .with_label_values(&[short_version(), full_version()])
+        .inc();
 
     // Log version and argument flags.
     info!(

src/cmd/src/cli.rs

@@ -14,6 +14,7 @@
 
 mod bench;
 mod cmd;
+mod export;
 mod helper;
 mod repl;
 // TODO(weny): Removes it
@@ -27,6 +28,7 @@ use common_telemetry::logging::LoggingOptions;
 pub use repl::Repl;
 use upgrade::UpgradeCommand;
 
+use self::export::ExportCommand;
 use crate::error::Result;
 use crate::options::{Options, TopLevelOptions};
 
@@ -78,17 +80,19 @@ impl Command {
 
 #[derive(Parser)]
 enum SubCommand {
-    Attach(AttachCommand),
+    // Attach(AttachCommand),
     Upgrade(UpgradeCommand),
     Bench(BenchTableMetadataCommand),
+    Export(ExportCommand),
 }
 
 impl SubCommand {
     async fn build(self) -> Result<Instance> {
         match self {
-            SubCommand::Attach(cmd) => cmd.build().await,
+            // SubCommand::Attach(cmd) => cmd.build().await,
             SubCommand::Upgrade(cmd) => cmd.build().await,
             SubCommand::Bench(cmd) => cmd.build().await,
+            SubCommand::Export(cmd) => cmd.build().await,
         }
     }
 }
@@ -104,51 +108,9 @@ pub(crate) struct AttachCommand {
 }
 
 impl AttachCommand {
+    #[allow(dead_code)]
     async fn build(self) -> Result<Instance> {
         let repl = Repl::try_new(&self).await?;
         Ok(Instance::Repl(repl))
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_load_options() {
-        let cmd = Command {
-            cmd: SubCommand::Attach(AttachCommand {
-                grpc_addr: String::from(""),
-                meta_addr: None,
-                disable_helper: false,
-            }),
-        };
-
-        let opts = cmd.load_options(TopLevelOptions::default()).unwrap();
-        let logging_opts = opts.logging_options();
-        assert_eq!("/tmp/greptimedb/logs", logging_opts.dir);
-        assert!(logging_opts.level.is_none());
-        assert!(!logging_opts.enable_jaeger_tracing);
-    }
-
-    #[test]
-    fn test_top_level_options() {
-        let cmd = Command {
-            cmd: SubCommand::Attach(AttachCommand {
-                grpc_addr: String::from(""),
-                meta_addr: None,
-                disable_helper: false,
-            }),
-        };
-
-        let opts = cmd
-            .load_options(TopLevelOptions {
-                log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
-                log_level: Some("debug".to_string()),
-            })
-            .unwrap();
-        let logging_opts = opts.logging_options();
-        assert_eq!("/tmp/greptimedb/test/logs", logging_opts.dir);
-        assert_eq!("debug", logging_opts.level.as_ref().unwrap());
-    }
-}

src/cmd/src/cli/bench.rs

@@ -157,6 +157,7 @@ fn create_region_routes() -> Vec<RegionRoute> {
                 addr: String::new(),
             }),
             follower_peers: vec![],
+            leader_status: None,
         });
     }
 

src/cmd/src/cli/export.rs

@@ -0,0 +1,395 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::Path;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use clap::{Parser, ValueEnum};
+use client::{Client, Database, DEFAULT_SCHEMA_NAME};
+use common_query::Output;
+use common_recordbatch::util::collect;
+use common_telemetry::{debug, error, info, warn};
+use datatypes::scalars::ScalarVector;
+use datatypes::vectors::{StringVector, Vector};
+use snafu::{OptionExt, ResultExt};
+use tokio::fs::File;
+use tokio::io::AsyncWriteExt;
+use tokio::sync::Semaphore;
+
+use crate::cli::{Instance, Tool};
+use crate::error::{
+    CollectRecordBatchesSnafu, ConnectServerSnafu, EmptyResultSnafu, Error, FileIoSnafu,
+    InvalidDatabaseNameSnafu, NotDataFromOutputSnafu, RequestDatabaseSnafu, Result,
+};
+
+type TableReference = (String, String, String);
+
+#[derive(Debug, Default, Clone, ValueEnum)]
+enum ExportTarget {
+    /// Corresponding to `SHOW CREATE TABLE`
+    #[default]
+    CreateTable,
+    /// Corresponding to `EXPORT TABLE`
+    TableData,
+}
+
+#[derive(Debug, Default, Parser)]
+pub struct ExportCommand {
+    /// Server address to connect
+    #[clap(long)]
+    addr: String,
+
+    /// Directory to put the exported data. E.g.: /tmp/greptimedb-export
+    #[clap(long)]
+    output_dir: String,
+
+    /// The name of the catalog to export. Default to "greptime-*"".
+    #[clap(long, default_value = "")]
+    database: String,
+
+    /// Parallelism of the export.
+    #[clap(long, short = 'j', default_value = "1")]
+    export_jobs: usize,
+
+    /// Max retry times for each job.
+    #[clap(long, default_value = "3")]
+    max_retry: usize,
+
+    /// Things to export
+    #[clap(long, short = 't', value_enum)]
+    target: ExportTarget,
+}
+
+impl ExportCommand {
+    pub async fn build(&self) -> Result<Instance> {
+        let client = Client::with_urls([self.addr.clone()]);
+        client
+            .health_check()
+            .await
+            .with_context(|_| ConnectServerSnafu {
+                addr: self.addr.clone(),
+            })?;
+        let (catalog, schema) = split_database(&self.database)?;
+        let database_client = Database::new(
+            catalog.clone(),
+            schema.clone().unwrap_or(DEFAULT_SCHEMA_NAME.to_string()),
+            client,
+        );
+
+        Ok(Instance::Tool(Box::new(Export {
+            client: database_client,
+            catalog,
+            schema,
+            output_dir: self.output_dir.clone(),
+            parallelism: self.export_jobs,
+            target: self.target.clone(),
+        })))
+    }
+}
+
+pub struct Export {
+    client: Database,
+    catalog: String,
+    schema: Option<String>,
+    output_dir: String,
+    parallelism: usize,
+    target: ExportTarget,
+}
+
+impl Export {
+    /// Iterate over all db names.
+    ///
+    /// Newbie: `db_name` is catalog + schema.
+    async fn iter_db_names(&self) -> Result<Vec<(String, String)>> {
+        if let Some(schema) = &self.schema {
+            Ok(vec![(self.catalog.clone(), schema.clone())])
+        } else {
+            let mut client = self.client.clone();
+            client.set_catalog(self.catalog.clone());
+            let result =
+                client
+                    .sql("show databases")
+                    .await
+                    .with_context(|_| RequestDatabaseSnafu {
+                        sql: "show databases".to_string(),
+                    })?;
+            let Output::Stream(stream) = result else {
+                NotDataFromOutputSnafu.fail()?
+            };
+            let record_batch = collect(stream)
+                .await
+                .context(CollectRecordBatchesSnafu)?
+                .pop()
+                .context(EmptyResultSnafu)?;
+            let schemas = record_batch
+                .column(0)
+                .as_any()
+                .downcast_ref::<StringVector>()
+                .unwrap();
+            let mut result = Vec::with_capacity(schemas.len());
+            for i in 0..schemas.len() {
+                let schema = schemas.get_data(i).unwrap().to_owned();
+                result.push((self.catalog.clone(), schema));
+            }
+            Ok(result)
+        }
+    }
+
+    /// Return a list of [`TableReference`] to be exported.
+    /// Includes all tables under the given `catalog` and `schema`
+    async fn get_table_list(&self, catalog: &str, schema: &str) -> Result<Vec<TableReference>> {
+        // TODO: SQL injection hurts
+        let sql = format!(
+            "select table_catalog, table_schema, table_name from \
+            information_schema.tables where table_type = \'BASE TABLE\'\
+            and table_catalog = \'{catalog}\' and table_schema = \'{schema}\'",
+        );
+        let mut client = self.client.clone();
+        client.set_catalog(catalog);
+        client.set_schema(schema);
+        let result = client
+            .sql(&sql)
+            .await
+            .with_context(|_| RequestDatabaseSnafu { sql })?;
+        let Output::Stream(stream) = result else {
+            NotDataFromOutputSnafu.fail()?
+        };
+        let Some(record_batch) = collect(stream)
+            .await
+            .context(CollectRecordBatchesSnafu)?
+            .pop()
+        else {
+            return Ok(vec![]);
+        };
+
+        debug!("Fetched table list: {}", record_batch.pretty_print());
+
+        if record_batch.num_rows() == 0 {
+            return Ok(vec![]);
+        }
+
+        let mut result = Vec::with_capacity(record_batch.num_rows());
+        let catalog_column = record_batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StringVector>()
+            .unwrap();
+        let schema_column = record_batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<StringVector>()
+            .unwrap();
+        let table_column = record_batch
+            .column(2)
+            .as_any()
+            .downcast_ref::<StringVector>()
+            .unwrap();
+        for i in 0..record_batch.num_rows() {
+            let catalog = catalog_column.get_data(i).unwrap().to_owned();
+            let schema = schema_column.get_data(i).unwrap().to_owned();
+            let table = table_column.get_data(i).unwrap().to_owned();
+            result.push((catalog, schema, table));
+        }
+
+        Ok(result)
+    }
+
+    async fn show_create_table(&self, catalog: &str, schema: &str, table: &str) -> Result<String> {
+        let sql = format!("show create table {}.{}.{}", catalog, schema, table);
+        let mut client = self.client.clone();
+        client.set_catalog(catalog);
+        client.set_schema(schema);
+        let result = client
+            .sql(&sql)
+            .await
+            .with_context(|_| RequestDatabaseSnafu { sql })?;
+        let Output::Stream(stream) = result else {
+            NotDataFromOutputSnafu.fail()?
+        };
+        let record_batch = collect(stream)
+            .await
+            .context(CollectRecordBatchesSnafu)?
+            .pop()
+            .context(EmptyResultSnafu)?;
+        let create_table = record_batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<StringVector>()
+            .unwrap()
+            .get_data(0)
+            .unwrap();
+
+        Ok(format!("{create_table};\n"))
+    }
+
+    async fn export_create_table(&self) -> Result<()> {
+        let semaphore = Arc::new(Semaphore::new(self.parallelism));
+        let db_names = self.iter_db_names().await?;
+        let db_count = db_names.len();
+        let mut tasks = Vec::with_capacity(db_names.len());
+        for (catalog, schema) in db_names {
+            let semaphore_moved = semaphore.clone();
+            tasks.push(async move {
+                let _permit = semaphore_moved.acquire().await.unwrap();
+                let table_list = self.get_table_list(&catalog, &schema).await?;
+                let table_count = table_list.len();
+                tokio::fs::create_dir_all(&self.output_dir)
+                    .await
+                    .context(FileIoSnafu)?;
+                let output_file =
+                    Path::new(&self.output_dir).join(format!("{catalog}-{schema}.sql"));
+                let mut file = File::create(output_file).await.context(FileIoSnafu)?;
+                for (c, s, t) in table_list {
+                    match self.show_create_table(&c, &s, &t).await {
+                        Err(e) => {
+                            error!(e; "Failed to export table {}.{}.{}", c, s, t)
+                        }
+                        Ok(create_table) => {
+                            file.write_all(create_table.as_bytes())
+                                .await
+                                .context(FileIoSnafu)?;
+                        }
+                    }
+                }
+                info!("finished exporting {catalog}.{schema} with {table_count} tables",);
+                Ok::<(), Error>(())
+            });
+        }
+
+        let success = futures::future::join_all(tasks)
+            .await
+            .into_iter()
+            .filter(|r| match r {
+                Ok(_) => true,
+                Err(e) => {
+                    error!(e; "export job failed");
+                    false
+                }
+            })
+            .count();
+
+        info!("success {success}/{db_count} jobs");
+
+        Ok(())
+    }
+
+    async fn export_table_data(&self) -> Result<()> {
+        let semaphore = Arc::new(Semaphore::new(self.parallelism));
+        let db_names = self.iter_db_names().await?;
+        let db_count = db_names.len();
+        let mut tasks = Vec::with_capacity(db_names.len());
+        for (catalog, schema) in db_names {
+            let semaphore_moved = semaphore.clone();
+            tasks.push(async move {
+                let _permit = semaphore_moved.acquire().await.unwrap();
+                tokio::fs::create_dir_all(&self.output_dir)
+                    .await
+                    .context(FileIoSnafu)?;
+                let output_dir = Path::new(&self.output_dir).join(format!("{catalog}-{schema}/"));
+
+                let mut client = self.client.clone();
+                client.set_catalog(catalog.clone());
+                client.set_schema(schema.clone());
+
+                // copy database to
+                let sql = format!(
+                    "copy database {} to '{}' with (format='parquet');",
+                    schema,
+                    output_dir.to_str().unwrap()
+                );
+                client
+                    .sql(sql.clone())
+                    .await
+                    .context(RequestDatabaseSnafu { sql })?;
+                info!("finished exporting {catalog}.{schema} data");
+
+                // export copy from sql
+                let dir_filenames = match output_dir.read_dir() {
+                    Ok(dir) => dir,
+                    Err(_) => {
+                        warn!("empty database {catalog}.{schema}");
+                        return Ok(());
+                    }
+                };
+
+                let copy_from_file =
+                    Path::new(&self.output_dir).join(format!("{catalog}-{schema}_copy_from.sql"));
+                let mut file = File::create(copy_from_file).await.context(FileIoSnafu)?;
+
+                let copy_from_sql = dir_filenames
+                    .into_iter()
+                    .map(|file| {
+                        let file = file.unwrap();
+                        let filename = file.file_name().into_string().unwrap();
+
+                        format!(
+                            "copy {} from '{}' with (format='parquet');\n",
+                            filename.replace(".parquet", ""),
+                            file.path().to_str().unwrap()
+                        )
+                    })
+                    .collect::<Vec<_>>()
+                    .join("");
+                file.write_all(copy_from_sql.as_bytes())
+                    .await
+                    .context(FileIoSnafu)?;
+
+                info!("finished exporting {catalog}.{schema} copy_from.sql");
+
+                Ok::<(), Error>(())
+            });
+        }
+
+        let success = futures::future::join_all(tasks)
+            .await
+            .into_iter()
+            .filter(|r| match r {
+                Ok(_) => true,
+                Err(e) => {
+                    error!(e; "export job failed");
+                    false
+                }
+            })
+            .count();
+
+        info!("success {success}/{db_count} jobs");
+
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl Tool for Export {
+    async fn do_work(&self) -> Result<()> {
+        match self.target {
+            ExportTarget::CreateTable => self.export_create_table().await,
+            ExportTarget::TableData => self.export_table_data().await,
+        }
+    }
+}
+
+/// Split at `-`.
+fn split_database(database: &str) -> Result<(String, Option<String>)> {
+    let (catalog, schema) = database
+        .split_once('-')
+        .with_context(|| InvalidDatabaseNameSnafu {
+            database: database.to_string(),
+        })?;
+    if schema == "*" {
+        Ok((catalog.to_string(), None))
+    } else {
+        Ok((catalog.to_string(), Some(schema.to_string())))
+    }
+}

src/cmd/src/datanode.rs

@@ -96,6 +96,8 @@ struct StartCommand {
     #[clap(long)]
     data_home: Option<String>,
     #[clap(long)]
+    wal_dir: Option<String>,
+    #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
     http_timeout: Option<u64>,
@@ -149,6 +151,10 @@ impl StartCommand {
             opts.storage.data_home = data_home.clone();
         }
 
+        if let Some(wal_dir) = &self.wal_dir {
+            opts.wal.dir = Some(wal_dir.clone());
+        }
+
         if let Some(http_addr) = &self.http_addr {
             opts.http.addr = http_addr.clone();
         }
@@ -188,6 +194,7 @@ mod tests {
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
     use datanode::config::{CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig};
+    use servers::heartbeat_options::HeartbeatOptions;
     use servers::Mode;
 
     use super::*;
@@ -204,11 +211,14 @@ mod tests {
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
 
+            [heartbeat]
+            interval = "300ms"
+
             [meta_client]
             metasrv_addrs = ["127.0.0.1:3002"]
-            timeout_millis = 3000
-            connect_timeout_millis = 5000
-            ddl_timeout_millis= 10000
+            timeout = "3s"
+            connect_timeout = "5s"
+            ddl_timeout = "10s"
             tcp_nodelay = true
 
             [wal]
@@ -251,25 +261,33 @@ mod tests {
 
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
+        assert_eq!("/other/wal", options.wal.dir.unwrap());
 
         assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
         assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
         assert_eq!(1024 * 1024 * 1024 * 50, options.wal.purge_threshold.0);
         assert!(!options.wal.sync_write);
 
+        let HeartbeatOptions {
+            interval: heart_beat_interval,
+            ..
+        } = options.heartbeat;
+
+        assert_eq!(300, heart_beat_interval.as_millis());
+
         let MetaClientOptions {
             metasrv_addrs: metasrv_addr,
-            timeout_millis,
-            connect_timeout_millis,
+            timeout,
+            connect_timeout,
+            ddl_timeout,
             tcp_nodelay,
-            ddl_timeout_millis,
             ..
         } = options.meta_client.unwrap();
 
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
-        assert_eq!(5000, connect_timeout_millis);
-        assert_eq!(10000, ddl_timeout_millis);
-        assert_eq!(3000, timeout_millis);
+        assert_eq!(5000, connect_timeout.as_millis());
+        assert_eq!(10000, ddl_timeout.as_millis());
+        assert_eq!(3000, timeout.as_millis());
         assert!(tcp_nodelay);
         assert_eq!("/tmp/greptimedb/", options.storage.data_home);
         assert!(matches!(
@@ -363,8 +381,8 @@ mod tests {
             rpc_runtime_size = 8
 
             [meta_client]
-            timeout_millis = 3000
-            connect_timeout_millis = 5000
+            timeout = "3s"
+            connect_timeout = "5s"
             tcp_nodelay = true
 
             [wal]
@@ -428,6 +446,7 @@ mod tests {
             || {
                 let command = StartCommand {
                     config_file: Some(file.path().to_str().unwrap().to_string()),
+                    wal_dir: Some("/other/wal/dir".to_string()),
                     env_prefix: env_prefix.to_string(),
                     ..Default::default()
                 };
@@ -455,6 +474,9 @@ mod tests {
                 // Should be read from config file, config file > env > default values.
                 assert_eq!(opts.storage.compaction.max_purge_tasks, 32);
 
+                // Should be read from cli, cli > config file > env > default values.
+                assert_eq!(opts.wal.dir.unwrap(), "/other/wal/dir");
+
                 // Should be default value.
                 assert_eq!(
                     opts.storage.manifest.checkpoint_margin,

src/cmd/src/error.rs

@@ -37,6 +37,18 @@ pub enum Error {
         source: common_meta::error::Error,
     },
 
+    #[snafu(display("Failed to start procedure manager"))]
+    StartProcedureManager {
+        location: Location,
+        source: common_procedure::error::Error,
+    },
+
+    #[snafu(display("Failed to stop procedure manager"))]
+    StopProcedureManager {
+        location: Location,
+        source: common_procedure::error::Error,
+    },
+
     #[snafu(display("Failed to start datanode"))]
     StartDatanode {
         location: Location,
@@ -174,12 +186,45 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to connect server at {addr}"))]
+    ConnectServer {
+        addr: String,
+        source: client::error::Error,
+        location: Location,
+    },
+
     #[snafu(display("Failed to serde json"))]
     SerdeJson {
         #[snafu(source)]
         error: serde_json::error::Error,
         location: Location,
     },
+
+    #[snafu(display("Expect data from output, but got another thing"))]
+    NotDataFromOutput { location: Location },
+
+    #[snafu(display("Empty result from output"))]
+    EmptyResult { location: Location },
+
+    #[snafu(display("Failed to manipulate file"))]
+    FileIo {
+        location: Location,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
+    #[snafu(display("Invalid database name: {}", database))]
+    InvalidDatabaseName {
+        location: Location,
+        database: String,
+    },
+
+    #[snafu(display("Failed to create directory {}", dir))]
+    CreateDir {
+        dir: String,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -198,12 +243,18 @@ impl ErrorExt for Error {
             Error::IterStream { source, .. } | Error::InitMetadata { source, .. } => {
                 source.status_code()
             }
+            Error::ConnectServer { source, .. } => source.status_code(),
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
             | Error::InvalidReplCommand { .. }
-            | Error::ConnectEtcd { .. } => StatusCode::InvalidArguments,
-
+            | Error::ConnectEtcd { .. }
+            | Error::NotDataFromOutput { .. }
+            | Error::CreateDir { .. }
+            | Error::EmptyResult { .. }
+            | Error::InvalidDatabaseName { .. } => StatusCode::InvalidArguments,
+            Error::StartProcedureManager { source, .. }
+            | Error::StopProcedureManager { source, .. } => source.status_code(),
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
             Error::RequestDatabase { source, .. } => source.status_code(),
             Error::CollectRecordBatches { source, .. }
@@ -215,7 +266,7 @@ impl ErrorExt for Error {
             Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
             Error::StartCatalogManager { source, .. } => source.status_code(),
 
-            Error::SerdeJson { .. } => StatusCode::Unexpected,
+            Error::SerdeJson { .. } | Error::FileIo { .. } => StatusCode::Unexpected,
         }
     }
 

src/cmd/src/frontend.rs

@@ -89,7 +89,7 @@ pub struct StartCommand {
     #[clap(long)]
     http_timeout: Option<u64>,
     #[clap(long)]
-    grpc_addr: Option<String>,
+    rpc_addr: Option<String>,
     #[clap(long)]
     mysql_addr: Option<String>,
     #[clap(long)]
@@ -150,7 +150,7 @@ impl StartCommand {
             opts.http.disable_dashboard = disable_dashboard;
         }
 
-        if let Some(addr) = &self.grpc_addr {
+        if let Some(addr) = &self.rpc_addr {
             opts.grpc.addr = addr.clone()
         }
 
@@ -188,6 +188,7 @@ impl StartCommand {
     }
 
     async fn build(self, mut opts: FrontendOptions) -> Result<Instance> {
+        #[allow(clippy::unnecessary_mut_passed)]
         let plugins = plugins::setup_frontend_plugins(&mut opts)
             .await
             .context(StartFrontendSnafu)?;
@@ -312,6 +313,7 @@ mod tests {
             ..Default::default()
         };
 
+        #[allow(clippy::unnecessary_mut_passed)]
         let plugins = plugins::setup_frontend_plugins(&mut fe_opts).await.unwrap();
 
         let provider = plugins.get::<UserProviderRef>().unwrap();
@@ -353,8 +355,8 @@ mod tests {
             addr = "127.0.0.1:4000"
 
             [meta_client]
-            timeout_millis = 3000
-            connect_timeout_millis = 5000
+            timeout = "3s"
+            connect_timeout = "5s"
             tcp_nodelay = true
 
             [mysql]

src/cmd/src/options.rs

@@ -30,7 +30,7 @@ pub const ENV_LIST_SEP: &str = ",";
 pub struct MixOptions {
     pub data_home: String,
     pub procedure: ProcedureConfig,
-    pub kv_store: KvStoreConfig,
+    pub metadata_store: KvStoreConfig,
     pub frontend: FrontendOptions,
     pub datanode: DatanodeOptions,
     pub logging: LoggingOptions,
@@ -144,8 +144,8 @@ mod tests {
             mysql_runtime_size = 2
 
             [meta_client]
-            timeout_millis = 3000
-            connect_timeout_millis = 5000
+            timeout = "3s"
+            connect_timeout = "5s"
             tcp_nodelay = true
 
             [wal]
@@ -263,6 +263,9 @@ mod tests {
                     ]
                 );
 
+                // Should be the values from config file, not environment variables.
+                assert_eq!(opts.wal.dir.unwrap(), "/tmp/greptimedb/wal");
+
                 // Should be default values.
                 assert_eq!(opts.node_id, None);
             },

src/cmd/src/standalone.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::sync::Arc;
+use std::{fs, path};
 
 use catalog::kvbackend::KvBackendCatalogManager;
 use catalog::CatalogManagerRef;
@@ -41,8 +42,9 @@ use servers::Mode;
 use snafu::ResultExt;
 
 use crate::error::{
-    IllegalConfigSnafu, InitMetadataSnafu, Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu,
-    StartDatanodeSnafu, StartFrontendSnafu,
+    CreateDirSnafu, IllegalConfigSnafu, InitMetadataSnafu, Result, ShutdownDatanodeSnafu,
+    ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
+    StopProcedureManagerSnafu,
 };
 use crate::options::{MixOptions, Options, TopLevelOptions};
 
@@ -95,9 +97,10 @@ pub struct StandaloneOptions {
     pub prom_store: PromStoreOptions,
     pub wal: WalConfig,
     pub storage: StorageConfig,
-    pub kv_store: KvStoreConfig,
+    pub metadata_store: KvStoreConfig,
     pub procedure: ProcedureConfig,
     pub logging: LoggingOptions,
+    pub user_provider: Option<String>,
     /// Options for different store engines.
     pub region_engine: Vec<RegionEngineConfig>,
 }
@@ -116,9 +119,10 @@ impl Default for StandaloneOptions {
             prom_store: PromStoreOptions::default(),
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
-            kv_store: KvStoreConfig::default(),
+            metadata_store: KvStoreConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
+            user_provider: None,
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig::default()),
                 RegionEngineConfig::File(FileEngineConfig::default()),
@@ -140,6 +144,7 @@ impl StandaloneOptions {
             prom_store: self.prom_store,
             meta_client: None,
             logging: self.logging,
+            user_provider: self.user_provider,
             ..Default::default()
         }
     }
@@ -159,6 +164,7 @@ impl StandaloneOptions {
 pub struct Instance {
     datanode: Datanode,
     frontend: FeInstance,
+    procedure_manager: ProcedureManagerRef,
 }
 
 impl Instance {
@@ -167,6 +173,11 @@ impl Instance {
         self.datanode.start().await.context(StartDatanodeSnafu)?;
         info!("Datanode instance started");
 
+        self.procedure_manager
+            .start()
+            .await
+            .context(StartProcedureManagerSnafu)?;
+
         self.frontend.start().await.context(StartFrontendSnafu)?;
         Ok(())
     }
@@ -177,6 +188,11 @@ impl Instance {
             .await
             .context(ShutdownFrontendSnafu)?;
 
+        self.procedure_manager
+            .stop()
+            .await
+            .context(StopProcedureManagerSnafu)?;
+
         self.datanode
             .shutdown()
             .await
@@ -277,7 +293,10 @@ impl StartCommand {
         if self.influxdb_enable {
             opts.influxdb.enable = self.influxdb_enable;
         }
-        let kv_store = opts.kv_store.clone();
+
+        opts.user_provider = self.user_provider.clone();
+
+        let metadata_store = opts.metadata_store.clone();
         let procedure = opts.procedure.clone();
         let frontend = opts.clone().frontend_options();
         let logging = opts.logging.clone();
@@ -285,7 +304,7 @@ impl StartCommand {
 
         Ok(Options::Standalone(Box::new(MixOptions {
             procedure,
-            kv_store,
+            metadata_store,
             data_home: datanode.storage.data_home.to_string(),
             frontend,
             datanode,
@@ -298,6 +317,7 @@ impl StartCommand {
     #[allow(clippy::diverging_sub_expression)]
     async fn build(self, opts: MixOptions) -> Result<Instance> {
         let mut fe_opts = opts.frontend;
+        #[allow(clippy::unnecessary_mut_passed)]
         let fe_plugins = plugins::setup_frontend_plugins(&mut fe_opts)
             .await
             .context(StartFrontendSnafu)?;
@@ -310,10 +330,15 @@ impl StartCommand {
             fe_opts, dn_opts
         );
 
+        // Ensure the data_home directory exists.
+        fs::create_dir_all(path::Path::new(&opts.data_home)).context(CreateDirSnafu {
+            dir: &opts.data_home,
+        })?;
+
         let metadata_dir = metadata_store_dir(&opts.data_home);
         let (kv_store, procedure_manager) = FeInstance::try_build_standalone_components(
             metadata_dir,
-            opts.kv_store,
+            opts.metadata_store,
             opts.procedure,
         )
         .await
@@ -342,7 +367,7 @@ impl StartCommand {
         let mut frontend = build_frontend(
             fe_plugins,
             kv_store,
-            procedure_manager,
+            procedure_manager.clone(),
             catalog_manager,
             region_server,
         )
@@ -353,7 +378,11 @@ impl StartCommand {
             .await
             .context(StartFrontendSnafu)?;
 
-        Ok(Instance { datanode, frontend })
+        Ok(Instance {
+            datanode,
+            frontend,
+            procedure_manager,
+        })
     }
 }
 
@@ -398,6 +427,7 @@ mod tests {
             ..Default::default()
         };
 
+        #[allow(clippy::unnecessary_mut_passed)]
         let plugins = plugins::setup_frontend_plugins(&mut fe_opts).await.unwrap();
 
         let provider = plugins.get::<UserProviderRef>().unwrap();
@@ -481,6 +511,8 @@ mod tests {
         assert_eq!(None, fe_opts.mysql.reject_no_database);
         assert!(fe_opts.influxdb.enable);
 
+        assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap());
+
         match &dn_opts.storage.store {
             datanode::config::ObjectStoreConfig::S3(s3_config) => {
                 assert_eq!(
@@ -614,7 +646,7 @@ mod tests {
         assert_eq!(options.influxdb, default_options.influxdb);
         assert_eq!(options.prom_store, default_options.prom_store);
         assert_eq!(options.wal, default_options.wal);
-        assert_eq!(options.kv_store, default_options.kv_store);
+        assert_eq!(options.metadata_store, default_options.metadata_store);
         assert_eq!(options.procedure, default_options.procedure);
         assert_eq!(options.logging, default_options.logging);
         assert_eq!(options.region_engine, default_options.region_engine);

src/common/base/src/readable_size.rs

@@ -14,10 +14,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This file is copied from https://github.com/tikv/raft-engine/blob/8dd2a39f359ff16f5295f35343f626e0c10132fa/src/util.rs without any modification.
+// This file is copied from https://github.com/tikv/raft-engine/blob/8dd2a39f359ff16f5295f35343f626e0c10132fa/src/util.rs
 
-use std::fmt;
-use std::fmt::{Display, Write};
+use std::fmt::{self, Debug, Display, Write};
 use std::ops::{Div, Mul};
 use std::str::FromStr;
 
@@ -34,7 +33,7 @@ pub const GIB: u64 = MIB * BINARY_DATA_MAGNITUDE;
 pub const TIB: u64 = GIB * BINARY_DATA_MAGNITUDE;
 pub const PIB: u64 = TIB * BINARY_DATA_MAGNITUDE;
 
-#[derive(Clone, Debug, Copy, PartialEq, Eq, PartialOrd)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd)]
 pub struct ReadableSize(pub u64);
 
 impl ReadableSize {
@@ -155,6 +154,12 @@ impl FromStr for ReadableSize {
     }
 }
 
+impl Debug for ReadableSize {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self)
+    }
+}
+
 impl Display for ReadableSize {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         if self.0 >= PIB {

src/common/config/src/lib.rs

@@ -20,6 +20,8 @@ use serde::{Deserialize, Serialize};
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
 pub struct WalConfig {
+    // wal directory
+    pub dir: Option<String>,
     // wal file size in bytes
     pub file_size: ReadableSize,
     // wal purge threshold in bytes
@@ -36,7 +38,8 @@ pub struct WalConfig {
 impl Default for WalConfig {
     fn default() -> Self {
         Self {
-            file_size: ReadableSize::mb(256),     // log file size 256MB
+            dir: None,
+            file_size: ReadableSize::mb(256), // log file size 256MB
             purge_threshold: ReadableSize::gb(4), // purge threshold 4GB
             purge_interval: Duration::from_secs(600),
             read_batch_size: 128,

src/common/error/src/ext.rs

@@ -39,17 +39,25 @@ pub trait ErrorExt: StackError {
     where
         Self: Sized,
     {
-        let error = self.last();
-        if let Some(external_error) = error.source() {
-            let external_root = external_error.sources().last().unwrap();
-
-            if error.to_string().is_empty() {
-                format!("{external_root}")
-            } else {
-                format!("{error}: {external_root}")
+        match self.status_code() {
+            StatusCode::Unknown | StatusCode::Internal => {
+                // masks internal error from end user
+                format!("Internal error: {}", self.status_code() as u32)
+            }
+            _ => {
+                let error = self.last();
+                if let Some(external_error) = error.source() {
+                    let external_root = external_error.sources().last().unwrap();
+
+                    if error.to_string().is_empty() {
+                        format!("{external_root}")
+                    } else {
+                        format!("{error}: {external_root}")
+                    }
+                } else {
+                    format!("{error}")
+                }
             }
-        } else {
-            format!("{error}")
         }
     }
 }

src/common/error/src/status_code.rs

@@ -14,10 +14,10 @@
 
 use std::fmt;
 
-use strum::EnumString;
+use strum::{AsRefStr, EnumString};
 
 /// Common status code for public API.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumString)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumString, AsRefStr)]
 pub enum StatusCode {
     // ====== Begin of common status code ==============
     /// Success.

src/common/function/src/scalars/math.rs

@@ -58,9 +58,15 @@ impl Function for RangeFunction {
         "range_fn"
     }
 
-    // range_fn will never been used, return_type could be arbitrary value, is not important
-    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::float64_datatype())
+    // The first argument to range_fn is the expression to be evaluated
+    fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        input_types
+            .first()
+            .cloned()
+            .ok_or(DataFusionError::Internal(
+                "No expr found in range_fn".into(),
+            ))
+            .context(GeneralDataFusionSnafu)
     }
 
     /// `range_fn` will never been used. As long as a legal signature is returned, the specific content of the signature does not matter.

src/common/function/src/scalars/math/rate.rs

@@ -16,7 +16,8 @@ use std::fmt;
 
 use common_query::error::{self, Result};
 use common_query::prelude::{Signature, Volatility};
-use datatypes::arrow::compute::kernels::{arithmetic, cast};
+use datafusion::arrow::compute::kernels::numeric;
+use datatypes::arrow::compute::kernels::cast;
 use datatypes::arrow::datatypes::DataType;
 use datatypes::prelude::*;
 use datatypes::vectors::{Helper, VectorRef};
@@ -51,11 +52,11 @@ impl Function for RateFunction {
         let val = &columns[0].to_arrow_array();
         let val_0 = val.slice(0, val.len() - 1);
         let val_1 = val.slice(1, val.len() - 1);
-        let dv = arithmetic::subtract_dyn(&val_1, &val_0).context(error::ArrowComputeSnafu)?;
+        let dv = numeric::sub(&val_1, &val_0).context(error::ArrowComputeSnafu)?;
         let ts = &columns[1].to_arrow_array();
         let ts_0 = ts.slice(0, ts.len() - 1);
         let ts_1 = ts.slice(1, ts.len() - 1);
-        let dt = arithmetic::subtract_dyn(&ts_1, &ts_0).context(error::ArrowComputeSnafu)?;
+        let dt = numeric::sub(&ts_1, &ts_0).context(error::ArrowComputeSnafu)?;
 
         let dv = cast::cast(&dv, &DataType::Float64).context(error::TypeCastSnafu {
             typ: DataType::Float64,
@@ -63,7 +64,7 @@ impl Function for RateFunction {
         let dt = cast::cast(&dt, &DataType::Float64).context(error::TypeCastSnafu {
             typ: DataType::Float64,
         })?;
-        let rate = arithmetic::divide_dyn(&dv, &dt).context(error::ArrowComputeSnafu)?;
+        let rate = numeric::div(&dv, &dt).context(error::ArrowComputeSnafu)?;
         let v = Helper::try_into_vector(&rate).context(error::FromArrowArraySnafu)?;
 
         Ok(v)

src/common/function/src/scalars/timestamp/greatest.rs

@@ -18,9 +18,9 @@ use common_query::error::{
     self, ArrowComputeSnafu, InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu,
 };
 use common_query::prelude::{Signature, Volatility};
+use datafusion::arrow::compute::kernels::cmp::gt;
 use datatypes::arrow::array::AsArray;
 use datatypes::arrow::compute::cast;
-use datatypes::arrow::compute::kernels::comparison::gt_dyn;
 use datatypes::arrow::compute::kernels::zip;
 use datatypes::arrow::datatypes::{DataType as ArrowDataType, Date32Type};
 use datatypes::prelude::ConcreteDataType;
@@ -72,7 +72,7 @@ impl Function for GreatestFunction {
                 let column2 = cast(&columns[1].to_arrow_array(), &ArrowDataType::Date32)
                     .context(ArrowComputeSnafu)?;
                 let column2 = column2.as_primitive::<Date32Type>();
-                let boolean_array = gt_dyn(&column1, &column2).context(ArrowComputeSnafu)?;
+                let boolean_array = gt(&column1, &column2).context(ArrowComputeSnafu)?;
                 let result =
                     zip::zip(&boolean_array, &column1, &column2).context(ArrowComputeSnafu)?;
                 Ok(Helper::try_into_vector(&result).context(error::FromArrowArraySnafu)?)
@@ -82,7 +82,7 @@ impl Function for GreatestFunction {
                 let column1 = column1.as_primitive::<Date32Type>();
                 let column2 = columns[1].to_arrow_array();
                 let column2 = column2.as_primitive::<Date32Type>();
-                let boolean_array = gt_dyn(&column1, &column2).context(ArrowComputeSnafu)?;
+                let boolean_array = gt(&column1, &column2).context(ArrowComputeSnafu)?;
                 let result =
                     zip::zip(&boolean_array, &column1, &column2).context(ArrowComputeSnafu)?;
                 Ok(Helper::try_into_vector(&result).context(error::FromArrowArraySnafu)?)

src/common/greptimedb-telemetry/src/lib.rs

@@ -15,6 +15,8 @@
 use std::env;
 use std::io::ErrorKind;
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
 use std::time::Duration;
 
 use common_runtime::error::{Error, Result};
@@ -24,7 +26,7 @@ use reqwest::{Client, Response};
 use serde::{Deserialize, Serialize};
 
 /// The URL to report telemetry data.
-pub const TELEMETRY_URL: &str = "https://api.greptime.cloud/db/otel/statistics";
+pub const TELEMETRY_URL: &str = "https://telemetry.greptimestats.com/db/otel/statistics";
 /// The local installation uuid cache file
 const UUID_FILE_NAME: &str = ".greptimedb-telemetry-uuid";
 
@@ -36,13 +38,29 @@ const GREPTIMEDB_TELEMETRY_CLIENT_CONNECT_TIMEOUT: Duration = Duration::from_sec
 const GREPTIMEDB_TELEMETRY_CLIENT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
 
 pub enum GreptimeDBTelemetryTask {
-    Enable(RepeatedTask<Error>),
+    Enable((RepeatedTask<Error>, Arc<AtomicBool>)),
     Disable,
 }
 
 impl GreptimeDBTelemetryTask {
-    pub fn enable(interval: Duration, task_fn: BoxedTaskFunction<Error>) -> Self {
-        GreptimeDBTelemetryTask::Enable(RepeatedTask::new(interval, task_fn))
+    pub fn should_report(&self, value: bool) {
+        match self {
+            GreptimeDBTelemetryTask::Enable((_, should_report)) => {
+                should_report.store(value, Ordering::Relaxed);
+            }
+            GreptimeDBTelemetryTask::Disable => {}
+        }
+    }
+
+    pub fn enable(
+        interval: Duration,
+        task_fn: BoxedTaskFunction<Error>,
+        should_report: Arc<AtomicBool>,
+    ) -> Self {
+        GreptimeDBTelemetryTask::Enable((
+            RepeatedTask::new(interval, task_fn).with_initial_delay(Some(Duration::ZERO)),
+            should_report,
+        ))
     }
 
     pub fn disable() -> Self {
@@ -51,7 +69,7 @@ impl GreptimeDBTelemetryTask {
 
     pub fn start(&self) -> Result<()> {
         match self {
-            GreptimeDBTelemetryTask::Enable(task) => {
+            GreptimeDBTelemetryTask::Enable((task, _)) => {
                 print_anonymous_usage_data_disclaimer();
                 task.start(common_runtime::bg_runtime())
             }
@@ -61,7 +79,7 @@ impl GreptimeDBTelemetryTask {
 
     pub async fn stop(&self) -> Result<()> {
         match self {
-            GreptimeDBTelemetryTask::Enable(task) => task.stop().await,
+            GreptimeDBTelemetryTask::Enable((task, _)) => task.stop().await,
             GreptimeDBTelemetryTask::Disable => Ok(()),
         }
     }
@@ -191,6 +209,8 @@ pub struct GreptimeDBTelemetry {
     client: Option<Client>,
     working_home: Option<String>,
     telemetry_url: &'static str,
+    should_report: Arc<AtomicBool>,
+    report_times: usize,
 }
 
 #[async_trait::async_trait]
@@ -200,13 +220,19 @@ impl TaskFunction<Error> for GreptimeDBTelemetry {
     }
 
     async fn call(&mut self) -> Result<()> {
-        self.report_telemetry_info().await;
+        if self.should_report.load(Ordering::Relaxed) {
+            self.report_telemetry_info().await;
+        }
         Ok(())
     }
 }
 
 impl GreptimeDBTelemetry {
-    pub fn new(working_home: Option<String>, statistics: Box<dyn Collector + Send + Sync>) -> Self {
+    pub fn new(
+        working_home: Option<String>,
+        statistics: Box<dyn Collector + Send + Sync>,
+        should_report: Arc<AtomicBool>,
+    ) -> Self {
         let client = Client::builder()
             .connect_timeout(GREPTIMEDB_TELEMETRY_CLIENT_CONNECT_TIMEOUT)
             .timeout(GREPTIMEDB_TELEMETRY_CLIENT_REQUEST_TIMEOUT)
@@ -216,6 +242,8 @@ impl GreptimeDBTelemetry {
             statistics,
             client: client.ok(),
             telemetry_url: TELEMETRY_URL,
+            should_report,
+            report_times: 0,
         }
     }
 
@@ -233,8 +261,11 @@ impl GreptimeDBTelemetry {
                 };
 
                 if let Some(client) = self.client.as_ref() {
-                    info!("reporting greptimedb version: {:?}", data);
+                    if self.report_times == 0 {
+                        info!("reporting greptimedb version: {:?}", data);
+                    }
                     let result = client.post(self.telemetry_url).json(&data).send().await;
+                    self.report_times += 1;
                     debug!("report version result: {:?}", result);
                     result.ok()
                 } else {
@@ -250,7 +281,8 @@ impl GreptimeDBTelemetry {
 mod tests {
     use std::convert::Infallible;
     use std::env;
-    use std::sync::atomic::AtomicUsize;
+    use std::sync::atomic::{AtomicBool, AtomicUsize};
+    use std::sync::Arc;
     use std::time::Duration;
 
     use common_test_util::ports;
@@ -370,7 +402,11 @@ mod tests {
         let working_home = working_home_temp.path().to_str().unwrap().to_string();
 
         let test_statistic = Box::new(TestStatistic);
-        let mut test_report = GreptimeDBTelemetry::new(Some(working_home.clone()), test_statistic);
+        let mut test_report = GreptimeDBTelemetry::new(
+            Some(working_home.clone()),
+            test_statistic,
+            Arc::new(AtomicBool::new(true)),
+        );
         let url = Box::leak(format!("{}:{}", "http://localhost", port).into_boxed_str());
         test_report.telemetry_url = url;
         let response = test_report.report_telemetry_info().await.unwrap();
@@ -384,7 +420,11 @@ mod tests {
         assert_eq!(1, body.nodes.unwrap());
 
         let failed_statistic = Box::new(FailedStatistic);
-        let mut failed_report = GreptimeDBTelemetry::new(Some(working_home), failed_statistic);
+        let mut failed_report = GreptimeDBTelemetry::new(
+            Some(working_home),
+            failed_statistic,
+            Arc::new(AtomicBool::new(true)),
+        );
         failed_report.telemetry_url = url;
         let response = failed_report.report_telemetry_info().await;
         assert!(response.is_none());

src/common/grpc/src/channel_manager.rs

@@ -16,6 +16,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
+use common_base::readable_size::ReadableSize;
 use common_telemetry::info;
 use dashmap::mapref::entry::Entry;
 use dashmap::DashMap;
@@ -31,8 +32,8 @@ use crate::error::{CreateChannelSnafu, InvalidConfigFilePathSnafu, InvalidTlsCon
 const RECYCLE_CHANNEL_INTERVAL_SECS: u64 = 60;
 pub const DEFAULT_GRPC_REQUEST_TIMEOUT_SECS: u64 = 10;
 pub const DEFAULT_GRPC_CONNECT_TIMEOUT_SECS: u64 = 1;
-pub const DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
-pub const DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE: usize = 512 * 1024 * 1024;
+pub const DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE: ReadableSize = ReadableSize::mb(512);
+pub const DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE: ReadableSize = ReadableSize::mb(512);
 
 lazy_static! {
     static ref ID: AtomicU64 = AtomicU64::new(0);
@@ -162,7 +163,14 @@ impl ChannelManager {
     }
 
     fn build_endpoint(&self, addr: &str) -> Result<Endpoint> {
-        let mut endpoint = Endpoint::new(format!("http://{addr}")).context(CreateChannelSnafu)?;
+        let http_prefix = if self.client_tls_config.is_some() {
+            "https"
+        } else {
+            "http"
+        };
+
+        let mut endpoint =
+            Endpoint::new(format!("{http_prefix}://{addr}")).context(CreateChannelSnafu)?;
 
         if let Some(dur) = self.config.timeout {
             endpoint = endpoint.timeout(dur);
@@ -250,9 +258,9 @@ pub struct ChannelConfig {
     pub tcp_nodelay: bool,
     pub client_tls: Option<ClientTlsOption>,
     // Max gRPC receiving(decoding) message size
-    pub max_recv_message_size: usize,
+    pub max_recv_message_size: ReadableSize,
     // Max gRPC sending(encoding) message size
-    pub max_send_message_size: usize,
+    pub max_send_message_size: ReadableSize,
 }
 
 impl Default for ChannelConfig {

src/common/grpc/src/flight.rs

@@ -241,7 +241,7 @@ mod test {
         .unwrap();
 
         let flight_data = batches_to_flight_data(
-            arrow_schema,
+            &arrow_schema,
             vec![
                 batch1.clone().into_df_record_batch(),
                 batch2.clone().into_df_record_batch(),

src/common/meta/Cargo.toml

@@ -28,7 +28,7 @@ etcd-client.workspace = true
 futures.workspace = true
 humantime-serde.workspace = true
 lazy_static.workspace = true
-metrics.workspace = true
+prometheus.workspace = true
 prost.workspace = true
 regex.workspace = true
 serde.workspace = true

src/common/meta/src/ddl/alter_table.rs

@@ -374,10 +374,11 @@ impl Procedure for AlterTableProcedure {
 
         let state = &self.data.state;
 
-        let _timer = common_telemetry::timer!(
-            metrics::METRIC_META_PROCEDURE_ALTER_TABLE,
-            &[("step", state.as_ref().to_string())]
-        );
+        let step = state.as_ref();
+
+        let _timer = metrics::METRIC_META_PROCEDURE_ALTER_TABLE
+            .with_label_values(&[step])
+            .start_timer();
 
         match state {
             AlterTableState::Prepare => self.on_prepare().await,

src/common/meta/src/ddl/create_table.rs

@@ -251,10 +251,9 @@ impl Procedure for CreateTableProcedure {
     async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
         let state = &self.creator.data.state;
 
-        let _timer = common_telemetry::timer!(
-            metrics::METRIC_META_PROCEDURE_CREATE_TABLE,
-            &[("step", state.as_ref().to_string())]
-        );
+        let _timer = metrics::METRIC_META_PROCEDURE_CREATE_TABLE
+            .with_label_values(&[state.as_ref()])
+            .start_timer();
 
         match state {
             CreateTableState::Prepare => self.on_prepare().await,

src/common/meta/src/ddl/drop_table.rs

@@ -197,10 +197,9 @@ impl Procedure for DropTableProcedure {
     async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
         let state = &self.data.state;
 
-        let _timer = common_telemetry::timer!(
-            metrics::METRIC_META_PROCEDURE_DROP_TABLE,
-            &[("step", state.as_ref().to_string())]
-        );
+        let _timer = metrics::METRIC_META_PROCEDURE_DROP_TABLE
+            .with_label_values(&[state.as_ref()])
+            .start_timer();
 
         match self.data.state {
             DropTableState::Prepare => self.on_prepare().await,

src/common/meta/src/ddl/truncate_table.rs

@@ -55,10 +55,9 @@ impl Procedure for TruncateTableProcedure {
     async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
         let state = &self.data.state;
 
-        let _timer = common_telemetry::timer!(
-            metrics::METRIC_META_PROCEDURE_TRUNCATE_TABLE,
-            &[("step", state.as_ref().to_string())]
-        );
+        let _timer = metrics::METRIC_META_PROCEDURE_TRUNCATE_TABLE
+            .with_label_values(&[state.as_ref()])
+            .start_timer();
 
         match self.data.state {
             TruncateTableState::Prepare => self.on_prepare().await,

src/common/meta/src/key.rs

@@ -258,7 +258,6 @@ impl<T: Serialize + DeserializeOwned> DeserializedValueWithBytes<T> {
         self.bytes.to_vec()
     }
 
-    #[cfg(feature = "testing")]
     /// Notes: used for test purpose.
     pub fn from_inner(inner: T) -> Self {
         let bytes = serde_json::to_vec(&inner).unwrap();
@@ -784,6 +783,7 @@ mod tests {
             },
             leader_peer: Some(Peer::new(datanode, "a2")),
             follower_peers: vec![],
+            leader_status: None,
         }
     }
 

src/common/meta/src/key/catalog_name.rs

@@ -16,10 +16,8 @@ use std::fmt::Display;
 use std::sync::Arc;
 
 use common_catalog::consts::DEFAULT_CATALOG_NAME;
-use common_telemetry::timer;
 use futures::stream::BoxStream;
 use futures::StreamExt;
-use metrics::increment_counter;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 
@@ -104,7 +102,7 @@ impl CatalogManager {
 
     /// Creates `CatalogNameKey`.
     pub async fn create(&self, catalog: CatalogNameKey<'_>, if_not_exists: bool) -> Result<()> {
-        let _timer = timer!(crate::metrics::METRIC_META_CREATE_CATALOG);
+        let _timer = crate::metrics::METRIC_META_CREATE_CATALOG.start_timer();
 
         let raw_key = catalog.as_raw_key();
         let raw_value = CatalogNameValue.try_as_raw_value()?;
@@ -113,7 +111,7 @@ impl CatalogManager {
             .put_conditionally(raw_key, raw_value, if_not_exists)
             .await?
         {
-            increment_counter!(crate::metrics::METRIC_META_CREATE_CATALOG);
+            crate::metrics::METRIC_META_CREATE_CATALOG_COUNTER.inc();
         }
 
         Ok(())

src/common/meta/src/key/schema_name.rs

@@ -18,11 +18,9 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use common_telemetry::timer;
 use futures::stream::BoxStream;
 use futures::StreamExt;
 use humantime_serde::re::humantime;
-use metrics::increment_counter;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 
@@ -146,7 +144,7 @@ impl SchemaManager {
         value: Option<SchemaNameValue>,
         if_not_exists: bool,
     ) -> Result<()> {
-        let _timer = timer!(crate::metrics::METRIC_META_CREATE_SCHEMA);
+        let _timer = crate::metrics::METRIC_META_CREATE_SCHEMA.start_timer();
 
         let raw_key = schema.as_raw_key();
         let raw_value = value.unwrap_or_default().try_as_raw_value()?;
@@ -155,7 +153,7 @@ impl SchemaManager {
             .put_conditionally(raw_key, raw_value, if_not_exists)
             .await?
         {
-            increment_counter!(crate::metrics::METRIC_META_CREATE_SCHEMA);
+            crate::metrics::METRIC_META_CREATE_SCHEMA_COUNTER.inc();
         }
 
         Ok(())

src/common/meta/src/kv_backend.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 pub mod memory;
+pub mod test;
 pub mod txn;
 
 use std::any::Any;
@@ -26,8 +27,7 @@ use crate::error::Error;
 use crate::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
     BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
-    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
-    RangeRequest, RangeResponse,
+    DeleteRangeResponse, PutRequest, PutResponse, RangeRequest, RangeResponse,
 };
 use crate::rpc::KeyValue;
 
@@ -65,9 +65,6 @@ where
         req: BatchDeleteRequest,
     ) -> Result<BatchDeleteResponse, Self::Error>;
 
-    /// MoveValue atomically renames the key to the given updated key.
-    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse, Self::Error>;
-
     // The following methods are implemented based on the above methods,
     // and a higher-level interface is provided for to simplify usage.
 

src/common/meta/src/kv_backend/memory.rs

@@ -17,12 +17,10 @@ use std::collections::btree_map::Entry;
 use std::collections::BTreeMap;
 use std::fmt::{Display, Formatter};
 use std::marker::PhantomData;
-use std::ops::Range;
 use std::sync::RwLock;
 
 use async_trait::async_trait;
 use common_error::ext::ErrorExt;
-use common_telemetry::timer;
 use serde::Serializer;
 
 use crate::kv_backend::txn::{Txn, TxnOp, TxnOpResponse, TxnRequest, TxnResponse};
@@ -31,8 +29,7 @@ use crate::metrics::METRIC_META_TXN_REQUEST;
 use crate::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
     BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
-    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
-    RangeRequest, RangeResponse,
+    DeleteRangeResponse, PutRequest, PutResponse, RangeRequest, RangeResponse,
 };
 use crate::rpc::KeyValue;
 
@@ -85,21 +82,25 @@ impl<T: ErrorExt + Send + Sync + 'static> KvBackend for MemoryKvBackend<T> {
     }
 
     async fn range(&self, req: RangeRequest) -> Result<RangeResponse, Self::Error> {
+        let range = req.range();
         let RangeRequest {
-            key,
-            range_end,
-            limit,
-            keys_only,
+            limit, keys_only, ..
         } = req;
 
         let kvs = self.kvs.read().unwrap();
+        let values = kvs.range(range);
 
-        let iter: Box<dyn Iterator<Item = (&Vec<u8>, &Vec<u8>)>> = if range_end.is_empty() {
-            Box::new(kvs.get_key_value(&key).into_iter())
-        } else {
-            Box::new(kvs.range(key..range_end))
-        };
-        let mut kvs = iter
+        let mut more = false;
+        let mut iter: i64 = 0;
+
+        let kvs = values
+            .take_while(|_| {
+                let take = limit == 0 || iter != limit;
+                iter += 1;
+                more = limit > 0 && iter > limit;
+
+                take
+            })
             .map(|(k, v)| {
                 let key = k.clone();
                 let value = if keys_only { vec![] } else { v.clone() };
@@ -107,13 +108,6 @@ impl<T: ErrorExt + Send + Sync + 'static> KvBackend for MemoryKvBackend<T> {
             })
             .collect::<Vec<_>>();
 
-        let more = if limit > 0 && kvs.len() > limit as usize {
-            kvs.truncate(limit as usize);
-            true
-        } else {
-            false
-        };
-
         Ok(RangeResponse { kvs, more })
     }
 
@@ -215,36 +209,32 @@ impl<T: ErrorExt + Send + Sync + 'static> KvBackend for MemoryKvBackend<T> {
         &self,
         req: DeleteRangeRequest,
     ) -> Result<DeleteRangeResponse, Self::Error> {
-        let DeleteRangeRequest {
-            key,
-            range_end,
-            prev_kv,
-        } = req;
+        let range = req.range();
+        let DeleteRangeRequest { prev_kv, .. } = req;
 
         let mut kvs = self.kvs.write().unwrap();
 
-        let prev_kvs = if range_end.is_empty() {
-            kvs.remove(&key)
-                .into_iter()
-                .map(|value| KeyValue {
-                    key: key.clone(),
-                    value,
-                })
-                .collect::<Vec<_>>()
-        } else {
-            let range = Range {
-                start: key,
-                end: range_end,
-            };
-            kvs.extract_if(|key, _| range.contains(key))
-                .map(Into::into)
-                .collect::<Vec<_>>()
-        };
+        let keys = kvs
+            .range(range)
+            .map(|(key, _)| key.clone())
+            .collect::<Vec<_>>();
 
-        Ok(DeleteRangeResponse {
-            deleted: prev_kvs.len() as i64,
-            prev_kvs: if prev_kv { prev_kvs } else { vec![] },
-        })
+        let mut prev_kvs = if prev_kv {
+            Vec::with_capacity(keys.len())
+        } else {
+            vec![]
+        };
+        let deleted = keys.len() as i64;
+
+        for key in keys {
+            if let Some(value) = kvs.remove(&key) {
+                if prev_kv {
+                    prev_kvs.push((key.clone(), value).into())
+                }
+            }
+        }
+
+        Ok(DeleteRangeResponse { deleted, prev_kvs })
     }
 
     async fn batch_delete(
@@ -271,27 +261,6 @@ impl<T: ErrorExt + Send + Sync + 'static> KvBackend for MemoryKvBackend<T> {
 
         Ok(BatchDeleteResponse { prev_kvs })
     }
-
-    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse, Self::Error> {
-        let MoveValueRequest { from_key, to_key } = req;
-
-        let mut kvs = self.kvs.write().unwrap();
-
-        let kv = if let Some(v) = kvs.remove(&from_key) {
-            kvs.insert(to_key, v.clone());
-            Some(KeyValue {
-                key: from_key,
-                value: v,
-            })
-        } else {
-            kvs.get(&to_key).map(|v| KeyValue {
-                key: to_key,
-                value: v.clone(),
-            })
-        };
-
-        Ok(MoveValueResponse(kv))
-    }
 }
 
 #[async_trait]
@@ -299,10 +268,9 @@ impl<T: ErrorExt + Send + Sync> TxnService for MemoryKvBackend<T> {
     type Error = T;
 
     async fn txn(&self, txn: Txn) -> Result<TxnResponse, Self::Error> {
-        let _timer = timer!(
-            METRIC_META_TXN_REQUEST,
-            &[("target", "memory"), ("op", "txn")]
-        );
+        let _timer = METRIC_META_TXN_REQUEST
+            .with_label_values(&["memory", "txn"])
+            .start_timer();
 
         let TxnRequest {
             compare,
@@ -358,313 +326,68 @@ impl<T: ErrorExt + Send + Sync> TxnService for MemoryKvBackend<T> {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::atomic::{AtomicU8, Ordering};
     use std::sync::Arc;
 
     use super::*;
     use crate::error::Error;
-    use crate::kv_backend::KvBackend;
-    use crate::rpc::store::{BatchGetRequest, BatchPutRequest};
-    use crate::rpc::KeyValue;
-    use crate::util;
+    use crate::kv_backend::test::{
+        prepare_kv, test_kv_batch_delete, test_kv_batch_get, test_kv_compare_and_put,
+        test_kv_delete_range, test_kv_put, test_kv_range, test_kv_range_2,
+    };
 
     async fn mock_mem_store_with_data() -> MemoryKvBackend<Error> {
         let kv_store = MemoryKvBackend::<Error>::new();
-        let kvs = mock_kvs();
-
-        assert!(kv_store
-            .batch_put(BatchPutRequest {
-                kvs,
-                ..Default::default()
-            })
-            .await
-            .is_ok());
-
-        assert!(kv_store
-            .put(PutRequest {
-                key: b"key11".to_vec(),
-                value: b"val11".to_vec(),
-                ..Default::default()
-            })
-            .await
-            .is_ok());
+        prepare_kv(&kv_store).await;
 
         kv_store
     }
 
-    fn mock_kvs() -> Vec<KeyValue> {
-        vec![
-            KeyValue {
-                key: b"key1".to_vec(),
-                value: b"val1".to_vec(),
-            },
-            KeyValue {
-                key: b"key2".to_vec(),
-                value: b"val2".to_vec(),
-            },
-            KeyValue {
-                key: b"key3".to_vec(),
-                value: b"val3".to_vec(),
-            },
-        ]
-    }
-
     #[tokio::test]
     async fn test_put() {
         let kv_store = mock_mem_store_with_data().await;
 
-        let resp = kv_store
-            .put(PutRequest {
-                key: b"key11".to_vec(),
-                value: b"val12".to_vec(),
-                prev_kv: false,
-            })
-            .await
-            .unwrap();
-        assert!(resp.prev_kv.is_none());
-
-        let resp = kv_store
-            .put(PutRequest {
-                key: b"key11".to_vec(),
-                value: b"val13".to_vec(),
-                prev_kv: true,
-            })
-            .await
-            .unwrap();
-        let prev_kv = resp.prev_kv.unwrap();
-        assert_eq!(b"key11", prev_kv.key());
-        assert_eq!(b"val12", prev_kv.value());
+        test_kv_put(kv_store).await;
     }
 
     #[tokio::test]
     async fn test_range() {
         let kv_store = mock_mem_store_with_data().await;
 
-        let key = b"key1".to_vec();
-        let range_end = util::get_prefix_end_key(b"key1");
+        test_kv_range(kv_store).await;
+    }
 
-        let resp = kv_store
-            .range(RangeRequest {
-                key: key.clone(),
-                range_end: range_end.clone(),
-                limit: 0,
-                keys_only: false,
-            })
-            .await
-            .unwrap();
+    #[tokio::test]
+    async fn test_range_2() {
+        let kv = MemoryKvBackend::<Error>::new();
 
-        assert_eq!(2, resp.kvs.len());
-        assert_eq!(b"key1", resp.kvs[0].key());
-        assert_eq!(b"val1", resp.kvs[0].value());
-        assert_eq!(b"key11", resp.kvs[1].key());
-        assert_eq!(b"val11", resp.kvs[1].value());
-
-        let resp = kv_store
-            .range(RangeRequest {
-                key: key.clone(),
-                range_end: range_end.clone(),
-                limit: 0,
-                keys_only: true,
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(2, resp.kvs.len());
-        assert_eq!(b"key1", resp.kvs[0].key());
-        assert_eq!(b"", resp.kvs[0].value());
-        assert_eq!(b"key11", resp.kvs[1].key());
-        assert_eq!(b"", resp.kvs[1].value());
-
-        let resp = kv_store
-            .range(RangeRequest {
-                key: key.clone(),
-                limit: 0,
-                keys_only: false,
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(1, resp.kvs.len());
-        assert_eq!(b"key1", resp.kvs[0].key());
-        assert_eq!(b"val1", resp.kvs[0].value());
-
-        let resp = kv_store
-            .range(RangeRequest {
-                key,
-                range_end,
-                limit: 1,
-                keys_only: false,
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(1, resp.kvs.len());
-        assert_eq!(b"key1", resp.kvs[0].key());
-        assert_eq!(b"val1", resp.kvs[0].value());
+        test_kv_range_2(kv).await;
     }
 
     #[tokio::test]
     async fn test_batch_get() {
         let kv_store = mock_mem_store_with_data().await;
 
-        let keys = vec![];
-        let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
-
-        assert!(resp.kvs.is_empty());
-
-        let keys = vec![b"key10".to_vec()];
-        let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
-
-        assert!(resp.kvs.is_empty());
-
-        let keys = vec![b"key1".to_vec(), b"key3".to_vec(), b"key4".to_vec()];
-        let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
-
-        assert_eq!(2, resp.kvs.len());
-        assert_eq!(b"key1", resp.kvs[0].key());
-        assert_eq!(b"val1", resp.kvs[0].value());
-        assert_eq!(b"key3", resp.kvs[1].key());
-        assert_eq!(b"val3", resp.kvs[1].value());
+        test_kv_batch_get(kv_store).await;
     }
 
     #[tokio::test(flavor = "multi_thread")]
     async fn test_compare_and_put() {
         let kv_store = Arc::new(MemoryKvBackend::<Error>::new());
-        let success = Arc::new(AtomicU8::new(0));
 
-        let mut joins = vec![];
-        for _ in 0..20 {
-            let kv_store_clone = kv_store.clone();
-            let success_clone = success.clone();
-            let join = tokio::spawn(async move {
-                let req = CompareAndPutRequest {
-                    key: b"key".to_vec(),
-                    expect: vec![],
-                    value: b"val_new".to_vec(),
-                };
-                let resp = kv_store_clone.compare_and_put(req).await.unwrap();
-                if resp.success {
-                    success_clone.fetch_add(1, Ordering::SeqCst);
-                }
-            });
-            joins.push(join);
-        }
-
-        for join in joins {
-            join.await.unwrap();
-        }
-
-        assert_eq!(1, success.load(Ordering::SeqCst));
+        test_kv_compare_and_put(kv_store).await;
     }
 
     #[tokio::test]
     async fn test_delete_range() {
         let kv_store = mock_mem_store_with_data().await;
 
-        let req = DeleteRangeRequest {
-            key: b"key3".to_vec(),
-            range_end: vec![],
-            prev_kv: true,
-        };
-
-        let resp = kv_store.delete_range(req).await.unwrap();
-        assert_eq!(1, resp.prev_kvs.len());
-        assert_eq!(b"key3", resp.prev_kvs[0].key());
-        assert_eq!(b"val3", resp.prev_kvs[0].value());
-
-        let resp = kv_store.get(b"key3").await.unwrap();
-        assert!(resp.is_none());
-
-        let req = DeleteRangeRequest {
-            key: b"key2".to_vec(),
-            range_end: vec![],
-            prev_kv: false,
-        };
-
-        let resp = kv_store.delete_range(req).await.unwrap();
-        assert!(resp.prev_kvs.is_empty());
-
-        let resp = kv_store.get(b"key2").await.unwrap();
-        assert!(resp.is_none());
-
-        let key = b"key1".to_vec();
-        let range_end = util::get_prefix_end_key(b"key1");
-
-        let req = DeleteRangeRequest {
-            key: key.clone(),
-            range_end: range_end.clone(),
-            prev_kv: true,
-        };
-        let resp = kv_store.delete_range(req).await.unwrap();
-        assert_eq!(2, resp.prev_kvs.len());
-
-        let req = RangeRequest {
-            key,
-            range_end,
-            ..Default::default()
-        };
-        let resp = kv_store.range(req).await.unwrap();
-        assert!(resp.kvs.is_empty());
-    }
-
-    #[tokio::test]
-    async fn test_move_value() {
-        let kv_store = mock_mem_store_with_data().await;
-
-        let req = MoveValueRequest {
-            from_key: b"key1".to_vec(),
-            to_key: b"key111".to_vec(),
-        };
-
-        let resp = kv_store.move_value(req).await.unwrap();
-        assert_eq!(b"key1", resp.0.as_ref().unwrap().key());
-        assert_eq!(b"val1", resp.0.as_ref().unwrap().value());
-
-        let kv_store = mock_mem_store_with_data().await;
-
-        let req = MoveValueRequest {
-            from_key: b"notexistkey".to_vec(),
-            to_key: b"key222".to_vec(),
-        };
-
-        let resp = kv_store.move_value(req).await.unwrap();
-        assert!(resp.0.is_none());
+        test_kv_delete_range(kv_store).await;
     }
 
     #[tokio::test]
     async fn test_batch_delete() {
         let kv_store = mock_mem_store_with_data().await;
 
-        assert!(kv_store.get(b"key1").await.unwrap().is_some());
-        assert!(kv_store.get(b"key100").await.unwrap().is_none());
-
-        let req = BatchDeleteRequest {
-            keys: vec![b"key1".to_vec(), b"key100".to_vec()],
-            prev_kv: true,
-        };
-        let resp = kv_store.batch_delete(req).await.unwrap();
-        assert_eq!(1, resp.prev_kvs.len());
-        assert_eq!(
-            vec![KeyValue {
-                key: b"key1".to_vec(),
-                value: b"val1".to_vec()
-            }],
-            resp.prev_kvs
-        );
-        assert!(kv_store.get(b"key1").await.unwrap().is_none());
-
-        assert!(kv_store.get(b"key2").await.unwrap().is_some());
-        assert!(kv_store.get(b"key3").await.unwrap().is_some());
-
-        let req = BatchDeleteRequest {
-            keys: vec![b"key2".to_vec(), b"key3".to_vec()],
-            prev_kv: false,
-        };
-        let resp = kv_store.batch_delete(req).await.unwrap();
-        assert!(resp.prev_kvs.is_empty());
-
-        assert!(kv_store.get(b"key2").await.unwrap().is_none());
-        assert!(kv_store.get(b"key3").await.unwrap().is_none());
+        test_kv_batch_delete(kv_store).await;
     }
 }

src/common/meta/src/kv_backend/test.rs

@@ -0,0 +1,352 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::{AtomicU8, Ordering};
+use std::sync::Arc;
+
+use super::{KvBackend, *};
+use crate::error::Error;
+use crate::rpc::store::{BatchGetRequest, PutRequest};
+use crate::rpc::KeyValue;
+use crate::util;
+
+pub fn mock_kvs() -> Vec<KeyValue> {
+    vec![
+        KeyValue {
+            key: b"key1".to_vec(),
+            value: b"val1".to_vec(),
+        },
+        KeyValue {
+            key: b"key2".to_vec(),
+            value: b"val2".to_vec(),
+        },
+        KeyValue {
+            key: b"key3".to_vec(),
+            value: b"val3".to_vec(),
+        },
+    ]
+}
+
+pub async fn prepare_kv(kv_store: &impl KvBackend) {
+    let kvs = mock_kvs();
+    assert!(kv_store
+        .batch_put(BatchPutRequest {
+            kvs,
+            ..Default::default()
+        })
+        .await
+        .is_ok());
+
+    assert!(kv_store
+        .put(PutRequest {
+            key: b"key11".to_vec(),
+            value: b"val11".to_vec(),
+            ..Default::default()
+        })
+        .await
+        .is_ok());
+}
+
+pub async fn test_kv_put(kv_store: impl KvBackend) {
+    let resp = kv_store
+        .put(PutRequest {
+            key: b"key11".to_vec(),
+            value: b"val12".to_vec(),
+            prev_kv: false,
+        })
+        .await
+        .unwrap();
+    assert!(resp.prev_kv.is_none());
+
+    let resp = kv_store
+        .put(PutRequest {
+            key: b"key11".to_vec(),
+            value: b"val13".to_vec(),
+            prev_kv: true,
+        })
+        .await
+        .unwrap();
+    let prev_kv = resp.prev_kv.unwrap();
+    assert_eq!(b"key11", prev_kv.key());
+    assert_eq!(b"val12", prev_kv.value());
+}
+
+pub async fn test_kv_range(kv_store: impl KvBackend) {
+    let key = b"key1".to_vec();
+    let range_end = util::get_prefix_end_key(b"key1");
+
+    let resp = kv_store
+        .range(RangeRequest {
+            key: key.clone(),
+            range_end: range_end.clone(),
+            limit: 0,
+            keys_only: false,
+        })
+        .await
+        .unwrap();
+
+    assert_eq!(2, resp.kvs.len());
+    assert_eq!(b"key1", resp.kvs[0].key());
+    assert_eq!(b"val1", resp.kvs[0].value());
+    assert_eq!(b"key11", resp.kvs[1].key());
+    assert_eq!(b"val11", resp.kvs[1].value());
+
+    let resp = kv_store
+        .range(RangeRequest {
+            key: key.clone(),
+            range_end: range_end.clone(),
+            limit: 0,
+            keys_only: true,
+        })
+        .await
+        .unwrap();
+
+    assert_eq!(2, resp.kvs.len());
+    assert_eq!(b"key1", resp.kvs[0].key());
+    assert_eq!(b"", resp.kvs[0].value());
+    assert_eq!(b"key11", resp.kvs[1].key());
+    assert_eq!(b"", resp.kvs[1].value());
+
+    let resp = kv_store
+        .range(RangeRequest {
+            key: key.clone(),
+            limit: 0,
+            keys_only: false,
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+    assert_eq!(1, resp.kvs.len());
+    assert_eq!(b"key1", resp.kvs[0].key());
+    assert_eq!(b"val1", resp.kvs[0].value());
+
+    let resp = kv_store
+        .range(RangeRequest {
+            key,
+            range_end,
+            limit: 1,
+            keys_only: false,
+        })
+        .await
+        .unwrap();
+
+    assert_eq!(1, resp.kvs.len());
+    assert_eq!(b"key1", resp.kvs[0].key());
+    assert_eq!(b"val1", resp.kvs[0].value());
+}
+
+pub async fn test_kv_range_2(kv_store: impl KvBackend) {
+    kv_store
+        .put(PutRequest::new().with_key("atest").with_value("value"))
+        .await
+        .unwrap();
+
+    kv_store
+        .put(PutRequest::new().with_key("test").with_value("value"))
+        .await
+        .unwrap();
+
+    // If both key and range_end are ‘\0’, then range represents all keys.
+    let result = kv_store
+        .range(RangeRequest::new().with_range(b"\0".to_vec(), b"\0".to_vec()))
+        .await
+        .unwrap();
+
+    assert_eq!(result.kvs.len(), 2);
+    assert!(!result.more);
+
+    // If range_end is ‘\0’, the range is all keys greater than or equal to the key argument.
+    let result = kv_store
+        .range(RangeRequest::new().with_range(b"a".to_vec(), b"\0".to_vec()))
+        .await
+        .unwrap();
+
+    assert_eq!(result.kvs.len(), 2);
+
+    let result = kv_store
+        .range(RangeRequest::new().with_range(b"b".to_vec(), b"\0".to_vec()))
+        .await
+        .unwrap();
+
+    assert_eq!(result.kvs.len(), 1);
+    assert_eq!(result.kvs[0].key, b"test");
+
+    // Fetches the keys >= "a", set limit to 1, the `more` should be true.
+    let result = kv_store
+        .range(
+            RangeRequest::new()
+                .with_range(b"a".to_vec(), b"\0".to_vec())
+                .with_limit(1),
+        )
+        .await
+        .unwrap();
+    assert_eq!(result.kvs.len(), 1);
+    assert!(result.more);
+
+    // Fetches the keys >= "a", set limit to 2, the `more` should be false.
+    let result = kv_store
+        .range(
+            RangeRequest::new()
+                .with_range(b"a".to_vec(), b"\0".to_vec())
+                .with_limit(2),
+        )
+        .await
+        .unwrap();
+    assert_eq!(result.kvs.len(), 2);
+    assert!(!result.more);
+
+    // Fetches the keys >= "a", set limit to 3, the `more` should be false.
+    let result = kv_store
+        .range(
+            RangeRequest::new()
+                .with_range(b"a".to_vec(), b"\0".to_vec())
+                .with_limit(3),
+        )
+        .await
+        .unwrap();
+    assert_eq!(result.kvs.len(), 2);
+    assert!(!result.more);
+}
+
+pub async fn test_kv_batch_get(kv_store: impl KvBackend) {
+    let keys = vec![];
+    let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
+
+    assert!(resp.kvs.is_empty());
+
+    let keys = vec![b"key10".to_vec()];
+    let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
+
+    assert!(resp.kvs.is_empty());
+
+    let keys = vec![b"key1".to_vec(), b"key3".to_vec(), b"key4".to_vec()];
+    let resp = kv_store.batch_get(BatchGetRequest { keys }).await.unwrap();
+
+    assert_eq!(2, resp.kvs.len());
+    assert_eq!(b"key1", resp.kvs[0].key());
+    assert_eq!(b"val1", resp.kvs[0].value());
+    assert_eq!(b"key3", resp.kvs[1].key());
+    assert_eq!(b"val3", resp.kvs[1].value());
+}
+
+pub async fn test_kv_compare_and_put(kv_store: Arc<dyn KvBackend<Error = Error>>) {
+    let success = Arc::new(AtomicU8::new(0));
+
+    let mut joins = vec![];
+    for _ in 0..20 {
+        let kv_store_clone = kv_store.clone();
+        let success_clone = success.clone();
+        let join = tokio::spawn(async move {
+            let req = CompareAndPutRequest {
+                key: b"key".to_vec(),
+                expect: vec![],
+                value: b"val_new".to_vec(),
+            };
+            let resp = kv_store_clone.compare_and_put(req).await.unwrap();
+            if resp.success {
+                success_clone.fetch_add(1, Ordering::SeqCst);
+            }
+        });
+        joins.push(join);
+    }
+
+    for join in joins {
+        join.await.unwrap();
+    }
+
+    assert_eq!(1, success.load(Ordering::SeqCst));
+}
+
+pub async fn test_kv_delete_range(kv_store: impl KvBackend) {
+    let req = DeleteRangeRequest {
+        key: b"key3".to_vec(),
+        range_end: vec![],
+        prev_kv: true,
+    };
+
+    let resp = kv_store.delete_range(req).await.unwrap();
+    assert_eq!(1, resp.prev_kvs.len());
+    assert_eq!(1, resp.deleted);
+    assert_eq!(b"key3", resp.prev_kvs[0].key());
+    assert_eq!(b"val3", resp.prev_kvs[0].value());
+
+    let resp = kv_store.get(b"key3").await.unwrap();
+    assert!(resp.is_none());
+
+    let req = DeleteRangeRequest {
+        key: b"key2".to_vec(),
+        range_end: vec![],
+        prev_kv: false,
+    };
+
+    let resp = kv_store.delete_range(req).await.unwrap();
+    assert_eq!(1, resp.deleted);
+    assert!(resp.prev_kvs.is_empty());
+
+    let resp = kv_store.get(b"key2").await.unwrap();
+    assert!(resp.is_none());
+
+    let key = b"key1".to_vec();
+    let range_end = util::get_prefix_end_key(b"key1");
+
+    let req = DeleteRangeRequest {
+        key: key.clone(),
+        range_end: range_end.clone(),
+        prev_kv: true,
+    };
+    let resp = kv_store.delete_range(req).await.unwrap();
+    assert_eq!(2, resp.prev_kvs.len());
+
+    let req = RangeRequest {
+        key,
+        range_end,
+        ..Default::default()
+    };
+    let resp = kv_store.range(req).await.unwrap();
+    assert!(resp.kvs.is_empty());
+}
+
+pub async fn test_kv_batch_delete(kv_store: impl KvBackend) {
+    assert!(kv_store.get(b"key1").await.unwrap().is_some());
+    assert!(kv_store.get(b"key100").await.unwrap().is_none());
+
+    let req = BatchDeleteRequest {
+        keys: vec![b"key1".to_vec(), b"key100".to_vec()],
+        prev_kv: true,
+    };
+    let resp = kv_store.batch_delete(req).await.unwrap();
+    assert_eq!(1, resp.prev_kvs.len());
+    assert_eq!(
+        vec![KeyValue {
+            key: b"key1".to_vec(),
+            value: b"val1".to_vec()
+        }],
+        resp.prev_kvs
+    );
+    assert!(kv_store.get(b"key1").await.unwrap().is_none());
+
+    assert!(kv_store.get(b"key2").await.unwrap().is_some());
+    assert!(kv_store.get(b"key3").await.unwrap().is_some());
+
+    let req = BatchDeleteRequest {
+        keys: vec![b"key2".to_vec(), b"key3".to_vec()],
+        prev_kv: false,
+    };
+    let resp = kv_store.batch_delete(req).await.unwrap();
+    assert!(resp.prev_kvs.is_empty());
+
+    assert!(kv_store.get(b"key2").await.unwrap().is_none());
+    assert!(kv_store.get(b"key3").await.unwrap().is_none());
+}

src/common/meta/src/metrics.rs

@@ -12,11 +12,42 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub const METRIC_META_TXN_REQUEST: &str = "meta.txn_request";
+use lazy_static::lazy_static;
+use prometheus::*;
 
-pub(crate) const METRIC_META_CREATE_CATALOG: &str = "meta.create_catalog";
-pub(crate) const METRIC_META_CREATE_SCHEMA: &str = "meta.create_schema";
-pub(crate) const METRIC_META_PROCEDURE_CREATE_TABLE: &str = "meta.procedure.create_table";
-pub(crate) const METRIC_META_PROCEDURE_DROP_TABLE: &str = "meta.procedure.drop_table";
-pub(crate) const METRIC_META_PROCEDURE_ALTER_TABLE: &str = "meta.procedure.alter_table";
-pub(crate) const METRIC_META_PROCEDURE_TRUNCATE_TABLE: &str = "meta.procedure.truncate_table";
+lazy_static! {
+    pub static ref METRIC_META_TXN_REQUEST: HistogramVec =
+        register_histogram_vec!("meta_txn_request", "meta txn request", &["target", "op"]).unwrap();
+    pub static ref METRIC_META_CREATE_CATALOG: Histogram =
+        register_histogram!("meta_create_catalog", "meta create catalog").unwrap();
+    pub static ref METRIC_META_CREATE_CATALOG_COUNTER: IntCounter =
+        register_int_counter!("meta_create_catalog_counter", "meta create catalog").unwrap();
+    pub static ref METRIC_META_CREATE_SCHEMA: Histogram =
+        register_histogram!("meta_create_schema", "meta create schema").unwrap();
+    pub static ref METRIC_META_CREATE_SCHEMA_COUNTER: IntCounter =
+        register_int_counter!("meta_create_schema_counter", "meta create schema").unwrap();
+    pub static ref METRIC_META_PROCEDURE_CREATE_TABLE: HistogramVec = register_histogram_vec!(
+        "meta_procedure_create_table",
+        "meta procedure create table",
+        &["step"]
+    )
+    .unwrap();
+    pub static ref METRIC_META_PROCEDURE_DROP_TABLE: HistogramVec = register_histogram_vec!(
+        "meta_procedure_drop_table",
+        "meta procedure drop table",
+        &["step"]
+    )
+    .unwrap();
+    pub static ref METRIC_META_PROCEDURE_ALTER_TABLE: HistogramVec = register_histogram_vec!(
+        "meta_procedure_alter_table",
+        "meta procedure alter table",
+        &["step"]
+    )
+    .unwrap();
+    pub static ref METRIC_META_PROCEDURE_TRUNCATE_TABLE: HistogramVec = register_histogram_vec!(
+        "meta_procedure_truncate_table",
+        "meta procedure truncate table",
+        &["step"]
+    )
+    .unwrap();
+}

src/common/meta/src/rpc/router.rs

@@ -15,65 +15,19 @@
 use std::collections::{BTreeMap, HashMap, HashSet};
 
 use api::v1::meta::{
-    Partition as PbPartition, Peer as PbPeer, Region as PbRegion, RegionRoute as PbRegionRoute,
-    RouteRequest as PbRouteRequest, RouteResponse as PbRouteResponse, Table as PbTable,
-    TableId as PbTableId, TableRoute as PbTableRoute, TableRouteValue as PbTableRouteValue,
+    Partition as PbPartition, Peer as PbPeer, Region as PbRegion, Table as PbTable,
+    TableRoute as PbTableRoute,
 };
 use serde::ser::SerializeSeq;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use snafu::OptionExt;
 use store_api::storage::{RegionId, RegionNumber};
-use table::metadata::TableId;
 
 use crate::error::{self, Result};
 use crate::key::RegionDistribution;
 use crate::peer::Peer;
-use crate::rpc::util;
 use crate::table_name::TableName;
 
-#[derive(Debug, Clone, Default)]
-pub struct RouteRequest {
-    pub table_ids: Vec<TableId>,
-}
-
-impl From<RouteRequest> for PbRouteRequest {
-    fn from(mut req: RouteRequest) -> Self {
-        Self {
-            header: None,
-            table_ids: req.table_ids.drain(..).map(|id| PbTableId { id }).collect(),
-        }
-    }
-}
-
-impl RouteRequest {
-    #[inline]
-    pub fn new(table_id: TableId) -> Self {
-        Self {
-            table_ids: vec![table_id],
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct RouteResponse {
-    pub table_routes: Vec<TableRoute>,
-}
-
-impl TryFrom<PbRouteResponse> for RouteResponse {
-    type Error = error::Error;
-
-    fn try_from(pb: PbRouteResponse) -> Result<Self> {
-        util::check_response_header(pb.header.as_ref())?;
-
-        let table_routes = pb
-            .table_routes
-            .into_iter()
-            .map(|x| TableRoute::try_from_raw(&pb.peers, x))
-            .collect::<Result<Vec<_>>>()?;
-        Ok(Self { table_routes })
-    }
-}
-
 pub fn region_distribution(region_routes: &[RegionRoute]) -> Result<RegionDistribution> {
     let mut regions_id_map = RegionDistribution::new();
     for route in region_routes.iter() {
@@ -194,112 +148,12 @@ impl TableRoute {
                 region,
                 leader_peer,
                 follower_peers,
+                leader_status: None,
             });
         }
 
         Ok(Self::new(table, region_routes))
     }
-
-    pub fn try_into_raw(self) -> Result<(Vec<PbPeer>, PbTableRoute)> {
-        let mut peers = HashSet::new();
-        self.region_routes
-            .iter()
-            .filter_map(|x| x.leader_peer.as_ref())
-            .for_each(|p| {
-                let _ = peers.insert(p.clone());
-            });
-        self.region_routes
-            .iter()
-            .flat_map(|x| x.follower_peers.iter())
-            .for_each(|p| {
-                let _ = peers.insert(p.clone());
-            });
-        let mut peers = peers.into_iter().map(Into::into).collect::<Vec<PbPeer>>();
-        peers.sort_by_key(|x| x.id);
-
-        let find_peer = |peer_id: u64| -> u64 {
-            peers
-                .iter()
-                .enumerate()
-                .find_map(|(i, x)| {
-                    if x.id == peer_id {
-                        Some(i as u64)
-                    } else {
-                        None
-                    }
-                })
-                .unwrap_or_else(|| {
-                    panic!("Peer {peer_id} must be present when collecting all peers.")
-                })
-        };
-
-        let mut region_routes = Vec::with_capacity(self.region_routes.len());
-        for region_route in self.region_routes.into_iter() {
-            let leader_peer_index = region_route.leader_peer.map(|x| find_peer(x.id)).context(
-                error::RouteInfoCorruptedSnafu {
-                    err_msg: "'leader_peer' is empty in region route",
-                },
-            )?;
-
-            let follower_peer_indexes = region_route
-                .follower_peers
-                .iter()
-                .map(|x| find_peer(x.id))
-                .collect::<Vec<_>>();
-
-            region_routes.push(PbRegionRoute {
-                region: Some(region_route.region.into()),
-                leader_peer_index,
-                follower_peer_indexes,
-            });
-        }
-
-        let table_route = PbTableRoute {
-            table: Some(self.table.into()),
-            region_routes,
-        };
-        Ok((peers, table_route))
-    }
-
-    pub fn find_leaders(&self) -> HashSet<Peer> {
-        find_leaders(&self.region_routes)
-    }
-
-    pub fn find_leader_regions(&self, datanode: &Peer) -> Vec<RegionNumber> {
-        find_leader_regions(&self.region_routes, datanode)
-    }
-
-    pub fn find_region_leader(&self, region_number: RegionNumber) -> Option<&Peer> {
-        self.region_leaders
-            .get(&region_number)
-            .and_then(|x| x.as_ref())
-    }
-}
-
-impl TryFrom<PbTableRouteValue> for TableRoute {
-    type Error = error::Error;
-
-    fn try_from(pb: PbTableRouteValue) -> Result<Self> {
-        TableRoute::try_from_raw(
-            &pb.peers,
-            pb.table_route.context(error::InvalidProtoMsgSnafu {
-                err_msg: "expected table_route",
-            })?,
-        )
-    }
-}
-
-impl TryFrom<TableRoute> for PbTableRouteValue {
-    type Error = error::Error;
-
-    fn try_from(table_route: TableRoute) -> Result<Self> {
-        let (peers, table_route) = table_route.try_into_raw()?;
-
-        Ok(PbTableRouteValue {
-            peers,
-            table_route: Some(table_route),
-        })
-    }
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
@@ -343,6 +197,45 @@ pub struct RegionRoute {
     pub region: Region,
     pub leader_peer: Option<Peer>,
     pub follower_peers: Vec<Peer>,
+    /// `None` by default.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub leader_status: Option<RegionStatus>,
+}
+
+/// The Status of the [Region].
+#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
+pub enum RegionStatus {
+    /// The following cases in which the [Region] will be downgraded.
+    ///
+    /// - The [Region] is unavailable(e.g., Crashed, Network disconnected).
+    /// - The [Region] was planned to migrate to another [Peer].
+    Downgraded,
+}
+
+impl RegionRoute {
+    /// Returns true if the Leader [Region] is downgraded.
+    ///
+    /// The following cases in which the [Region] will be downgraded.
+    ///
+    /// - The [Region] is unavailable(e.g., Crashed, Network disconnected).
+    /// - The [Region] was planned to migrate to another [Peer].
+    ///
+    pub fn is_leader_downgraded(&self) -> bool {
+        matches!(self.leader_status, Some(RegionStatus::Downgraded))
+    }
+
+    /// Marks the Leader [Region] as downgraded.
+    ///
+    /// We should downgrade a [Region] before deactivating it:
+    ///
+    /// - During the [Region] Failover Procedure.
+    /// - Migrating a [Region].
+    ///
+    /// **Notes:** Meta Server will stop renewing the lease for the downgraded [Region].
+    ///
+    pub fn downgrade_leader(&mut self) {
+        self.leader_status = Some(RegionStatus::Downgraded)
+    }
 }
 
 pub struct RegionRoutes(pub Vec<RegionRoute>);
@@ -456,14 +349,50 @@ impl From<PbPartition> for Partition {
 
 #[cfg(test)]
 mod tests {
-    use api::v1::meta::{
-        Partition as PbPartition, Peer as PbPeer, Region as PbRegion, RegionRoute as PbRegionRoute,
-        RouteRequest as PbRouteRequest, RouteResponse as PbRouteResponse, Table as PbTable,
-        TableName as PbTableName, TableRoute as PbTableRoute,
-    };
-
     use super::*;
 
+    #[test]
+    fn test_leader_is_downgraded() {
+        let mut region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_status: None,
+        };
+
+        assert!(!region_route.is_leader_downgraded());
+
+        region_route.downgrade_leader();
+
+        assert!(region_route.is_leader_downgraded());
+    }
+
+    #[test]
+    fn test_region_route_decode() {
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_status: None,
+        };
+
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}]}"#;
+
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+
+        assert_eq!(decoded, region_route);
+    }
+
     #[test]
     fn test_de_serialize_partition() {
         let p = Partition {
@@ -476,182 +405,4 @@ mod tests {
 
         assert_eq!(got, p);
     }
-
-    #[test]
-    fn test_route_request_trans() {
-        let req = RouteRequest {
-            table_ids: vec![1, 2],
-        };
-
-        let into_req: PbRouteRequest = req.into();
-
-        assert!(into_req.header.is_none());
-        assert_eq!(1, into_req.table_ids.get(0).unwrap().id);
-        assert_eq!(2, into_req.table_ids.get(1).unwrap().id);
-    }
-
-    #[test]
-    fn test_route_response_trans() {
-        let res = PbRouteResponse {
-            header: None,
-            peers: vec![
-                PbPeer {
-                    id: 1,
-                    addr: "peer1".to_string(),
-                },
-                PbPeer {
-                    id: 2,
-                    addr: "peer2".to_string(),
-                },
-            ],
-            table_routes: vec![PbTableRoute {
-                table: Some(PbTable {
-                    id: 1,
-                    table_name: Some(PbTableName {
-                        catalog_name: "c1".to_string(),
-                        schema_name: "s1".to_string(),
-                        table_name: "t1".to_string(),
-                    }),
-                    table_schema: b"schema".to_vec(),
-                }),
-                region_routes: vec![PbRegionRoute {
-                    region: Some(PbRegion {
-                        id: 1,
-                        name: "region1".to_string(),
-                        partition: Some(PbPartition {
-                            column_list: vec![b"c1".to_vec(), b"c2".to_vec()],
-                            value_list: vec![b"v1".to_vec(), b"v2".to_vec()],
-                        }),
-                        attrs: Default::default(),
-                    }),
-                    leader_peer_index: 0,
-                    follower_peer_indexes: vec![1],
-                }],
-            }],
-        };
-
-        let res: RouteResponse = res.try_into().unwrap();
-        let mut table_routes = res.table_routes;
-        assert_eq!(1, table_routes.len());
-        let table_route = table_routes.remove(0);
-        let table = table_route.table;
-        assert_eq!(1, table.id);
-        assert_eq!("c1", table.table_name.catalog_name);
-        assert_eq!("s1", table.table_name.schema_name);
-        assert_eq!("t1", table.table_name.table_name);
-
-        let mut region_routes = table_route.region_routes;
-        assert_eq!(1, region_routes.len());
-        let region_route = region_routes.remove(0);
-        let region = region_route.region;
-        assert_eq!(1, region.id);
-        assert_eq!("region1", region.name);
-        let partition = region.partition.unwrap();
-        assert_eq!(vec![b"c1".to_vec(), b"c2".to_vec()], partition.column_list);
-        assert_eq!(vec![b"v1".to_vec(), b"v2".to_vec()], partition.value_list);
-
-        assert_eq!(1, region_route.leader_peer.as_ref().unwrap().id);
-        assert_eq!("peer1", region_route.leader_peer.as_ref().unwrap().addr);
-
-        assert_eq!(1, region_route.follower_peers.len());
-        assert_eq!(2, region_route.follower_peers.get(0).unwrap().id);
-        assert_eq!("peer2", region_route.follower_peers.get(0).unwrap().addr);
-    }
-
-    #[test]
-    fn test_table_route_raw_conversion() {
-        let raw_peers = vec![
-            PbPeer {
-                id: 1,
-                addr: "a1".to_string(),
-            },
-            PbPeer {
-                id: 2,
-                addr: "a2".to_string(),
-            },
-            PbPeer {
-                id: 3,
-                addr: "a3".to_string(),
-            },
-        ];
-
-        // region distribution:
-        // region id => leader peer id + [follower peer id]
-        // 1 => 2 + [1, 3]
-        // 2 => 1 + [2, 3]
-
-        let raw_table_route = PbTableRoute {
-            table: Some(PbTable {
-                id: 1,
-                table_name: Some(PbTableName {
-                    catalog_name: "c1".to_string(),
-                    schema_name: "s1".to_string(),
-                    table_name: "t1".to_string(),
-                }),
-                table_schema: vec![],
-            }),
-            region_routes: vec![
-                PbRegionRoute {
-                    region: Some(PbRegion {
-                        id: 1,
-                        name: "r1".to_string(),
-                        partition: None,
-                        attrs: HashMap::new(),
-                    }),
-                    leader_peer_index: 1,
-                    follower_peer_indexes: vec![0, 2],
-                },
-                PbRegionRoute {
-                    region: Some(PbRegion {
-                        id: 2,
-                        name: "r2".to_string(),
-                        partition: None,
-                        attrs: HashMap::new(),
-                    }),
-                    leader_peer_index: 0,
-                    follower_peer_indexes: vec![1, 2],
-                },
-            ],
-        };
-        let table_route = TableRoute {
-            table: Table {
-                id: 1,
-                table_name: TableName::new("c1", "s1", "t1"),
-                table_schema: vec![],
-            },
-            region_routes: vec![
-                RegionRoute {
-                    region: Region {
-                        id: 1.into(),
-                        name: "r1".to_string(),
-                        partition: None,
-                        attrs: BTreeMap::new(),
-                    },
-                    leader_peer: Some(Peer::new(2, "a2")),
-                    follower_peers: vec![Peer::new(1, "a1"), Peer::new(3, "a3")],
-                },
-                RegionRoute {
-                    region: Region {
-                        id: 2.into(),
-                        name: "r2".to_string(),
-                        partition: None,
-                        attrs: BTreeMap::new(),
-                    },
-                    leader_peer: Some(Peer::new(1, "a1")),
-                    follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
-                },
-            ],
-            region_leaders: HashMap::from([
-                (2, Some(Peer::new(1, "a1"))),
-                (1, Some(Peer::new(2, "a2"))),
-            ]),
-        };
-
-        let from_raw = TableRoute::try_from_raw(&raw_peers, raw_table_route.clone()).unwrap();
-        assert_eq!(from_raw, table_route);
-
-        let into_raw = table_route.try_into_raw().unwrap();
-        assert_eq!(into_raw.0, raw_peers);
-        assert_eq!(into_raw.1, raw_table_route);
-    }
 }

src/common/meta/src/rpc/store.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::fmt::{Display, Formatter};
+use std::ops::Bound;
 
 use api::v1::meta::{
     BatchDeleteRequest as PbBatchDeleteRequest, BatchDeleteResponse as PbBatchDeleteResponse,
@@ -20,8 +21,7 @@ use api::v1::meta::{
     BatchPutRequest as PbBatchPutRequest, BatchPutResponse as PbBatchPutResponse,
     CompareAndPutRequest as PbCompareAndPutRequest,
     CompareAndPutResponse as PbCompareAndPutResponse, DeleteRangeRequest as PbDeleteRangeRequest,
-    DeleteRangeResponse as PbDeleteRangeResponse, MoveValueRequest as PbMoveValueRequest,
-    MoveValueResponse as PbMoveValueResponse, PutRequest as PbPutRequest,
+    DeleteRangeResponse as PbDeleteRangeResponse, PutRequest as PbPutRequest,
     PutResponse as PbPutResponse, RangeRequest as PbRangeRequest, RangeResponse as PbRangeResponse,
     ResponseHeader as PbResponseHeader,
 };
@@ -30,6 +30,17 @@ use crate::error;
 use crate::error::Result;
 use crate::rpc::{util, KeyValue};
 
+pub fn to_range(key: Vec<u8>, range_end: Vec<u8>) -> (Bound<Vec<u8>>, Bound<Vec<u8>>) {
+    match (&key[..], &range_end[..]) {
+        (_, []) => (Bound::Included(key.clone()), Bound::Included(key)),
+        // If both key and range_end are ‘\0’, then range represents all keys.
+        ([0], [0]) => (Bound::Unbounded, Bound::Unbounded),
+        // If range_end is ‘\0’, the range is all keys greater than or equal to the key argument.
+        (_, [0]) => (Bound::Included(key), Bound::Unbounded),
+        (_, _) => (Bound::Included(key), Bound::Excluded(range_end)),
+    }
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct RangeRequest {
     /// key is the first key for the range, If range_end is not given, the
@@ -96,6 +107,11 @@ impl RangeRequest {
         }
     }
 
+    /// Returns the `RangeBounds`.
+    pub fn range(&self) -> (Bound<Vec<u8>>, Bound<Vec<u8>>) {
+        to_range(self.key.clone(), self.range_end.clone())
+    }
+
     /// key is the first key for the range, If range_end is not given, the
     /// request only looks up key.
     #[inline]
@@ -690,6 +706,11 @@ impl DeleteRangeRequest {
         }
     }
 
+    /// Returns the `RangeBounds`.
+    pub fn range(&self) -> (Bound<Vec<u8>>, Bound<Vec<u8>>) {
+        to_range(self.key.clone(), self.range_end.clone())
+    }
+
     /// key is the first key to delete in the range. If range_end is not given,
     /// the range is defined to contain only the key argument.
     #[inline]
@@ -772,71 +793,6 @@ impl DeleteRangeResponse {
     }
 }
 
-#[derive(Debug, Clone, Default)]
-pub struct MoveValueRequest {
-    /// If from_key dose not exist, return the value of to_key (if it exists).
-    /// If from_key exists, move the value of from_key to to_key (i.e. rename),
-    /// and return the value.
-    pub from_key: Vec<u8>,
-    pub to_key: Vec<u8>,
-}
-
-impl From<MoveValueRequest> for PbMoveValueRequest {
-    fn from(req: MoveValueRequest) -> Self {
-        Self {
-            header: None,
-            from_key: req.from_key,
-            to_key: req.to_key,
-        }
-    }
-}
-
-impl From<PbMoveValueRequest> for MoveValueRequest {
-    fn from(value: PbMoveValueRequest) -> Self {
-        Self {
-            from_key: value.from_key,
-            to_key: value.to_key,
-        }
-    }
-}
-
-impl MoveValueRequest {
-    #[inline]
-    pub fn new(from_key: impl Into<Vec<u8>>, to_key: impl Into<Vec<u8>>) -> Self {
-        Self {
-            from_key: from_key.into(),
-            to_key: to_key.into(),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct MoveValueResponse(pub Option<KeyValue>);
-
-impl TryFrom<PbMoveValueResponse> for MoveValueResponse {
-    type Error = error::Error;
-
-    fn try_from(pb: PbMoveValueResponse) -> Result<Self> {
-        util::check_response_header(pb.header.as_ref())?;
-
-        Ok(Self(pb.kv.map(KeyValue::new)))
-    }
-}
-
-impl MoveValueResponse {
-    pub fn to_proto_resp(self, header: PbResponseHeader) -> PbMoveValueResponse {
-        PbMoveValueResponse {
-            header: Some(header),
-            kv: self.0.map(Into::into),
-        }
-    }
-
-    #[inline]
-    pub fn take_kv(&mut self) -> Option<KeyValue> {
-        self.0.take()
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use api::v1::meta::{
@@ -844,10 +800,8 @@ mod tests {
         CompareAndPutRequest as PbCompareAndPutRequest,
         CompareAndPutResponse as PbCompareAndPutResponse,
         DeleteRangeRequest as PbDeleteRangeRequest, DeleteRangeResponse as PbDeleteRangeResponse,
-        KeyValue as PbKeyValue, MoveValueRequest as PbMoveValueRequest,
-        MoveValueResponse as PbMoveValueResponse, PutRequest as PbPutRequest,
-        PutResponse as PbPutResponse, RangeRequest as PbRangeRequest,
-        RangeResponse as PbRangeResponse,
+        KeyValue as PbKeyValue, PutRequest as PbPutRequest, PutResponse as PbPutResponse,
+        RangeRequest as PbRangeRequest, RangeResponse as PbRangeResponse,
     };
 
     use super::*;
@@ -1150,34 +1104,4 @@ mod tests {
         assert_eq!(b"v2".to_vec(), kv1.value().to_vec());
         assert_eq!(b"v2".to_vec(), kv1.take_value());
     }
-
-    #[test]
-    fn test_move_value_request_trans() {
-        let (from_key, to_key) = (b"test_key1".to_vec(), b"test_key2".to_vec());
-
-        let req = MoveValueRequest::new(from_key.clone(), to_key.clone());
-
-        let into_req: PbMoveValueRequest = req.into();
-        assert!(into_req.header.is_none());
-        assert_eq!(from_key, into_req.from_key);
-        assert_eq!(to_key, into_req.to_key);
-    }
-
-    #[test]
-    fn test_move_value_response_trans() {
-        let pb_res = PbMoveValueResponse {
-            header: None,
-            kv: Some(PbKeyValue {
-                key: b"k1".to_vec(),
-                value: b"v1".to_vec(),
-            }),
-        };
-
-        let mut res: MoveValueResponse = pb_res.try_into().unwrap();
-        let mut kv = res.take_kv().unwrap();
-        assert_eq!(b"k1".to_vec(), kv.key().to_vec());
-        assert_eq!(b"k1".to_vec(), kv.take_key());
-        assert_eq!(b"v1".to_vec(), kv.value().to_vec());
-        assert_eq!(b"v1".to_vec(), kv.take_value());
-    }
 }

src/common/meta/src/sequence.rs

@@ -166,8 +166,7 @@ mod tests {
     use crate::rpc::store::{
         BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse,
         BatchPutRequest, BatchPutResponse, CompareAndPutResponse, DeleteRangeRequest,
-        DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
-        RangeRequest, RangeResponse,
+        DeleteRangeResponse, PutRequest, PutResponse, RangeRequest, RangeResponse,
     };
 
     #[tokio::test]
@@ -247,10 +246,6 @@ mod tests {
             async fn batch_delete(&self, _: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
                 unreachable!()
             }
-
-            async fn move_value(&self, _: MoveValueRequest) -> Result<MoveValueResponse> {
-                unreachable!()
-            }
         }
 
         let kv_store = Arc::new(Noop {});

src/common/procedure/src/error.rs

@@ -34,6 +34,9 @@ pub enum Error {
     #[snafu(display("Loader {} is already registered", name))]
     LoaderConflict { name: String, location: Location },
 
+    #[snafu(display("Procedure Manager is stopped"))]
+    ManagerNotStart { location: Location },
+
     #[snafu(display("Failed to serialize to json"))]
     ToJson {
         #[snafu(source)]
@@ -148,7 +151,8 @@ impl ErrorExt for Error {
             | Error::FromJson { .. }
             | Error::RetryTimesExceeded { .. }
             | Error::RetryLater { .. }
-            | Error::WaitWatcher { .. } => StatusCode::Internal,
+            | Error::WaitWatcher { .. }
+            | Error::ManagerNotStart { .. } => StatusCode::Internal,
             Error::LoaderConflict { .. } | Error::DuplicateProcedure { .. } => {
                 StatusCode::InvalidArguments
             }

src/common/procedure/src/lib.rs

@@ -14,6 +14,8 @@
 
 //! Common traits and structures for the procedure framework.
 
+#![feature(assert_matches)]
+
 pub mod error;
 pub mod local;
 pub mod options;

src/common/procedure/src/local.rs

@@ -16,20 +16,21 @@ mod lock;
 mod runner;
 
 use std::collections::{HashMap, VecDeque};
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::{Duration, Instant};
 
 use async_trait::async_trait;
 use backon::ExponentialBuilder;
 use common_runtime::{RepeatedTask, TaskFunction};
-use common_telemetry::logging;
+use common_telemetry::{info, logging};
 use snafu::{ensure, ResultExt};
 use tokio::sync::watch::{self, Receiver, Sender};
-use tokio::sync::Notify;
+use tokio::sync::{Mutex as TokioMutex, Notify};
 
 use crate::error::{
-    DuplicateProcedureSnafu, Error, LoaderConflictSnafu, Result, StartRemoveOutdatedMetaTaskSnafu,
-    StopRemoveOutdatedMetaTaskSnafu,
+    DuplicateProcedureSnafu, Error, LoaderConflictSnafu, ManagerNotStartSnafu, Result,
+    StartRemoveOutdatedMetaTaskSnafu, StopRemoveOutdatedMetaTaskSnafu,
 };
 use crate::local::lock::LockMap;
 use crate::local::runner::Runner;
@@ -135,6 +136,8 @@ pub(crate) struct ManagerContext {
     messages: Mutex<HashMap<ProcedureId, ProcedureMessage>>,
     /// Ids and finished time of finished procedures.
     finished_procedures: Mutex<VecDeque<(ProcedureId, Instant)>>,
+    /// Running flag.
+    running: Arc<AtomicBool>,
 }
 
 #[async_trait]
@@ -153,9 +156,29 @@ impl ManagerContext {
             procedures: RwLock::new(HashMap::new()),
             messages: Mutex::new(HashMap::new()),
             finished_procedures: Mutex::new(VecDeque::new()),
+            running: Arc::new(AtomicBool::new(false)),
         }
     }
 
+    #[cfg(test)]
+    pub(crate) fn set_running(&self) {
+        self.running.store(true, Ordering::Relaxed);
+    }
+
+    /// Set the running flag.
+    pub(crate) fn start(&self) {
+        self.running.store(true, Ordering::Relaxed);
+    }
+
+    pub(crate) fn stop(&self) {
+        self.running.store(false, Ordering::Relaxed);
+    }
+
+    /// Return `ProcedureManager` is running.
+    pub(crate) fn running(&self) -> bool {
+        self.running.load(Ordering::Relaxed)
+    }
+
     /// Returns true if the procedure with specific `procedure_id` exists.
     fn contains_procedure(&self, procedure_id: ProcedureId) -> bool {
         let procedures = self.procedures.read().unwrap();
@@ -368,29 +391,37 @@ pub struct LocalManager {
     procedure_store: Arc<ProcedureStore>,
     max_retry_times: usize,
     retry_delay: Duration,
-    remove_outdated_meta_task: RepeatedTask<Error>,
+    /// GC task.
+    remove_outdated_meta_task: TokioMutex<Option<RepeatedTask<Error>>>,
+    config: ManagerConfig,
 }
 
 impl LocalManager {
     /// Create a new [LocalManager] with specific `config`.
     pub fn new(config: ManagerConfig, state_store: StateStoreRef) -> LocalManager {
         let manager_ctx = Arc::new(ManagerContext::new());
-        let remove_outdated_meta_task = RepeatedTask::new(
-            config.remove_outdated_meta_task_interval,
-            Box::new(RemoveOutdatedMetaFunction {
-                manager_ctx: manager_ctx.clone(),
-                ttl: config.remove_outdated_meta_ttl,
-            }),
-        );
+
         LocalManager {
             manager_ctx,
             procedure_store: Arc::new(ProcedureStore::new(&config.parent_path, state_store)),
             max_retry_times: config.max_retry_times,
             retry_delay: config.retry_delay,
-            remove_outdated_meta_task,
+            remove_outdated_meta_task: TokioMutex::new(None),
+            config,
         }
     }
 
+    /// Build remove outedated meta task
+    pub fn build_remove_outdated_meta_task(&self) -> RepeatedTask<Error> {
+        RepeatedTask::new(
+            self.config.remove_outdated_meta_task_interval,
+            Box::new(RemoveOutdatedMetaFunction {
+                manager_ctx: self.manager_ctx.clone(),
+                ttl: self.config.remove_outdated_meta_ttl,
+            }),
+        )
+    }
+
     /// Submit a root procedure with given `procedure_id`.
     fn submit_root(
         &self,
@@ -398,6 +429,8 @@ impl LocalManager {
         step: u32,
         procedure: BoxedProcedure,
     ) -> Result<Watcher> {
+        ensure!(self.manager_ctx.running(), ManagerNotStartSnafu);
+
         let meta = Arc::new(ProcedureMeta::new(procedure_id, None, procedure.lock_key()));
         let runner = Runner {
             meta: meta.clone(),
@@ -426,44 +459,8 @@ impl LocalManager {
 
         Ok(watcher)
     }
-}
-
-#[async_trait]
-impl ProcedureManager for LocalManager {
-    fn register_loader(&self, name: &str, loader: BoxedProcedureLoader) -> Result<()> {
-        let mut loaders = self.manager_ctx.loaders.lock().unwrap();
-        ensure!(!loaders.contains_key(name), LoaderConflictSnafu { name });
-
-        let _ = loaders.insert(name.to_string(), loader);
-
-        Ok(())
-    }
-
-    fn start(&self) -> Result<()> {
-        self.remove_outdated_meta_task
-            .start(common_runtime::bg_runtime())
-            .context(StartRemoveOutdatedMetaTaskSnafu)?;
-        Ok(())
-    }
-
-    async fn stop(&self) -> Result<()> {
-        self.remove_outdated_meta_task
-            .stop()
-            .await
-            .context(StopRemoveOutdatedMetaTaskSnafu)?;
-        Ok(())
-    }
-
-    async fn submit(&self, procedure: ProcedureWithId) -> Result<Watcher> {
-        let procedure_id = procedure.id;
-        ensure!(
-            !self.manager_ctx.contains_procedure(procedure_id),
-            DuplicateProcedureSnafu { procedure_id }
-        );
-
-        self.submit_root(procedure.id, 0, procedure.procedure)
-    }
 
+    /// Recovers unfinished procedures and reruns them.
     async fn recover(&self) -> Result<()> {
         logging::info!("LocalManager start to recover");
         let recover_start = Instant::now();
@@ -519,6 +516,64 @@ impl ProcedureManager for LocalManager {
 
         Ok(())
     }
+}
+
+#[async_trait]
+impl ProcedureManager for LocalManager {
+    fn register_loader(&self, name: &str, loader: BoxedProcedureLoader) -> Result<()> {
+        let mut loaders = self.manager_ctx.loaders.lock().unwrap();
+        ensure!(!loaders.contains_key(name), LoaderConflictSnafu { name });
+
+        let _ = loaders.insert(name.to_string(), loader);
+
+        Ok(())
+    }
+
+    async fn start(&self) -> Result<()> {
+        let mut task = self.remove_outdated_meta_task.lock().await;
+
+        if task.is_some() {
+            return Ok(());
+        }
+
+        let task_inner = self.build_remove_outdated_meta_task();
+
+        task_inner
+            .start(common_runtime::bg_runtime())
+            .context(StartRemoveOutdatedMetaTaskSnafu)?;
+
+        *task = Some(task_inner);
+
+        self.manager_ctx.start();
+
+        info!("LocalManager is start.");
+
+        self.recover().await
+    }
+
+    async fn stop(&self) -> Result<()> {
+        let mut task = self.remove_outdated_meta_task.lock().await;
+
+        if let Some(task) = task.take() {
+            task.stop().await.context(StopRemoveOutdatedMetaTaskSnafu)?;
+        }
+
+        self.manager_ctx.stop();
+
+        info!("LocalManager is stopped.");
+
+        Ok(())
+    }
+
+    async fn submit(&self, procedure: ProcedureWithId) -> Result<Watcher> {
+        let procedure_id = procedure.id;
+        ensure!(
+            !self.manager_ctx.contains_procedure(procedure_id),
+            DuplicateProcedureSnafu { procedure_id }
+        );
+
+        self.submit_root(procedure.id, 0, procedure.procedure)
+    }
 
     async fn procedure_state(&self, procedure_id: ProcedureId) -> Result<Option<ProcedureState>> {
         Ok(self.manager_ctx.state(procedure_id))
@@ -569,12 +624,14 @@ pub(crate) mod test_util {
 
 #[cfg(test)]
 mod tests {
+    use std::assert_matches::assert_matches;
+
     use common_error::mock::MockError;
     use common_error::status_code::StatusCode;
     use common_test_util::temp_dir::create_temp_dir;
 
     use super::*;
-    use crate::error::Error;
+    use crate::error::{self, Error};
     use crate::store::state_store::ObjectStateStore;
     use crate::{Context, Procedure, Status};
 
@@ -691,6 +748,7 @@ mod tests {
         };
         let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
         let manager = LocalManager::new(config, state_store);
+        manager.manager_ctx.start();
 
         manager
             .register_loader("ProcedureToLoad", ProcedureToLoad::loader())
@@ -714,6 +772,7 @@ mod tests {
         };
         let state_store = Arc::new(ObjectStateStore::new(object_store.clone()));
         let manager = LocalManager::new(config, state_store);
+        manager.manager_ctx.start();
 
         manager
             .register_loader("ProcedureToLoad", ProcedureToLoad::loader())
@@ -762,6 +821,7 @@ mod tests {
         };
         let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
         let manager = LocalManager::new(config, state_store);
+        manager.manager_ctx.start();
 
         let procedure_id = ProcedureId::random();
         assert!(manager
@@ -812,6 +872,7 @@ mod tests {
         };
         let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
         let manager = LocalManager::new(config, state_store);
+        manager.manager_ctx.start();
 
         #[derive(Debug)]
         struct MockProcedure {
@@ -864,6 +925,66 @@ mod tests {
     }
 
     #[tokio::test]
+    async fn test_procedure_manager_stopped() {
+        let dir = create_temp_dir("procedure_manager_stopped");
+        let config = ManagerConfig {
+            parent_path: "data/".to_string(),
+            max_retry_times: 3,
+            retry_delay: Duration::from_millis(500),
+            ..Default::default()
+        };
+        let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
+        let manager = LocalManager::new(config, state_store);
+
+        let mut procedure = ProcedureToLoad::new("submit");
+        procedure.lock_key = LockKey::single("test.submit");
+        let procedure_id = ProcedureId::random();
+        assert_matches!(
+            manager
+                .submit(ProcedureWithId {
+                    id: procedure_id,
+                    procedure: Box::new(procedure),
+                })
+                .await
+                .unwrap_err(),
+            error::Error::ManagerNotStart { .. }
+        );
+    }
+
+    #[tokio::test]
+    async fn test_procedure_manager_restart() {
+        let dir = create_temp_dir("procedure_manager_restart");
+        let config = ManagerConfig {
+            parent_path: "data/".to_string(),
+            max_retry_times: 3,
+            retry_delay: Duration::from_millis(500),
+            ..Default::default()
+        };
+        let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
+        let manager = LocalManager::new(config, state_store);
+
+        manager.start().await.unwrap();
+        manager.stop().await.unwrap();
+        manager.start().await.unwrap();
+
+        let mut procedure = ProcedureToLoad::new("submit");
+        procedure.lock_key = LockKey::single("test.submit");
+        let procedure_id = ProcedureId::random();
+        assert!(manager
+            .submit(ProcedureWithId {
+                id: procedure_id,
+                procedure: Box::new(procedure),
+            })
+            .await
+            .is_ok());
+        assert!(manager
+            .procedure_state(procedure_id)
+            .await
+            .unwrap()
+            .is_some());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_remove_outdated_meta_task() {
         let dir = create_temp_dir("remove_outdated_meta_task");
         let object_store = test_util::new_object_store(&dir);
@@ -876,6 +997,7 @@ mod tests {
         };
         let state_store = Arc::new(ObjectStateStore::new(object_store.clone()));
         let manager = LocalManager::new(config, state_store);
+        manager.manager_ctx.set_running();
 
         let mut procedure = ProcedureToLoad::new("submit");
         procedure.lock_key = LockKey::single("test.submit");
@@ -889,8 +1011,9 @@ mod tests {
             .is_ok());
         let mut watcher = manager.procedure_watcher(procedure_id).unwrap();
         watcher.changed().await.unwrap();
-        manager.start().unwrap();
-        tokio::time::sleep(Duration::from_millis(10)).await;
+
+        manager.start().await.unwrap();
+        tokio::time::sleep(Duration::from_millis(300)).await;
         assert!(manager
             .procedure_state(procedure_id)
             .await
@@ -902,6 +1025,8 @@ mod tests {
         let mut procedure = ProcedureToLoad::new("submit");
         procedure.lock_key = LockKey::single("test.submit");
         let procedure_id = ProcedureId::random();
+
+        manager.manager_ctx.set_running();
         assert!(manager
             .submit(ProcedureWithId {
                 id: procedure_id,
@@ -911,11 +1036,33 @@ mod tests {
             .is_ok());
         let mut watcher = manager.procedure_watcher(procedure_id).unwrap();
         watcher.changed().await.unwrap();
-        tokio::time::sleep(Duration::from_millis(10)).await;
+        tokio::time::sleep(Duration::from_millis(300)).await;
         assert!(manager
             .procedure_state(procedure_id)
             .await
             .unwrap()
             .is_some());
+
+        // After restart
+        let mut procedure = ProcedureToLoad::new("submit");
+        procedure.lock_key = LockKey::single("test.submit");
+        let procedure_id = ProcedureId::random();
+        assert!(manager
+            .submit(ProcedureWithId {
+                id: procedure_id,
+                procedure: Box::new(procedure),
+            })
+            .await
+            .is_ok());
+        let mut watcher = manager.procedure_watcher(procedure_id).unwrap();
+        watcher.changed().await.unwrap();
+
+        manager.start().await.unwrap();
+        tokio::time::sleep(Duration::from_millis(300)).await;
+        assert!(manager
+            .procedure_state(procedure_id)
+            .await
+            .unwrap()
+            .is_none());
     }
 }

src/common/procedure/src/local/runner.rs

@@ -19,7 +19,7 @@ use backon::{BackoffBuilder, ExponentialBuilder};
 use common_telemetry::logging;
 use tokio::time;
 
-use crate::error::{ProcedurePanicSnafu, Result};
+use crate::error::{self, ProcedurePanicSnafu, Result};
 use crate::local::{ManagerContext, ProcedureMeta, ProcedureMetaRef};
 use crate::store::ProcedureStore;
 use crate::ProcedureState::Retrying;
@@ -102,7 +102,6 @@ impl Drop for ProcedureGuard {
     }
 }
 
-// TODO(yingwen): Support cancellation.
 pub(crate) struct Runner {
     pub(crate) meta: ProcedureMetaRef,
     pub(crate) procedure: BoxedProcedure,
@@ -114,6 +113,11 @@ pub(crate) struct Runner {
 }
 
 impl Runner {
+    /// Return `ProcedureManager` is running.
+    pub(crate) fn running(&self) -> bool {
+        self.manager_ctx.running()
+    }
+
     /// Run the procedure.
     pub(crate) async fn run(mut self) {
         // Ensure we can update the procedure state.
@@ -152,6 +156,12 @@ impl Runner {
             let procedure_ids = self.manager_ctx.procedures_in_tree(&self.meta);
             // Clean resources.
             self.manager_ctx.on_procedures_finish(&procedure_ids);
+
+            // If `ProcedureManager` is stopped, it stops the current task immediately without deleting the procedure.
+            if !self.running() {
+                return;
+            }
+
             for id in procedure_ids {
                 if let Err(e) = self.store.delete_procedure(id).await {
                     logging::error!(
@@ -186,6 +196,13 @@ impl Runner {
         let mut retry = self.exponential_builder.build();
         let mut retry_times = 0;
         loop {
+            // Don't store state if `ProcedureManager` is stopped.
+            if !self.running() {
+                self.meta.set_state(ProcedureState::Failed {
+                    error: Arc::new(error::ManagerNotStartSnafu {}.build()),
+                });
+                return;
+            }
             match self.execute_once(ctx).await {
                 ExecResult::Done | ExecResult::Failed => return,
                 ExecResult::Continue => (),
@@ -238,6 +255,14 @@ impl Runner {
                     status.need_persist(),
                 );
 
+                // Don't store state if `ProcedureManager` is stopped.
+                if !self.running() {
+                    self.meta.set_state(ProcedureState::Failed {
+                        error: Arc::new(error::ManagerNotStartSnafu {}.build()),
+                    });
+                    return ExecResult::Failed;
+                }
+
                 if status.need_persist() {
                     if let Err(err) = self.persist_procedure().await {
                         self.meta.set_state(ProcedureState::retrying(Arc::new(err)));
@@ -272,6 +297,14 @@ impl Runner {
                     e.is_retry_later(),
                 );
 
+                // Don't store state if `ProcedureManager` is stopped.
+                if !self.running() {
+                    self.meta.set_state(ProcedureState::Failed {
+                        error: Arc::new(error::ManagerNotStartSnafu {}.build()),
+                    });
+                    return ExecResult::Failed;
+                }
+
                 if e.is_retry_later() {
                     self.meta.set_state(ProcedureState::retrying(Arc::new(e)));
                     return ExecResult::RetryLater;
@@ -581,6 +614,7 @@ mod tests {
         let object_store = test_util::new_object_store(&dir);
         let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
         let mut runner = new_runner(meta, Box::new(normal), procedure_store.clone());
+        runner.manager_ctx.start();
 
         let res = runner.execute_once(&ctx).await;
         assert!(res.is_continue(), "{res:?}");
@@ -641,6 +675,7 @@ mod tests {
         let object_store = test_util::new_object_store(&dir);
         let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
         let mut runner = new_runner(meta, Box::new(suspend), procedure_store);
+        runner.manager_ctx.start();
 
         let res = runner.execute_once(&ctx).await;
         assert!(res.is_continue(), "{res:?}");
@@ -742,6 +777,7 @@ mod tests {
         let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
         let mut runner = new_runner(meta.clone(), Box::new(parent), procedure_store.clone());
         let manager_ctx = Arc::new(ManagerContext::new());
+        manager_ctx.start();
         // Manually add this procedure to the manager ctx.
         assert!(manager_ctx.try_insert_procedure(meta));
         // Replace the manager ctx.
@@ -769,6 +805,70 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_running_is_stopped() {
+        let exec_fn = move |_| async move { Ok(Status::Executing { persist: true }) }.boxed();
+        let normal = ProcedureAdapter {
+            data: "normal".to_string(),
+            lock_key: LockKey::single("catalog.schema.table"),
+            exec_fn,
+        };
+
+        let dir = create_temp_dir("test_running_is_stopped");
+        let meta = normal.new_meta(ROOT_ID);
+        let ctx = context_without_provider(meta.id);
+        let object_store = test_util::new_object_store(&dir);
+        let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
+        let mut runner = new_runner(meta, Box::new(normal), procedure_store.clone());
+        runner.manager_ctx.start();
+
+        let res = runner.execute_once(&ctx).await;
+        assert!(res.is_continue(), "{res:?}");
+        check_files(
+            &object_store,
+            &procedure_store,
+            ctx.procedure_id,
+            &["0000000000.step"],
+        )
+        .await;
+
+        runner.manager_ctx.stop();
+        let res = runner.execute_once(&ctx).await;
+        assert!(res.is_failed());
+        // Shouldn't write any files
+        check_files(
+            &object_store,
+            &procedure_store,
+            ctx.procedure_id,
+            &["0000000000.step"],
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    async fn test_running_is_stopped_on_error() {
+        let exec_fn =
+            |_| async { Err(Error::external(MockError::new(StatusCode::Unexpected))) }.boxed();
+        let normal = ProcedureAdapter {
+            data: "fail".to_string(),
+            lock_key: LockKey::single("catalog.schema.table"),
+            exec_fn,
+        };
+
+        let dir = create_temp_dir("test_running_is_stopped_on_error");
+        let meta = normal.new_meta(ROOT_ID);
+        let ctx = context_without_provider(meta.id);
+        let object_store = test_util::new_object_store(&dir);
+        let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
+        let mut runner = new_runner(meta, Box::new(normal), procedure_store.clone());
+        runner.manager_ctx.stop();
+
+        let res = runner.execute_once(&ctx).await;
+        assert!(res.is_failed(), "{res:?}");
+        // Shouldn't write any files
+        check_files(&object_store, &procedure_store, ctx.procedure_id, &[]).await;
+    }
+
     #[tokio::test]
     async fn test_execute_on_error() {
         let exec_fn =
@@ -785,6 +885,7 @@ mod tests {
         let object_store = test_util::new_object_store(&dir);
         let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
         let mut runner = new_runner(meta.clone(), Box::new(fail), procedure_store.clone());
+        runner.manager_ctx.start();
 
         let res = runner.execute_once(&ctx).await;
         assert!(res.is_failed(), "{res:?}");
@@ -826,6 +927,7 @@ mod tests {
         let object_store = test_util::new_object_store(&dir);
         let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
         let mut runner = new_runner(meta.clone(), Box::new(retry_later), procedure_store.clone());
+        runner.manager_ctx.start();
 
         let res = runner.execute_once(&ctx).await;
         assert!(res.is_retry_later(), "{res:?}");
@@ -863,6 +965,8 @@ mod tests {
             Box::new(exceed_max_retry_later),
             procedure_store,
         );
+        runner.manager_ctx.start();
+
         runner.exponential_builder = ExponentialBuilder::default()
             .with_min_delay(Duration::from_millis(1))
             .with_max_times(3);
@@ -933,8 +1037,8 @@ mod tests {
         let object_store = test_util::new_object_store(&dir);
         let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
         let mut runner = new_runner(meta.clone(), Box::new(parent), procedure_store);
-
         let manager_ctx = Arc::new(ManagerContext::new());
+        manager_ctx.start();
         // Manually add this procedure to the manager ctx.
         assert!(manager_ctx.try_insert_procedure(meta.clone()));
         // Replace the manager ctx.

src/common/procedure/src/procedure.rs

@@ -279,8 +279,14 @@ pub trait ProcedureManager: Send + Sync + 'static {
     /// Registers loader for specific procedure type `name`.
     fn register_loader(&self, name: &str, loader: BoxedProcedureLoader) -> Result<()>;
 
-    fn start(&self) -> Result<()>;
+    /// Starts the background GC task.
+    ///
+    /// Recovers unfinished procedures and reruns them.
+    ///
+    /// Callers should ensure all loaders are registered.
+    async fn start(&self) -> Result<()>;
 
+    /// Stops the background GC task.
     async fn stop(&self) -> Result<()>;
 
     /// Submits a procedure to execute.
@@ -288,11 +294,6 @@ pub trait ProcedureManager: Send + Sync + 'static {
     /// Returns a [Watcher] to watch the created procedure.
     async fn submit(&self, procedure: ProcedureWithId) -> Result<Watcher>;
 
-    /// Recovers unfinished procedures and reruns them.
-    ///
-    /// Callers should ensure all loaders are registered.
-    async fn recover(&self) -> Result<()>;
-
     /// Query the procedure state.
     ///
     /// Returns `Ok(None)` if the procedure doesn't exist.

src/common/procedure/src/watcher.rs

@@ -71,6 +71,7 @@ mod tests {
         };
         let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
         let manager = LocalManager::new(config, state_store);
+        manager.start().await.unwrap();
 
         #[derive(Debug)]
         struct MockProcedure {

src/common/query/src/columnar_value.rs

@@ -33,7 +33,7 @@ impl ColumnarValue {
         match self {
             ColumnarValue::Vector(vector) => vector.data_type(),
             ColumnarValue::Scalar(scalar_value) => {
-                ConcreteDataType::from_arrow_type(&scalar_value.get_datatype())
+                ConcreteDataType::from_arrow_type(&scalar_value.data_type())
             }
         }
     }

src/common/recordbatch/src/recordbatch.rs

@@ -13,8 +13,10 @@
 // limitations under the License.
 
 use std::collections::HashMap;
+use std::slice;
 use std::sync::Arc;
 
+use datafusion::arrow::util::pretty::pretty_format_batches;
 use datatypes::schema::SchemaRef;
 use datatypes::value::Value;
 use datatypes::vectors::{Helper, VectorRef};
@@ -169,6 +171,13 @@ impl RecordBatch {
 
         Ok(vectors)
     }
+
+    /// Pretty display this record batch like a table
+    pub fn pretty_print(&self) -> String {
+        pretty_format_batches(slice::from_ref(&self.df_record_batch))
+            .map(|t| t.to_string())
+            .unwrap_or("failed to pretty display a record batch".to_string())
+    }
 }
 
 impl Serialize for RecordBatch {

src/common/runtime/Cargo.toml

@@ -9,9 +9,10 @@ async-trait.workspace = true
 common-error = { workspace = true }
 common-macro = { workspace = true }
 common-telemetry = { workspace = true }
-metrics.workspace = true
+lazy_static.workspace = true
 once_cell.workspace = true
 paste.workspace = true
+prometheus.workspace = true
 snafu.workspace = true
 tokio-util.workspace = true
 tokio.workspace = true

src/common/runtime/src/metrics.rs

@@ -13,6 +13,22 @@
 // limitations under the License.
 
 //! Runtime metrics
-pub const THREAD_NAME_LABEL: &str = "thread.name";
-pub const METRIC_RUNTIME_THREADS_ALIVE: &str = "runtime.threads.alive";
-pub const METRIC_RUNTIME_THREADS_IDLE: &str = "runtime.threads.idle";
+use lazy_static::lazy_static;
+use prometheus::*;
+
+pub const THREAD_NAME_LABEL: &str = "thread_name";
+
+lazy_static! {
+    pub static ref METRIC_RUNTIME_THREADS_ALIVE: IntGaugeVec = register_int_gauge_vec!(
+        "runtime_threads_alive",
+        "runtime threads alive",
+        &[THREAD_NAME_LABEL]
+    )
+    .unwrap();
+    pub static ref METRIC_RUNTIME_THREADS_IDLE: IntGaugeVec = register_int_gauge_vec!(
+        "runtime_threads_idle",
+        "runtime threads idle",
+        &[THREAD_NAME_LABEL]
+    )
+    .unwrap();
+}

src/common/runtime/src/repeated_task.rs

@@ -40,6 +40,7 @@ pub type BoxedTaskFunction<E> = Box<dyn TaskFunction<E> + Send + Sync + 'static>
 struct TaskInner<E> {
     /// The repeated task handle. This handle is Some if the task is started.
     task_handle: Option<JoinHandle<()>>,
+
     /// The task_fn to run. This is Some if the task is not started.
     task_fn: Option<BoxedTaskFunction<E>>,
 }
@@ -50,6 +51,7 @@ pub struct RepeatedTask<E> {
     inner: Mutex<TaskInner<E>>,
     started: AtomicBool,
     interval: Duration,
+    initial_delay: Option<Duration>,
 }
 
 impl<E> std::fmt::Display for RepeatedTask<E> {
@@ -75,6 +77,9 @@ impl<E> Drop for RepeatedTask<E> {
 }
 
 impl<E: ErrorExt + 'static> RepeatedTask<E> {
+    /// Creates a new repeated task. The `initial_delay` is the delay before the first execution.
+    /// `initial_delay` default is None, the initial interval uses the `interval`.
+    /// You can use `with_initial_delay` to set the `initial_delay`.
     pub fn new(interval: Duration, task_fn: BoxedTaskFunction<E>) -> Self {
         Self {
             name: task_fn.name().to_string(),
@@ -85,9 +90,15 @@ impl<E: ErrorExt + 'static> RepeatedTask<E> {
             }),
             started: AtomicBool::new(false),
             interval,
+            initial_delay: None,
         }
     }
 
+    pub fn with_initial_delay(mut self, initial_delay: Option<Duration>) -> Self {
+        self.initial_delay = initial_delay;
+        self
+    }
+
     pub fn started(&self) -> bool {
         self.started.load(Ordering::Relaxed)
     }
@@ -99,17 +110,21 @@ impl<E: ErrorExt + 'static> RepeatedTask<E> {
             IllegalStateSnafu { name: &self.name }
         );
 
-        let interval = self.interval;
         let child = self.cancel_token.child_token();
         // Safety: The task is not started.
         let mut task_fn = inner.task_fn.take().unwrap();
+        let interval = self.interval;
+        let mut initial_delay = self.initial_delay;
         // TODO(hl): Maybe spawn to a blocking runtime.
         let handle = runtime.spawn(async move {
             loop {
-                tokio::select! {
-                    _ = tokio::time::sleep(interval) => {}
-                    _ = child.cancelled() => {
-                        return;
+                let sleep_time = initial_delay.take().unwrap_or(interval);
+                if sleep_time > Duration::ZERO {
+                    tokio::select! {
+                        _ = tokio::time::sleep(sleep_time) => {}
+                        _ = child.cancelled() => {
+                            return;
+                        }
                     }
                 }
                 if let Err(e) = task_fn.call().await {
@@ -192,4 +207,21 @@ mod tests {
 
         assert_eq!(n.load(Ordering::Relaxed), 5);
     }
+
+    #[tokio::test]
+    async fn test_repeated_task_prior_exec() {
+        common_telemetry::init_default_ut_logging();
+
+        let n = Arc::new(AtomicI32::new(0));
+        let task_fn = TickTask { n: n.clone() };
+
+        let task = RepeatedTask::new(Duration::from_millis(100), Box::new(task_fn))
+            .with_initial_delay(Some(Duration::ZERO));
+
+        task.start(crate::bg_runtime()).unwrap();
+        tokio::time::sleep(Duration::from_millis(550)).await;
+        task.stop().await.unwrap();
+
+        assert_eq!(n.load(Ordering::Relaxed), 6);
+    }
 }

src/common/runtime/src/runtime.rs

@@ -18,7 +18,6 @@ use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 
-use metrics::{decrement_gauge, increment_gauge};
 use snafu::ResultExt;
 use tokio::runtime::{Builder as RuntimeBuilder, Handle};
 use tokio::sync::oneshot;
@@ -172,29 +171,33 @@ impl Builder {
 
 fn on_thread_start(thread_name: String) -> impl Fn() + 'static {
     move || {
-        let labels = [(THREAD_NAME_LABEL, thread_name.clone())];
-        increment_gauge!(METRIC_RUNTIME_THREADS_ALIVE, 1.0, &labels);
+        METRIC_RUNTIME_THREADS_ALIVE
+            .with_label_values(&[thread_name.as_str()])
+            .inc();
     }
 }
 
 fn on_thread_stop(thread_name: String) -> impl Fn() + 'static {
     move || {
-        let labels = [(THREAD_NAME_LABEL, thread_name.clone())];
-        decrement_gauge!(METRIC_RUNTIME_THREADS_ALIVE, 1.0, &labels);
+        METRIC_RUNTIME_THREADS_ALIVE
+            .with_label_values(&[thread_name.as_str()])
+            .dec();
     }
 }
 
 fn on_thread_park(thread_name: String) -> impl Fn() + 'static {
     move || {
-        let labels = [(THREAD_NAME_LABEL, thread_name.clone())];
-        increment_gauge!(METRIC_RUNTIME_THREADS_IDLE, 1.0, &labels);
+        METRIC_RUNTIME_THREADS_IDLE
+            .with_label_values(&[thread_name.as_str()])
+            .inc();
     }
 }
 
 fn on_thread_unpark(thread_name: String) -> impl Fn() + 'static {
     move || {
-        let labels = [(THREAD_NAME_LABEL, thread_name.clone())];
-        decrement_gauge!(METRIC_RUNTIME_THREADS_IDLE, 1.0, &labels);
+        METRIC_RUNTIME_THREADS_IDLE
+            .with_label_values(&[thread_name.as_str()])
+            .dec();
     }
 }
 
@@ -204,14 +207,13 @@ mod tests {
     use std::thread;
     use std::time::Duration;
 
-    use common_telemetry::metric;
+    use common_telemetry::dump_metrics;
     use tokio::sync::oneshot;
     use tokio_test::assert_ok;
 
     use super::*;
 
     fn runtime() -> Arc<Runtime> {
-        common_telemetry::init_default_metrics_recorder();
         let runtime = Builder::default()
             .worker_threads(2)
             .thread_name("test_spawn_join")
@@ -221,7 +223,6 @@ mod tests {
 
     #[test]
     fn test_metric() {
-        common_telemetry::init_default_metrics_recorder();
         let runtime = Builder::default()
             .worker_threads(5)
             .thread_name("test_runtime_metric")
@@ -236,8 +237,7 @@ mod tests {
 
         thread::sleep(Duration::from_millis(10));
 
-        let handle = metric::try_handle().unwrap();
-        let metric_text = handle.render();
+        let metric_text = dump_metrics().unwrap();
 
         assert!(metric_text.contains("runtime_threads_idle{thread_name=\"test_runtime_metric\"}"));
         assert!(metric_text.contains("runtime_threads_alive{thread_name=\"test_runtime_metric\"}"));