ci: refine release-cn-artifacts action (#2600)

* ci: add copy-image.sh and upload-artifacts-to-s3.sh

* ci: remove unused options in dev build

* ci: use 'upload-artifacts-to-s3.sh' and 'copy-image.sh' in release-cn-artifacts action

* refactor: refine copy-image.sh

.github/actions/build-dev-builder-image/action.yml

@@ -1,70 +0,0 @@
-name: Build and push dev-builder image
-description: Build and push dev-builder image to DockerHub and ACR
-inputs:
-  dockerhub-image-registry:
-    description: The dockerhub image registry to store the images
-    required: false
-    default: docker.io
-  dockerhub-image-registry-username:
-    description: The dockerhub username to login to the image registry
-    required: true
-  dockerhub-image-registry-token:
-    description: The dockerhub token to login to the image registry
-    required: true
-  dockerhub-image-namespace:
-    description: The dockerhub namespace of the image registry to store the images
-    required: false
-    default: greptime
-  acr-image-registry:
-    description: The ACR image registry to store the images
-    required: true
-  acr-image-registry-username:
-    description: The ACR username to login to the image registry
-    required: true
-  acr-image-registry-password:
-    description: The ACR password to login to the image registry
-    required: true
-  acr-image-namespace:
-    description: The ACR namespace of the image registry to store the images
-    required: false
-    default: greptime
-  version:
-    description: Version of the dev-builder
-    required: false
-    default: latest
-runs:
-  using: composite
-  steps:
-    - name: Login to Dockerhub
-      uses: docker/login-action@v2
-      with:
-        registry: ${{ inputs.dockerhub-image-registry }}
-        username: ${{ inputs.dockerhub-image-registry-username }}
-        password: ${{ inputs.dockerhub-image-registry-token }}
-
-    - name: Build and push dev builder image to dockerhub
-      shell: bash
-      run:
-        make dev-builder \
-        BUILDX_MULTI_PLATFORM_BUILD=true \
-        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
-        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-        IMAGE_TAG=${{ inputs.version }}
-
-    - name: Login to ACR
-      uses: docker/login-action@v2
-      continue-on-error: true
-      with:
-        registry: ${{ inputs.acr-image-registry }}
-        username: ${{ inputs.acr-image-registry-username }}
-        password: ${{ inputs.acr-image-registry-password }}
-
-    - name: Build and push dev builder image to ACR
-      shell: bash
-      continue-on-error: true
-      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
-        make dev-builder \
-        BUILDX_MULTI_PLATFORM_BUILD=true \
-        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
-        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
-        IMAGE_TAG=${{ inputs.version }}

.github/actions/build-dev-builder-images/action.yml

@@ -0,0 +1,76 @@
+name: Build and push dev-builder images
+description: Build and push dev-builder images to DockerHub and ACR
+inputs:
+  dockerhub-image-registry:
+    description: The dockerhub image registry to store the images
+    required: false
+    default: docker.io
+  dockerhub-image-registry-username:
+    description: The dockerhub username to login to the image registry
+    required: true
+  dockerhub-image-registry-token:
+    description: The dockerhub token to login to the image registry
+    required: true
+  dockerhub-image-namespace:
+    description: The dockerhub namespace of the image registry to store the images
+    required: false
+    default: greptime
+  version:
+    description: Version of the dev-builder
+    required: false
+    default: latest
+  build-dev-builder-ubuntu:
+    description: Build dev-builder-ubuntu image
+    required: false
+    default: 'true'
+  build-dev-builder-centos:
+    description: Build dev-builder-centos image
+    required: false
+    default: 'true'
+  build-dev-builder-android:
+    description: Build dev-builder-android image
+    required: false
+    default: 'true'
+runs:
+  using: composite
+  steps:
+    - name: Login to Dockerhub
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.dockerhub-image-registry }}
+        username: ${{ inputs.dockerhub-image-registry-username }}
+        password: ${{ inputs.dockerhub-image-registry-token }}
+
+    - name: Build and push dev-builder-ubuntu image
+      shell: bash
+      if: ${{ inputs.build-dev-builder-ubuntu == 'true' }}
+      run: |
+        make dev-builder \
+          BASE_IMAGE=ubuntu \
+          BUILDX_MULTI_PLATFORM_BUILD=true \
+          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+          IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push dev-builder-centos image
+      shell: bash
+      if: ${{ inputs.build-dev-builder-centos == 'true' }}
+      run: |
+        make dev-builder \
+          BASE_IMAGE=centos \
+          BUILDX_MULTI_PLATFORM_BUILD=true \
+          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+          IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push dev-builder-android image # Only build image for amd64 platform.
+      shell: bash
+      if: ${{ inputs.build-dev-builder-android == 'true' }}
+      run: |
+        make dev-builder \
+          BASE_IMAGE=android \
+          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+          IMAGE_TAG=${{ inputs.version }} && \
+        
+        docker push ${{ inputs.dockerhub-image-registry }}/${{ inputs.dockerhub-image-namespace }}/dev-builder-android:${{ inputs.version }}

.github/actions/build-greptime-binary/action.yml

@@ -16,47 +16,48 @@ inputs:
   version:
     description: Version of the artifact
     required: true
-  release-to-s3-bucket:
-    description: S3 bucket to store released artifacts
-    required: true
-  aws-access-key-id:
-    description: AWS access key id
-    required: true
-  aws-secret-access-key:
-    description: AWS secret access key
-    required: true
-  aws-region:
-    description: AWS region
-    required: true
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
   working-dir:
     description: Working directory to build the artifacts
     required: false
     default: .
+  build-android-artifacts:
+    description: Build android artifacts
+    required: false
+    default: 'false'
 runs:
   using: composite
   steps:
     - name: Build greptime binary
       shell: bash
+      if: ${{ inputs.build-android-artifacts == 'false' }}
       run: |
         cd ${{ inputs.working-dir }} && \
-        make build-greptime-by-buildx \
+        make build-by-dev-builder \
           CARGO_PROFILE=${{ inputs.cargo-profile }} \
           FEATURES=${{ inputs.features }} \
           BASE_IMAGE=${{ inputs.base-image }}
 
     - name: Upload artifacts
       uses: ./.github/actions/upload-artifacts
+      if: ${{ inputs.build-android-artifacts == 'false' }}
       with:
         artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: ./greptime
+        target-file: ./target/${{ inputs.cargo-profile }}/greptime
+        version: ${{ inputs.version }}
+        working-dir: ${{ inputs.working-dir }}
+
+    # TODO(zyy17): We can remove build-android-artifacts flag in the future.
+    - name: Build greptime binary
+      shell: bash
+      if: ${{ inputs.build-android-artifacts == 'true' }}
+      run: |
+        cd ${{ inputs.working-dir }} && make strip-android-bin
+
+    - name: Upload android artifacts
+      uses: ./.github/actions/upload-artifacts
+      if: ${{ inputs.build-android-artifacts == 'true' }}
+      with:
+        artifacts-dir: ${{ inputs.artifacts-dir }}
+        target-file: ./target/aarch64-linux-android/release/greptime
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}
         working-dir: ${{ inputs.working-dir }}

.github/actions/build-images/action.yml

@@ -40,7 +40,7 @@ runs:
         image-registry-password: ${{ inputs.image-registry-password }}
         image-name: ${{ inputs.image-name }}
         image-tag: ${{ inputs.version }}
-        docker-file: docker/ci/Dockerfile
+        docker-file: docker/ci/ubuntu/Dockerfile
         amd64-artifact-name: greptime-linux-amd64-pyo3-${{ inputs.version }}
         arm64-artifact-name: greptime-linux-arm64-pyo3-${{ inputs.version }}
         platforms: linux/amd64,linux/arm64
@@ -56,7 +56,7 @@ runs:
         image-registry-password: ${{ inputs.image-registry-password }}
         image-name: ${{ inputs.image-name }}-centos
         image-tag: ${{ inputs.version }}
-        docker-file: docker/ci/Dockerfile-centos
+        docker-file: docker/ci/centos/Dockerfile
         amd64-artifact-name: greptime-linux-amd64-centos-${{ inputs.version }}
         platforms: linux/amd64
         push-latest-tag: ${{ inputs.push-latest-tag }}

.github/actions/build-linux-artifacts/action.yml

@@ -13,26 +13,10 @@ inputs:
   disable-run-tests:
     description: Disable running integration tests
     required: true
-  release-to-s3-bucket:
-    description: S3 bucket to store released artifacts
-    required: true
-  aws-access-key-id:
-    description: AWS access key id
-    required: true
-  aws-secret-access-key:
-    description: AWS secret access key
-    required: true
-  aws-region:
-    description: AWS region
-    required: true
   dev-mode:
     description: Enable dev mode, only build standard greptime
     required: false
     default: 'false'
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
   working-dir:
     description: Working directory to build the artifacts
     required: false
@@ -64,11 +48,6 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }}
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}
         working-dir: ${{ inputs.working-dir }}
 
     - name: Build greptime without pyo3
@@ -80,13 +59,13 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}
         working-dir: ${{ inputs.working-dir }}
 
+    - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build.
+      shell: bash
+      run: |
+        rm -rf ./target/
+
     - name: Build greptime on centos base image
       uses: ./.github/actions/build-greptime-binary
       if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Only build centos7 base image for amd64.
@@ -96,9 +75,14 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-        upload-to-s3: ${{ inputs.upload-to-s3 }}
         working-dir: ${{ inputs.working-dir }}
+
+    - name: Build greptime on android base image
+      uses: ./.github/actions/build-greptime-binary
+      if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Only build android base image on amd64.
+      with:
+        base-image: android
+        artifacts-dir: greptime-android-arm64-${{ inputs.version }}
+        version: ${{ inputs.version }}
+        working-dir: ${{ inputs.working-dir }}
+        build-android-artifacts: true

.github/actions/build-macos-artifacts/action.yml

@@ -19,21 +19,9 @@ inputs:
   disable-run-tests:
     description: Disable running integration tests
     required: true
-  release-to-s3-bucket:
-    description: S3 bucket to store released artifacts
-    required: true
   artifacts-dir:
     description: Directory to store artifacts
     required: true
-  aws-access-key-id:
-    description: AWS access key id
-    required: true
-  aws-secret-access-key:
-    description: AWS secret access key
-    required: true
-  aws-region:
-    description: AWS region
-    required: true
 runs:
   using: composite
   steps:
@@ -99,7 +87,3 @@ runs:
         artifacts-dir: ${{ inputs.artifacts-dir }}
         target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
         version: ${{ inputs.version }}
-        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}

.github/actions/build-windows-artifacts/action.yml

@@ -0,0 +1,80 @@
+name: Build Windows artifacts
+description: Build Windows artifacts
+inputs:
+  arch:
+    description: Architecture to build
+    required: true
+  rust-toolchain:
+    description: Rust toolchain to use
+    required: true
+  cargo-profile:
+    description: Cargo profile to build
+    required: true
+  features:
+    description: Cargo features to build
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  disable-run-tests:
+    description: Disable running integration tests
+    required: true
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: true
+runs:
+  using: composite
+  steps:
+    - uses: arduino/setup-protoc@v1
+
+    - name: Install rust toolchain
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ inputs.rust-toolchain }}
+        targets: ${{ inputs.arch }}
+        components: llvm-tools-preview
+
+    - name: Rust Cache
+      uses: Swatinem/rust-cache@v2
+
+    - name: Install Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Install PyArrow Package
+      shell: pwsh
+      run: pip install pyarrow
+
+    - name: Install WSL distribution
+      uses: Vampire/setup-wsl@v2
+      with:
+        distribution: Ubuntu-22.04
+
+    - name: Install latest nextest release # For integration tests.
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      uses: taiki-e/install-action@nextest
+
+    - name: Run integration tests
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      shell: pwsh
+      run: make test sqlness-test
+
+    - name: Upload sqlness logs
+      if: ${{ failure() }} # Only upload logs when the integration tests failed.
+      uses: actions/upload-artifact@v3
+      with:
+        name: sqlness-logs
+        path: ${{ runner.temp }}/greptime-*.log
+        retention-days: 3
+
+    - name: Build greptime binary
+      shell: pwsh
+      run: cargo build --profile ${{ inputs.cargo-profile }} --features ${{ inputs.features }} --target ${{ inputs.arch }}
+
+    - name: Upload artifacts
+      uses: ./.github/actions/upload-artifacts
+      with:
+        artifacts-dir: ${{ inputs.artifacts-dir }}
+        target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
+        version: ${{ inputs.version }}

.github/actions/publish-github-release/action.yml

@@ -1,5 +1,5 @@
-name: Release artifacts
-description: Release artifacts
+name: Publish GitHub release
+description: Publish GitHub release
 inputs:
   version:
     description: Version to release

.github/actions/release-cn-artifacts/action.yaml

@@ -0,0 +1,138 @@
+name: Release CN artifacts
+description: Release artifacts to CN region
+inputs:
+  src-image-registry:
+    description: The source image registry to store the images
+    required: true
+    default: docker.io
+  src-image-namespace:
+    description: The namespace of the source image registry to store the images
+    required: true
+    default: greptime
+  src-image-name:
+    description: The name of the source image
+    required: false
+    default: greptimedb
+  dst-image-registry:
+    description: The destination image registry to store the images
+    required: true
+  dst-image-namespace:
+    description: The namespace of the destination image registry to store the images
+    required: true
+    default: greptime
+  dst-image-registry-username:
+    description: The username to login to the image registry
+    required: true
+  dst-image-registry-password:
+    description: The password to login to the image registry
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  dev-mode:
+    description: Enable dev mode, only push standard greptime
+    required: false
+    default: 'false'
+  push-latest-tag:
+    description: Whether to push the latest tag of the image
+    required: false
+    default: 'true'
+  aws-cn-s3-bucket:
+    description: S3 bucket to store released artifacts in CN region
+    required: true
+  aws-cn-access-key-id:
+    description: AWS access key id in CN region
+    required: true
+  aws-cn-secret-access-key:
+    description: AWS secret access key in CN region
+    required: true
+  aws-cn-region:
+    description: AWS region in CN
+    required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: false
+    default: 'artifacts'
+  update-version-info:
+    description: Update the version info in S3
+    required: false
+    default: 'true'
+  upload-max-retry-times:
+    description: Max retry times for uploading artifacts to S3
+    required: false
+    default: "20"
+  upload-retry-timeout:
+    description: Timeout for uploading artifacts to S3
+    required: false
+    default: "30" # minutes
+runs:
+  using: composite
+  steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v3
+      with:
+        path: ${{ inputs.artifacts-dir }}
+
+    - name: Release artifacts to cn region
+      uses: nick-invision/retry@v2
+      if: ${{ inputs.upload-to-s3 == 'true' }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-cn-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-cn-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-cn-region }}
+        UPDATE_VERSION_INFO: ${{ inputs.update-version-info }}
+      with:
+        max_attempts: ${{ inputs.upload-max-retry-times }}
+        timeout_minutes: ${{ inputs.upload-retry-timeout }}
+        command: |
+          ./.github/scripts/upload-artifacts-to-s3.sh \
+            ${{ inputs.artifacts-dir }} \
+            ${{ inputs.version }} \
+            ${{ inputs.aws-cn-s3-bucket }}
+
+    - name: Push greptimedb image from Dockerhub to ACR
+      shell: bash
+      env:
+        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
+        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
+      run: |
+        ./.github/scripts/copy-image.sh \
+         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}:${{ inputs.version }} \
+         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
+
+    - name: Push latest greptimedb image from Dockerhub to ACR
+      shell: bash
+      if: ${{ inputs.push-latest-tag == 'true' }}
+      env:
+        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
+        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
+      run: |
+        ./.github/scripts/copy-image.sh \
+         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}:latest \
+         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
+
+    - name: Push greptimedb-centos image from DockerHub to ACR
+      shell: bash
+      if: ${{ inputs.dev-mode == 'false' }}
+      env:
+        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
+        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
+      run: |
+        ./.github/scripts/copy-image.sh \
+         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}-centos:latest \
+         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
+
+    - name: Push greptimedb-centos image from DockerHub to ACR
+      shell: bash
+      if: ${{ inputs.dev-mode == 'false' && inputs.push-latest-tag == 'true' }}
+      env:
+        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
+        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
+      run: |
+        ./.github/scripts/copy-image.sh \
+         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}-centos:latest \
+         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}

.github/actions/upload-artifacts/action.yml

@@ -10,22 +10,6 @@ inputs:
   version:
     description: Version of the artifact
     required: true
-  release-to-s3-bucket:
-    description: S3 bucket to store released artifacts
-    required: true
-  aws-access-key-id:
-    description: AWS access key id
-    required: true
-  aws-secret-access-key:
-    description: AWS secret access key
-    required: true
-  aws-region:
-    description: AWS region
-    required: true
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
   working-dir:
     description: Working directory to upload the artifacts
     required: false
@@ -49,9 +33,21 @@ runs:
       working-directory: ${{ inputs.working-dir }}
       shell: bash
       run: |
-        tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }} && \
+        tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }}
+
+    - name: Calculate checksum
+      if: runner.os != 'Windows'
+      working-directory: ${{ inputs.working-dir }}
+      shell: bash
+      run: |
         echo $(shasum -a 256 ${{ inputs.artifacts-dir }}.tar.gz | cut -f1 -d' ') > ${{ inputs.artifacts-dir }}.sha256sum
 
+    - name: Calculate checksum on Windows
+      if: runner.os == 'Windows'
+      working-directory: ${{ inputs.working-dir }}
+      shell: pwsh
+      run: Get-FileHash ${{ inputs.artifacts-dir }}.tar.gz -Algorithm SHA256 | select -ExpandProperty Hash > ${{ inputs.artifacts-dir }}.sha256sum
+
     # Note: The artifacts will be double zip compressed(related issue: https://github.com/actions/upload-artifact/issues/39).
     # However, when we use 'actions/download-artifact@v3' to download the artifacts, it will be automatically unzipped.
     - name: Upload artifacts
@@ -65,34 +61,3 @@ runs:
       with:
         name: ${{ inputs.artifacts-dir }}.sha256sum
         path: ${{ inputs.working-dir }}/${{ inputs.artifacts-dir }}.sha256sum
-
-    - name: Configure AWS credentials
-      if: ${{ inputs.upload-to-s3 == 'true' }}
-      uses: aws-actions/configure-aws-credentials@v2
-      with:
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
-
-    - name: Upload artifacts to S3
-      if: ${{ inputs.upload-to-s3 == 'true' }}
-      uses: nick-invision/retry@v2
-      with:
-        max_attempts: 20
-        timeout_minutes: 5
-        # The bucket layout will be:
-        # releases/greptimedb
-        # ├── v0.1.0
-        # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
-        # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
-        # └── v0.2.0
-        #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
-        #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
-        command: |
-          cd ${{ inputs.working-dir }} && \
-          aws s3 cp \
-            ${{ inputs.artifacts-dir }}.tar.gz \
-            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.tar.gz && \
-          aws s3 cp \
-            ${{ inputs.artifacts-dir }}.sha256sum \
-            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.sha256sum

.github/scripts/copy-image.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+SRC_IMAGE=$1
+DST_REGISTRY=$2
+SKOPEO_STABLE_IMAGE="quay.io/skopeo/stable:latest"
+
+# Check if necessary variables are set.
+function check_vars() {
+  for var in DST_REGISTRY_USERNAME DST_REGISTRY_PASSWORD DST_REGISTRY SRC_IMAGE; do
+    if [ -z "${!var}" ]; then
+      echo "$var is not set or empty."
+      echo "Usage: DST_REGISTRY_USERNAME=<your-dst-registry-username> DST_REGISTRY_PASSWORD=<your-dst-registry-password> $0 <dst-registry> <src-image>"
+      exit 1
+    fi
+  done
+}
+
+# Copies images from DockerHub to the destination registry.
+function copy_images_from_dockerhub() {
+  # Check if docker is installed.
+  if ! command -v docker &> /dev/null; then
+    echo "docker is not installed. Please install docker to continue."
+    exit 1
+  fi
+
+  # Extract the name and tag of the source image.
+  IMAGE_NAME=$(echo "$SRC_IMAGE" | sed "s/.*\///")
+
+  echo "Copying $SRC_IMAGE to $DST_REGISTRY/$IMAGE_NAME"
+
+  docker run "$SKOPEO_STABLE_IMAGE" copy -a docker://"$SRC_IMAGE" \
+    --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+    docker://"$DST_REGISTRY/$IMAGE_NAME"
+}
+
+function main() {
+  check_vars
+  copy_images_from_dockerhub
+}
+
+# Usage example:
+# DST_REGISTRY_USERNAME=123 DST_REGISTRY_PASSWORD=456 \
+#   ./copy-image.sh greptime/greptimedb:v0.4.0 greptime-registry.cn-hangzhou.cr.aliyuncs.com
+main

.github/scripts/upload-artifacts-to-s3.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+ARTIFACTS_DIR=$1
+VERSION=$2
+AWS_S3_BUCKET=$3
+RELEASE_DIRS="releases/greptimedb"
+GREPTIMEDB_REPO="GreptimeTeam/greptimedb"
+
+# Check if necessary variables are set.
+function check_vars() {
+  for var in AWS_S3_BUCKET VERSION ARTIFACTS_DIR; do
+    if [ -z "${!var}" ]; then
+      echo "$var is not set or empty."
+      echo "Usage: $0 <artifacts-dir> <version> <aws-s3-bucket>"
+      exit 1
+    fi
+  done
+}
+
+# Uploads artifacts to AWS S3 bucket.
+function upload_artifacts() {
+  # The bucket layout will be:
+  # releases/greptimedb
+  # ├── latest-version.txt
+  # ├── latest-nightly-version.txt
+  # ├── v0.1.0
+  # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
+  # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
+  # └── v0.2.0
+  #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
+  #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
+  find "$ARTIFACTS_DIR" -type f \( -name "*.tar.gz" -o -name "*.sha256sum" \) | while IFS= read -r file; do
+    aws s3 cp \
+      "$file" "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/$VERSION/$(basename "$file")"
+  done
+}
+
+# Updates the latest version information in AWS S3 if UPDATE_VERSION_INFO is true.
+function update_version_info() {
+  if [ "$UPDATE_VERSION_INFO" == "true" ]; then
+    # If it's the officail release(like v1.0.0, v1.0.1, v1.0.2, etc.), update latest-version.txt.
+    if [[ "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+      echo "Updating latest-version.txt"
+      echo "$VERSION" > latest-version.txt
+      aws s3 cp \
+        latest-version.txt "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/latest-version.txt"
+    fi
+
+    # If it's the nightly release, update latest-nightly-version.txt.
+    if [[ "$VERSION" == *"nightly"* ]]; then
+      echo "Updating latest-nightly-version.txt"
+      echo "$VERSION" > latest-nightly-version.txt
+      aws s3 cp \
+        latest-nightly-version.txt "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/latest-nightly-version.txt"
+    fi
+  fi
+}
+
+# Downloads artifacts from Github if DOWNLOAD_ARTIFACTS_FROM_GITHUB is true.
+function download_artifacts_from_github() {
+  if [ "$DOWNLOAD_ARTIFACTS_FROM_GITHUB" == "true" ]; then
+    # Check if jq is installed.
+    if ! command -v jq &> /dev/null; then
+      echo "jq is not installed. Please install jq to continue."
+      exit 1
+    fi
+
+    # Get the latest release API response.
+    RELEASES_API_RESPONSE=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/$GREPTIMEDB_REPO/releases/latest")
+
+    # Extract download URLs for the artifacts.
+    # Exclude source code archives which are typically named as 'greptimedb-<version>.zip' or 'greptimedb-<version>.tar.gz'.
+    ASSET_URLS=$(echo "$RELEASES_API_RESPONSE" | jq -r '.assets[] | select(.name | test("greptimedb-.*\\.(zip|tar\\.gz)$") | not) | .browser_download_url')
+
+    # Download each asset.
+    while IFS= read -r url; do
+      if [ -n "$url" ]; then
+        curl -LJO "$url"
+        echo "Downloaded: $url"
+      fi
+    done <<< "$ASSET_URLS"
+  fi
+}
+
+function main() {
+  check_vars
+  download_artifacts_from_github
+  upload_artifacts
+  update_version_info
+}
+
+# Usage example:
+#   AWS_ACCESS_KEY_ID=<your_access_key_id> \
+#   AWS_SECRET_ACCESS_KEY=<your_secret_access_key> \
+#   AWS_DEFAULT_REGION=<your_region> \
+#   UPDATE_VERSION_INFO=true \
+#   DOWNLOAD_ARTIFACTS_FROM_GITHUB=false \
+#     ./upload-artifacts-to-s3.sh <artifacts-dir> <version> <aws-s3-bucket>
+main

.github/workflows/apidoc.yml

@@ -17,7 +17,7 @@ env:
 
 jobs:
   apidoc:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v3
     - uses: arduino/setup-protoc@v1

.github/workflows/dev-build.yml

@@ -16,11 +16,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-latest
-          - ubuntu-latest-8-cores
-          - ubuntu-latest-16-cores
-          - ubuntu-latest-32-cores
-          - ubuntu-latest-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -78,7 +78,7 @@ jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -164,12 +164,7 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
           dev-mode: true # Only build the standard greptime binary.
-          upload-to-s3: false # No need to upload to S3.
           working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
 
   build-linux-arm64-artifacts:
@@ -198,12 +193,7 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
           dev-mode: true # Only build the standard greptime binary.
-          upload-to-s3: false # No need to upload to S3.
           working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
 
   release-images-to-dockerhub:
@@ -214,7 +204,7 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       build-result: ${{ steps.set-build-result.outputs.build-result }}
     steps:
@@ -239,41 +229,44 @@ jobs:
         run: |
           echo "build-result=success" >> $GITHUB_OUTPUT
 
-  release-images-to-acr:
-    name: Build and push images to ACR
+  release-cn-artifacts:
+    name: Release artifacts to CN region
     if: ${{ inputs.release_images || github.event_name == 'schedule' }}
     needs: [
       allocate-runners,
-      build-linux-amd64-artifacts,
-      build-linux-arm64-artifacts,
+      release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
-    # When we push to ACR, it's easy to fail due to some unknown network issues.
-    # However, we don't want to fail the whole workflow because of this.
-    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
+    runs-on: ubuntu-20.04
     continue-on-error: true
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Build and push images to ACR
-        uses: ./.github/actions/build-images
+      - name: Release artifacts to CN region
+        uses: ./.github/actions/release-cn-artifacts
         with:
-          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-name: ${{ env.IMAGE_NAME }}
-          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          src-image-registry: docker.io
+          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          src-image-name: greptimedb
+          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          push-latest-tag: false # Don't push the latest tag to registry.
-          dev-mode: true # Only build the standard images.
+          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: true                     # Only build the standard images(exclude centos images).
+          push-latest-tag: false             # Don't push the latest tag to registry.
+          update-version-info: false         # Don't update the version info in S3.
 
   stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -298,7 +291,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -325,7 +318,7 @@ jobs:
     needs: [
       release-images-to-dockerhub
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
@@ -334,11 +327,11 @@ jobs:
         if: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }}
         with:
           payload: |
-            {"text": "GreptimeDB ${{ env.NEXT_RELEASE_VERSION }} build successful"}
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}
 
       - name: Notifiy nightly build failed result
         uses: slackapi/slack-github-action@v1.23.0
         if: ${{ needs.release-images-to-dockerhub.outputs.build-result != 'success' }}
         with:
           payload: |
-            {"text": "GreptimeDB ${{ env.NEXT_RELEASE_VERSION }} build failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'"}
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."}

.github/workflows/develop.yml

@@ -24,13 +24,17 @@ on:
 
 name: CI
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
   RUST_TOOLCHAIN: nightly-2023-08-07
 
 jobs:
   typos:
     name: Spell Check with Typos
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - uses: crate-ci/typos@v1.13.10
@@ -38,7 +42,7 @@ jobs:
   check:
     name: Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -56,7 +60,7 @@ jobs:
   toml:
     name: Toml Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -76,7 +80,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest-8-cores, windows-latest-8-cores ]
+        os: [ ubuntu-20.04-8-cores ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -101,7 +105,7 @@ jobs:
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -120,7 +124,7 @@ jobs:
   clippy:
     name: Clippy
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -138,7 +142,7 @@ jobs:
 
   coverage:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-20.04-8-cores
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -184,43 +188,3 @@ jobs:
           flags: rust
           fail_ci_if_error: false
           verbose: true
-
-  test-on-windows:
-    if: github.event.pull_request.draft == false
-    runs-on: windows-latest-8-cores
-    timeout-minutes: 60
-    steps:
-      - run: git config --global core.autocrlf false
-      - uses: actions/checkout@v3
-      - uses: arduino/setup-protoc@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          components: llvm-tools-preview
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-      - name: Install Cargo Nextest
-        uses: taiki-e/install-action@nextest
-      - name: Install Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Install PyArrow Package
-        run: pip install pyarrow
-      - name: Install WSL distribution
-        uses: Vampire/setup-wsl@v2
-        with:
-          distribution: Ubuntu-22.04
-      - name: Running tests
-        run: cargo nextest run -F pyo3_backend,dashboard
-        env:
-          RUST_BACKTRACE: 1
-          CARGO_INCREMENTAL: 0
-          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-          GT_S3_REGION: ${{ secrets.S3_REGION }}
-          UNITTEST_LOG_DIR: "__unittest_logs"

.github/workflows/doc-issue.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   doc_issue:
     if: github.event.label.name == 'doc update required'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: create an issue in doc repo
         uses: dacbd/create-issue-action@main
@@ -25,7 +25,7 @@ jobs:
             ${{ github.event.issue.html_url || github.event.pull_request.html_url }}
   cloud_issue:
     if: github.event.label.name == 'cloud followup required'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: create an issue in cloud repo
         uses: dacbd/create-issue-action@main

.github/workflows/docs.yml

@@ -30,7 +30,7 @@ name: CI
 jobs:
   typos:
     name: Spell Check with Typos
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - uses: crate-ci/typos@v1.13.10
@@ -38,33 +38,33 @@ jobs:
   check:
     name: Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   clippy:
     name: Clippy
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   coverage:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'
 
   sqlness:
     name: Sqlness Test
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - run: 'echo "No action required"'

.github/workflows/license.yaml

@@ -8,7 +8,7 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
 jobs:
   license-header-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     name: license-header-check
     steps:
     - uses: actions/checkout@v2

.github/workflows/nightly-build.yml

@@ -14,11 +14,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.2xlarge-amd64
         options:
-          - ubuntu-latest
-          - ubuntu-latest-8-cores
-          - ubuntu-latest-16-cores
-          - ubuntu-latest-32-cores
-          - ubuntu-latest-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -70,7 +70,7 @@ jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -147,10 +147,6 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -170,10 +166,6 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub
@@ -183,7 +175,7 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       nightly-build-result: ${{ steps.set-nightly-build-result.outputs.nightly-build-result }}
     steps:
@@ -206,15 +198,14 @@ jobs:
         run: |
           echo "nightly-build-result=success" >> $GITHUB_OUTPUT
 
-  release-images-to-acr:
-    name: Build and push images to ACR
+  release-cn-artifacts:
+    name: Release artifacts to CN region
     if: ${{ inputs.release_images || github.event_name == 'schedule' }}
     needs: [
       allocate-runners,
-      build-linux-amd64-artifacts,
-      build-linux-arm64-artifacts,
+      release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -224,21 +215,30 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Build and push images to ACR
-        uses: ./.github/actions/build-images
+      - name: Release artifacts to CN region
+        uses: ./.github/actions/release-cn-artifacts
         with:
-          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          src-image-registry: docker.io
+          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          src-image-name: greptimedb
+          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          push-latest-tag: false # Don't push the latest tag to registry.
+          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: false
+          update-version-info: false  # Don't update version info in S3.
+          push-latest-tag: false      # Don't push the latest tag to registry.
 
   stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -263,7 +263,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -290,7 +290,7 @@ jobs:
     needs: [
       release-images-to-dockerhub
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:
@@ -299,11 +299,11 @@ jobs:
         if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result == 'success' }}
         with:
           payload: |
-            {"text": "GreptimeDB nightly build successful"}
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}
 
       - name: Notifiy nightly build failed result
         uses: slackapi/slack-github-action@v1.23.0
         if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result != 'success' }}
         with:
           payload: |
-            {"text": "GreptimeDB nightly build failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/nightly-build.yml'"}
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."}

.github/workflows/nightly-ci.yml

@@ -0,0 +1,82 @@
+# Nightly CI: runs tests every night for our second tier plaforms (Windows)
+
+on:
+  schedule:
+    - cron: '0 23 * * 1-5'
+  workflow_dispatch:
+
+name: Nightly CI
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  RUST_TOOLCHAIN: nightly-2023-08-07
+
+jobs:
+  sqlness:
+    name: Sqlness Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ windows-latest-8-cores ]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4.1.0
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Run sqlness
+        run: cargo sqlness
+      - name: Upload sqlness logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: sqlness-logs
+          path: ${{ runner.temp }}/greptime-*.log
+          retention-days: 3
+
+  test-on-windows:
+    runs-on: windows-latest-8-cores
+    timeout-minutes: 60
+    steps:
+      - run: git config --global core.autocrlf false
+      - uses: actions/checkout@v4.1.0
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
+      - name: Install WSL distribution
+        uses: Vampire/setup-wsl@v2
+        with:
+          distribution: Ubuntu-22.04
+      - name: Running tests
+        run: cargo nextest run -F pyo3_backend,dashboard
+        env:
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          GT_S3_REGION: ${{ secrets.S3_REGION }}
+          UNITTEST_LOG_DIR: "__unittest_logs"

.github/workflows/pr-title-checker.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 10
     steps:
       - uses: thehanimo/pr-title-checker@v1.3.4
@@ -19,7 +19,7 @@ jobs:
           pass_on_octokit_error: false
           configuration_path: ".github/pr-title-checker-config.json"
   breaking:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     timeout-minutes: 10
     steps:
       - uses: thehanimo/pr-title-checker@v1.3.4

.github/workflows/release-dev-builder-images.yaml

@@ -0,0 +1,85 @@
+name: Release dev-builder images
+
+on:
+  workflow_dispatch: # Allows you to run this workflow manually.
+    inputs:
+      version:
+        description: Version of the dev-builder
+        required: false
+        default: latest
+      release_dev_builder_ubuntu_image:
+        type: boolean
+        description: Release dev-builder-ubuntu image
+        required: false
+        default: false
+      release_dev_builder_centos_image:
+        type: boolean
+        description: Release dev-builder-centos image
+        required: false
+        default: false
+      release_dev_builder_android_image:
+        type: boolean
+        description: Release dev-builder-android image
+        required: false
+        default: false
+
+jobs:
+  release-dev-builder-images:
+    name: Release dev builder images
+    if: ${{ inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }} # Only manually trigger this job.
+    runs-on: ubuntu-20.04-16-cores
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push dev builder images
+        uses: ./.github/actions/build-dev-builder-images
+        with:
+          version: ${{ inputs.version }}
+          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          build-dev-builder-ubuntu: ${{ inputs.release_dev_builder_ubuntu_image }}
+          build-dev-builder-centos: ${{ inputs.release_dev_builder_centos_image }}
+          build-dev-builder-android: ${{ inputs.release_dev_builder_android_image }}
+
+  release-dev-builder-images-cn: # Note: Be careful issue: https://github.com/containers/skopeo/issues/1874 and we decide to use the latest stable skopeo container.
+    name: Release dev builder images to CN region
+    runs-on: ubuntu-20.04
+    needs: [
+      release-dev-builder-images
+    ]
+    steps:
+      - name: Push dev-builder-ubuntu image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_ubuntu_image }}
+        env:
+          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
+          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }} \
+            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }}
+
+      - name: Push dev-builder-centos image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_centos_image }}
+        env:
+          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
+          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }} \
+            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }}
+
+      - name: Push dev-builder-android image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_android_image }}
+        env:
+          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
+          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }} \
+            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }}

.github/workflows/release.yml

@@ -18,11 +18,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-latest
-          - ubuntu-latest-8-cores
-          - ubuntu-latest-16-cores
-          - ubuntu-latest-32-cores
-          - ubuntu-latest-64-cores
+          - ubuntu-20.04
+          - ubuntu-20.04-8-cores
+          - ubuntu-20.04-16-cores
+          - ubuntu-20.04-32-cores
+          - ubuntu-20.04-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -63,7 +63,12 @@ on:
         description: Build macos artifacts
         required: false
         default: false
-      release_artifacts:
+      build_windows_artifacts:
+        type: boolean
+        description: Build Windows artifacts
+        required: false
+        default: false
+      publish_github_release:
         type: boolean
         description: Create GitHub release and upload artifacts
         required: false
@@ -73,11 +78,6 @@ on:
         description: Build and push images to DockerHub and ACR
         required: false
         default: false
-      release_dev_builder_image:
-        type: boolean
-        description: Release dev-builder image
-        required: false
-        default: false
 
 # Use env variables to control all the release process.
 env:
@@ -91,17 +91,18 @@ env:
   # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
   # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.4.0
+  NEXT_RELEASE_VERSION: v0.5.0
 
 jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
       macos-runner: ${{ inputs.macos_runner || vars.DEFAULT_MACOS_RUNNER }}
+      windows-runner: windows-latest-8-cores
 
       # The following EC2 resource id will be used for resource releasing.
       linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
@@ -177,10 +178,6 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -200,10 +197,6 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
   build-macos-artifacts:
     name: Build macOS artifacts
@@ -245,11 +238,43 @@ jobs:
           features: ${{ matrix.features }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
           artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
-          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+
+  build-windows-artifacts:
+    name: Build Windows artifacts
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ${{ needs.allocate-runners.outputs.windows-runner }}
+            arch: x86_64-pc-windows-msvc
+            features: servers/dashboard
+            artifacts-dir-prefix: greptime-windows-amd64
+          - os: ${{ needs.allocate-runners.outputs.windows-runner }}
+            arch: x86_64-pc-windows-msvc
+            features: pyo3_backend,servers/dashboard
+            artifacts-dir-prefix: greptime-windows-amd64-pyo3
+    runs-on: ${{ matrix.os }}
+    needs: [
+      allocate-runners,
+    ]
+    if: ${{ inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    steps:
+      - run: git config --global core.autocrlf false
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-windows-artifacts
+        with:
+          arch: ${{ matrix.arch }}
+          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          features: ${{ matrix.features }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub
@@ -274,15 +299,14 @@ jobs:
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
 
-  release-images-to-acr:
-    name: Build and push images to ACR
+  release-cn-artifacts:
+    name: Release artifacts to CN region
     if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
     needs: [
       allocate-runners,
-      build-linux-amd64-artifacts,
-      build-linux-arm64-artifacts,
+      release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-2004-16-cores
+    runs-on: ubuntu-20.04
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -292,18 +316,28 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Build and push images to ACR
-        uses: ./.github/actions/build-images
+      - name: Release artifacts to CN region
+        uses: ./.github/actions/release-cn-artifacts
         with:
-          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          src-image-registry: docker.io
+          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          src-image-name: greptimedb
+          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
           version: ${{ needs.allocate-runners.outputs.version }}
+          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: false
+          update-version-info: true
+          push-latest-tag: true
 
-  release-artifacts:
+  publish-github-release:
     name: Create GitHub release and upload artifacts
-    if: ${{ inputs.release_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ inputs.publish_github_release || github.event_name == 'push' || github.event_name == 'schedule' }}
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -311,36 +345,17 @@ jobs:
       build-macos-artifacts,
       release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Release artifacts
-        uses: ./.github/actions/release-artifacts
+      - name: Publish GitHub release
+        uses: ./.github/actions/publish-github-release
         with:
           version: ${{ needs.allocate-runners.outputs.version }}
 
-  release-dev-builder-image:
-    name: Release dev builder image
-    if: ${{ inputs.release_dev_builder_image }} # Only manually trigger this job.
-    runs-on: ubuntu-latest-16-cores
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Build and push dev builder image
-        uses: ./.github/actions/build-dev-builder-image
-        with:
-          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
-          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
-          acr-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          acr-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          acr-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
-
   ### Stop runners ###
   # It's very necessary to split the job of releasing runners into 'stop-linux-amd64-runner' and 'stop-linux-arm64-runner'.
   # Because we can terminate the specified EC2 instance immediately after the job is finished without uncessary waiting.
@@ -348,7 +363,7 @@ jobs:
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -373,7 +388,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,

Cargo.toml

@@ -2,15 +2,17 @@
 members = [
     "benchmarks",
     "src/api",
+    "src/auth",
     "src/catalog",
     "src/client",
     "src/cmd",
     "src/common/base",
     "src/common/catalog",
+    "src/common/config",
     "src/common/datasource",
     "src/common/error",
     "src/common/function",
-    "src/common/function-macro",
+    "src/common/macro",
     "src/common/greptimedb-telemetry",
     "src/common/grpc",
     "src/common/grpc-expr",
@@ -28,15 +30,16 @@ members = [
     "src/common/version",
     "src/datanode",
     "src/datatypes",
-    "src/file-table-engine",
+    "src/file-engine",
     "src/frontend",
     "src/log-store",
     "src/meta-client",
     "src/meta-srv",
-    "src/mito",
     "src/mito2",
     "src/object-store",
+    "src/operator",
     "src/partition",
+    "src/plugins",
     "src/promql",
     "src/query",
     "src/script",
@@ -46,90 +49,101 @@ members = [
     "src/storage",
     "src/store-api",
     "src/table",
-    "src/table-procedure",
     "tests-integration",
     "tests/runner",
 ]
 resolver = "2"
 
 [workspace.package]
-version = "0.3.2"
+version = "0.4.0"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
+aquamarine = "0.3"
 arrow = { version = "43.0" }
-etcd-client = "0.11"
 arrow-array = "43.0"
 arrow-flight = "43.0"
 arrow-schema = { version = "43.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
 derive_builder = "0.12"
+etcd-client = "0.11"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "940694cfd05c1e93c1dd7aab486184c9e2853098" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "1f1dd532a111e3834cc3019c5605e2993ffb9dc3" }
+humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
+metrics = "0.20"
+moka = "0.12"
 once_cell = "1.18"
 opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
 parquet = "43.0"
 paste = "1.0"
 prost = "0.11"
+raft-engine = { git = "https://github.com/tikv/raft-engine.git", rev = "22dfb426cd994602b57725ef080287d3e53db479" }
 rand = "0.8"
 regex = "1.8"
+reqwest = { version = "0.11", default-features = false, features = [
+    "json",
+    "rustls-tls-native-roots",
+    "stream",
+] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+smallvec = "1"
 snafu = { version = "0.7", features = ["backtraces"] }
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "c3814f08afa19786b13d72b1731a1e8b3cac4ab9", features = [
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4", features = [
     "visitor",
 ] }
+strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
 tokio = { version = "1.28", features = ["full"] }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.7"
 tonic = { version = "0.9", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
-metrics = "0.20"
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
 ## workspaces members
 api = { path = "src/api" }
+auth = { path = "src/auth" }
 catalog = { path = "src/catalog" }
 client = { path = "src/client" }
 cmd = { path = "src/cmd" }
 common-base = { path = "src/common/base" }
 common-catalog = { path = "src/common/catalog" }
+common-config = { path = "src/common/config" }
 common-datasource = { path = "src/common/datasource" }
 common-error = { path = "src/common/error" }
 common-function = { path = "src/common/function" }
-common-function-macro = { path = "src/common/function-macro" }
 common-greptimedb-telemetry = { path = "src/common/greptimedb-telemetry" }
 common-grpc = { path = "src/common/grpc" }
 common-grpc-expr = { path = "src/common/grpc-expr" }
+common-macro = { path = "src/common/macro" }
 common-mem-prof = { path = "src/common/mem-prof" }
 common-meta = { path = "src/common/meta" }
+common-pprof = { path = "src/common/pprof" }
 common-procedure = { path = "src/common/procedure" }
 common-procedure-test = { path = "src/common/procedure-test" }
-common-pprof = { path = "src/common/pprof" }
 common-query = { path = "src/common/query" }
 common-recordbatch = { path = "src/common/recordbatch" }
 common-runtime = { path = "src/common/runtime" }
-substrait = { path = "src/common/substrait" }
 common-telemetry = { path = "src/common/telemetry" }
 common-test-util = { path = "src/common/test-util" }
 common-time = { path = "src/common/time" }
 common-version = { path = "src/common/version" }
 datanode = { path = "src/datanode" }
 datatypes = { path = "src/datatypes" }
-file-table-engine = { path = "src/file-table-engine" }
+file-engine = { path = "src/file-engine" }
 frontend = { path = "src/frontend" }
 log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
@@ -137,7 +151,9 @@ meta-srv = { path = "src/meta-srv" }
 mito = { path = "src/mito" }
 mito2 = { path = "src/mito2" }
 object-store = { path = "src/object-store" }
+operator = { path = "src/operator" }
 partition = { path = "src/partition" }
+plugins = { path = "src/plugins" }
 promql = { path = "src/promql" }
 query = { path = "src/query" }
 script = { path = "src/script" }
@@ -146,8 +162,8 @@ session = { path = "src/session" }
 sql = { path = "src/sql" }
 storage = { path = "src/storage" }
 store-api = { path = "src/store-api" }
+substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }
-table-procedure = { path = "src/table-procedure" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"

Makefile

@@ -12,6 +12,8 @@ BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu
 RUST_TOOLCHAIN ?= $(shell cat rust-toolchain.toml | grep channel | cut -d'"' -f2)
 CARGO_REGISTRY_CACHE ?= ${HOME}/.cargo/registry
+ARCH := $(shell uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
+OUTPUT_DIR := $(shell if [ "$(RELEASE)" = "true" ]; then echo "release"; elif [ ! -z "$(CARGO_PROFILE)" ]; then echo "$(CARGO_PROFILE)" ; else echo "debug"; fi)
 
 # The arguments for running integration tests.
 ETCD_VERSION ?= v3.5.9
@@ -43,35 +45,60 @@ ifneq ($(strip $(TARGET)),)
 	CARGO_BUILD_OPTS += --target ${TARGET}
 endif
 
+ifneq ($(strip $(RELEASE)),)
+	CARGO_BUILD_OPTS += --release
+endif
+
 ifeq ($(BUILDX_MULTI_PLATFORM_BUILD), true)
 	BUILDX_MULTI_PLATFORM_BUILD_OPTS := --platform linux/amd64,linux/arm64 --push
 else
 	BUILDX_MULTI_PLATFORM_BUILD_OPTS := -o type=docker
 endif
 
+ifneq ($(strip $(CARGO_BUILD_EXTRA_OPTS)),)
+	CARGO_BUILD_OPTS += ${CARGO_BUILD_EXTRA_OPTS}
+endif
+
 ##@ Build
 
 .PHONY: build
-build: ## Build debug version greptime. If USE_DEV_BUILDER is true, the binary will be built in dev-builder.
-ifeq ($(USE_DEV_BUILDER), true)
-	docker run --network=host \
-	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
-	make build CARGO_PROFILE=${CARGO_PROFILE} FEATURES=${FEATURES} TARGET_DIR=${TARGET_DIR}
-else
-	cargo build ${CARGO_BUILD_OPTS}
-endif
+build: ## Build debug version greptime.
+	cargo ${CARGO_EXTENSION} build ${CARGO_BUILD_OPTS}
 
-.PHONY: release
-release:  ## Build release version greptime. If USE_DEV_BUILDER is true, the binary will be built in dev-builder.
-ifeq ($(USE_DEV_BUILDER), true)
+.POHNY: build-by-dev-builder
+build-by-dev-builder: ## Build greptime by dev-builder.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
-	make release CARGO_PROFILE=${CARGO_PROFILE} FEATURES=${FEATURES} TARGET_DIR=${TARGET_DIR}
-else
-	cargo build --release ${CARGO_BUILD_OPTS}
-endif
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
+	make build \
+	CARGO_EXTENSION="${CARGO_EXTENSION}" \
+	CARGO_PROFILE=${CARGO_PROFILE} \
+	FEATURES=${FEATURES} \
+	TARGET_DIR=${TARGET_DIR} \
+	TARGET=${TARGET} \
+	RELEASE=${RELEASE} \
+	CARGO_BUILD_EXTRA_OPTS="${CARGO_BUILD_EXTRA_OPTS}"
+
+.PHONY: build-android-bin
+build-android-bin: ## Build greptime binary for android.
+	docker run --network=host \
+	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \
+	make build \
+	CARGO_EXTENSION="ndk --platform 23 -t aarch64-linux-android" \
+	CARGO_PROFILE=release \
+	FEATURES="${FEATURES}" \
+	TARGET_DIR="${TARGET_DIR}" \
+	TARGET="${TARGET}" \
+	RELEASE="${RELEASE}" \
+	CARGO_BUILD_EXTRA_OPTS="--bin greptime --no-default-features"
+
+.PHONY: strip-android-bin
+strip-android-bin: build-android-bin ## Strip greptime binary for android.
+	docker run --network=host \
+	-v ${PWD}:/greptimedb \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \
+	bash -c '$${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip /greptimedb/target/aarch64-linux-android/release/greptime'
 
 .PHONY: clean
 clean: ## Clean the project.
@@ -90,30 +117,27 @@ check-toml: ## Check all TOML files.
 	taplo format --check
 
 .PHONY: docker-image
-docker-image: multi-platform-buildx ## Build docker image.
+docker-image: build-by-dev-builder ## Build docker image.
+	mkdir -p ${ARCH} && \
+	cp ./target/${OUTPUT_DIR}/greptime ${ARCH}/greptime && \
+	docker build -f docker/ci/${BASE_IMAGE}/Dockerfile -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb:${IMAGE_TAG} . && \
+	rm -r ${ARCH}
+
+.PHONY: docker-image-buildx
+docker-image-buildx: multi-platform-buildx ## Build docker image by buildx.
 	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
-	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" --build-arg="FEATURES=${FEATURES}" \
-	  -f docker/${BASE_IMAGE}/Dockerfile \
+	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" \
+	  --build-arg="FEATURES=${FEATURES}" \
+	  --build-arg="OUTPUT_DIR=${OUTPUT_DIR}" \
+	  -f docker/buildx/${BASE_IMAGE}/Dockerfile \
 	  -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
 
-.PHONY: build-greptime-by-buildx
-build-greptime-by-buildx: multi-platform-buildx ## Build greptime binary by docker buildx. The binary will be copied to the current directory.
-	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
-	  --target=builder \
-	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" --build-arg="FEATURES=${FEATURES}" \
-	  -f docker/${BASE_IMAGE}/Dockerfile \
-	  -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb-builder:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
-
-	docker run --rm -v ${PWD}:/data \
-      --entrypoint cp ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb-builder:${IMAGE_TAG} \
-      /out/target/${CARGO_PROFILE}/greptime /data/greptime
-
 .PHONY: dev-builder
 dev-builder: multi-platform-buildx ## Build dev-builder image.
 	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
 	--build-arg="RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" \
-	-f docker/dev-builder/Dockerfile \
-	-t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
+	-f docker/dev-builder/${BASE_IMAGE}/Dockerfile \
+	-t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
 
 .PHONY: multi-platform-buildx
 multi-platform-buildx: ## Create buildx multi-platform builder.
@@ -155,7 +179,7 @@ stop-etcd: ## Stop single node etcd for testing purpose.
 run-it-in-container: start-etcd ## Run integration tests in dev-builder.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v /tmp:/tmp \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
 	make test sqlness-test BUILD_JOBS=${BUILD_JOBS}
 
 ##@ General

README.md

@@ -27,6 +27,14 @@
     <a href="https://greptime.com/slack"><img src="https://img.shields.io/badge/slack-GreptimeDB-0abd59?logo=slack" alt="slack" /></a>
 </p>
 
+## Upcoming Event
+Come and meet us in **KubeCon + CloudNativeCon North America 2023!**
+<p align="center">
+  <picture>
+    <img alt="KubeCon + CloudNativeCon North Logo" src="./docs/banner/KCCNC_NA_2023_1000x200_Email Banner.png" width="800px">
+  </picture>
+</p>
+
 ## What is GreptimeDB
 
 GreptimeDB is an open-source time-series database with a special focus on
@@ -96,11 +104,11 @@ Or if you built from docker:
 docker run -p 4002:4002 -v "$(pwd):/tmp/greptimedb" greptime/greptimedb standalone start
 ```
 
-Please see [the online document site](https://docs.greptime.com/getting-started/overview#install-greptimedb) for more installation options and [operations info](https://docs.greptime.com/user-guide/operations/overview).
+Please see the online document site for more installation options and [operations info](https://docs.greptime.com/user-guide/operations/overview).
 
 ### Get started
 
-Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview#connect) on our [official document site](https://docs.greptime.com/).
+Read the [complete getting started guide](https://docs.greptime.com/getting-started/try-out-greptimedb) on our [official document site](https://docs.greptime.com/).
 
 To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients/overview).
 

benchmarks/Cargo.toml

@@ -6,8 +6,10 @@ license.workspace = true
 
 [dependencies]
 arrow.workspace = true
+chrono.workspace = true
 clap = { version = "4.0", features = ["derive"] }
 client = { workspace = true }
+futures-util.workspace = true
 indicatif = "0.17.1"
 itertools.workspace = true
 parquet.workspace = true

benchmarks/src/bin/nyc-taxi.rs

@@ -27,16 +27,16 @@ use arrow::record_batch::RecordBatch;
 use clap::Parser;
 use client::api::v1::column::Values;
 use client::api::v1::{
-    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests,
+    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests, SemanticType,
 };
-use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use client::{Client, Database, Output, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use futures_util::TryStreamExt;
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
 use tokio::task::JoinSet;
 
 const CATALOG_NAME: &str = "greptime";
 const SCHEMA_NAME: &str = "public";
-const TABLE_NAME: &str = "nyc_taxi";
 
 #[derive(Parser)]
 #[command(name = "NYC benchmark runner")]
@@ -74,7 +74,12 @@ fn get_file_list<P: AsRef<Path>>(path: P) -> Vec<PathBuf> {
         .collect()
 }
 
+fn new_table_name() -> String {
+    format!("nyc_taxi_{}", chrono::Utc::now().timestamp())
+}
+
 async fn write_data(
+    table_name: &str,
     batch_size: usize,
     db: &Database,
     path: PathBuf,
@@ -104,8 +109,7 @@ async fn write_data(
         }
         let (columns, row_count) = convert_record_batch(record_batch);
         let request = InsertRequest {
-            table_name: TABLE_NAME.to_string(),
-            region_number: 0,
+            table_name: table_name.to_string(),
             columns,
             row_count,
         };
@@ -114,7 +118,7 @@ async fn write_data(
         };
 
         let now = Instant::now();
-        let _ = db.insert(requests).await.unwrap();
+        db.insert(requests).await.unwrap();
         let elapsed = now.elapsed();
         total_rpc_elapsed_ms += elapsed.as_millis();
         progress_bar.inc(row_count as _);
@@ -132,6 +136,11 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
 
     for (array, field) in record_batch.columns().iter().zip(fields.iter()) {
         let (values, datatype) = build_values(array);
+        let semantic_type = match field.name().as_str() {
+            "VendorID" => SemanticType::Tag,
+            "tpep_pickup_datetime" => SemanticType::Timestamp,
+            _ => SemanticType::Field,
+        };
 
         let column = Column {
             column_name: field.name().clone(),
@@ -142,8 +151,7 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
                 .map(|bitmap| bitmap.buffer().as_slice().to_vec())
                 .unwrap_or_default(),
             datatype: datatype.into(),
-            // datatype and semantic_type are set to default
-            ..Default::default()
+            semantic_type: semantic_type as i32,
         };
         columns.push(column);
     }
@@ -189,7 +197,7 @@ fn build_values(column: &ArrayRef) -> (Values, ColumnDataType) {
             let values = array.values();
             (
                 Values {
-                    ts_microsecond_values: values.to_vec(),
+                    timestamp_microsecond_values: values.to_vec(),
                     ..Default::default()
                 },
                 ColumnDataType::TimestampMicrosecond,
@@ -244,156 +252,193 @@ fn is_record_batch_full(batch: &RecordBatch) -> bool {
     batch.columns().iter().all(|col| col.null_count() == 0)
 }
 
-fn create_table_expr() -> CreateTableExpr {
+fn create_table_expr(table_name: &str) -> CreateTableExpr {
     CreateTableExpr {
         catalog_name: CATALOG_NAME.to_string(),
         schema_name: SCHEMA_NAME.to_string(),
-        table_name: TABLE_NAME.to_string(),
+        table_name: table_name.to_string(),
         desc: "".to_string(),
         column_defs: vec![
             ColumnDef {
                 name: "VendorID".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                data_type: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Tag as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "tpep_pickup_datetime".to_string(),
-                datatype: ColumnDataType::TimestampMicrosecond as i32,
-                is_nullable: true,
+                data_type: ColumnDataType::TimestampMicrosecond as i32,
+                is_nullable: false,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Timestamp as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "tpep_dropoff_datetime".to_string(),
-                datatype: ColumnDataType::TimestampMicrosecond as i32,
+                data_type: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "passenger_count".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "trip_distance".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "RatecodeID".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "store_and_fwd_flag".to_string(),
-                datatype: ColumnDataType::String as i32,
+                data_type: ColumnDataType::String as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "PULocationID".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                data_type: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "DOLocationID".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                data_type: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "payment_type".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                data_type: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "fare_amount".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "extra".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "mta_tax".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "tip_amount".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "tolls_amount".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "improvement_surcharge".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "total_amount".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "congestion_surcharge".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "airport_fee".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
         ],
         time_index: "tpep_pickup_datetime".to_string(),
         primary_keys: vec!["VendorID".to_string()],
-        create_if_not_exists: false,
+        create_if_not_exists: true,
         table_options: Default::default(),
-        region_numbers: vec![0],
         table_id: None,
         engine: "mito".to_string(),
     }
 }
 
-fn query_set() -> HashMap<String, String> {
+fn query_set(table_name: &str) -> HashMap<String, String> {
     HashMap::from([
         (
             "count_all".to_string(), 
-            format!("SELECT COUNT(*) FROM {TABLE_NAME};"),
+            format!("SELECT COUNT(*) FROM {table_name};"),
         ),
         (
             "fare_amt_by_passenger".to_string(),
-            format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {TABLE_NAME} GROUP BY passenger_count"),
+            format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {table_name} GROUP BY passenger_count"),
         )
     ])
 }
 
-async fn do_write(args: &Args, db: &Database) {
+async fn do_write(args: &Args, db: &Database, table_name: &str) {
     let mut file_list = get_file_list(args.path.clone().expect("Specify data path in argument"));
     let mut write_jobs = JoinSet::new();
 
-    let create_table_result = db.create(create_table_expr()).await;
+    let create_table_result = db.create(create_table_expr(table_name)).await;
     println!("Create table result: {create_table_result:?}");
 
     let progress_bar_style = ProgressStyle::with_template(
@@ -411,8 +456,10 @@ async fn do_write(args: &Args, db: &Database) {
             let db = db.clone();
             let mpb = multi_progress_bar.clone();
             let pb_style = progress_bar_style.clone();
-            let _ = write_jobs
-                .spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+            let table_name = table_name.to_string();
+            let _ = write_jobs.spawn(async move {
+                write_data(&table_name, batch_size, &db, path, mpb, pb_style).await
+            });
         }
     }
     while write_jobs.join_next().await.is_some() {
@@ -421,24 +468,32 @@ async fn do_write(args: &Args, db: &Database) {
             let db = db.clone();
             let mpb = multi_progress_bar.clone();
             let pb_style = progress_bar_style.clone();
-            let _ = write_jobs
-                .spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+            let table_name = table_name.to_string();
+            let _ = write_jobs.spawn(async move {
+                write_data(&table_name, batch_size, &db, path, mpb, pb_style).await
+            });
         }
     }
 }
 
-async fn do_query(num_iter: usize, db: &Database) {
-    for (query_name, query) in query_set() {
+async fn do_query(num_iter: usize, db: &Database, table_name: &str) {
+    for (query_name, query) in query_set(table_name) {
         println!("Running query: {query}");
         for i in 0..num_iter {
             let now = Instant::now();
-            let _res = db.sql(&query).await.unwrap();
+            let res = db.sql(&query).await.unwrap();
+            match res {
+                Output::AffectedRows(_) | Output::RecordBatches(_) => (),
+                Output::Stream(stream) => {
+                    stream.try_collect::<Vec<_>>().await.unwrap();
+                }
+            }
             let elapsed = now.elapsed();
             println!(
                 "query {}, iteration {}: {}ms",
                 query_name,
                 i,
-                elapsed.as_millis()
+                elapsed.as_millis(),
             );
         }
     }
@@ -455,13 +510,14 @@ fn main() {
         .block_on(async {
             let client = Client::with_urls(vec![&args.endpoint]);
             let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+            let table_name = new_table_name();
 
             if !args.skip_write {
-                do_write(&args, &db).await;
+                do_write(&args, &db, &table_name).await;
             }
 
             if !args.skip_read {
-                do_query(args.iter_num, &db).await;
+                do_query(args.iter_num, &db, &table_name).await;
             }
         })
 }

config/datanode.example.toml

@@ -1,7 +1,5 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
-# Whether to use in-memory catalog, see `standalone.example.toml`.
-enable_memory_catalog = false
 # The datanode identifier, should be unique.
 node_id = 42
 # gRPC server address, "127.0.0.1:3001" by default.
@@ -10,19 +8,24 @@ rpc_addr = "127.0.0.1:3001"
 rpc_hostname = "127.0.0.1"
 # The number of gRPC server worker threads, 8 by default.
 rpc_runtime_size = 8
+# Start services after regions have obtained leases.
+# It will block the datanode start if it can't receive leases in the heartbeat from metasrv.
+require_lease_before_startup = false
 
 [heartbeat]
-# Interval for sending heartbeat messages to the Metasrv in milliseconds, 5000 by default.
-interval_millis = 5000
+# Interval for sending heartbeat messages to the Metasrv, 3 seconds by default.
+interval = "3s"
 
 # Metasrv client options.
-[meta_client_options]
+[meta_client]
 # Metasrv address list.
 metasrv_addrs = ["127.0.0.1:3002"]
-# Operation timeout in milliseconds, 3000 by default.
-timeout_millis = 3000
-# Connect server timeout in milliseconds, 5000 by default.
-connect_timeout_millis = 5000
+# Heartbeat timeout, 500 milliseconds by default.
+heartbeat_timeout = "500ms"
+# Operation timeout, 3 seconds by default.
+timeout = "3s"
+# Connect server timeout, 1 second by default.
+connect_timeout = "1s"
 # `TCP_NODELAY` option for accepted connections, true by default.
 tcp_nodelay = true
 
@@ -44,6 +47,12 @@ type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
 
+# Cache configuration for object storage such as 'S3' etc.
+# The local file cache directory
+# cache_path = "/path/local_cache"
+# The local file cache capacity in bytes.
+# cache_capacity = "256MB"
+
 # Compaction options, see `standalone.example.toml`.
 [storage.compaction]
 max_inflight_tasks = 4
@@ -57,8 +66,6 @@ max_purge_tasks = 32
 checkpoint_margin = 10
 # Region manifest logs and checkpoints gc execution duration
 gc_duration = '10m'
-# Whether to try creating a manifest checkpoint on region opening
-checkpoint_on_startup = false
 
 # Storage flush options
 [storage.flush]
@@ -73,10 +80,27 @@ auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
 
-# Procedure storage options, see `standalone.example.toml`.
-[procedure]
-max_retry_times = 3
-retry_delay = "500ms"
+# Mito engine options
+[[region_engine]]
+[region_engine.mito]
+# Number of region workers
+num_workers = 8
+# Request channel size of each worker
+worker_channel_size = 128
+# Max batch size for a worker to handle requests
+worker_request_batch_size = 64
+# Number of meta action updated to trigger a new checkpoint for the manifest
+manifest_checkpoint_distance = 10
+# Manifest compression type
+manifest_compress_type = "Uncompressed"
+# Max number of running background jobs
+max_background_jobs = 4
+# Interval to auto flush a region if it has not flushed yet.
+auto_flush_interval = "1h"
+# Global write buffer size for all regions.
+global_write_buffer_size = "1GB"
+# Global write buffer size threshold to reject write requests (default 2G).
+global_write_buffer_reject_size = "2GB"
 
 # Log options
 # [logging]

config/frontend.example.toml

@@ -2,68 +2,67 @@
 mode = "distributed"
 
 [heartbeat]
-# Interval for sending heartbeat task to the Metasrv in milliseconds, 5000 by default.
-interval_millis = 5000
-# Interval for retry sending heartbeat task in milliseconds, 5000 by default.
-retry_interval_millis = 5000
+# Interval for sending heartbeat task to the Metasrv, 5 seconds by default.
+interval = "5s"
+# Interval for retry sending heartbeat task, 5 seconds by default.
+retry_interval = "5s"
 
 # HTTP server options, see `standalone.example.toml`.
-[http_options]
+[http]
 addr = "127.0.0.1:4000"
 timeout = "30s"
 body_limit = "64MB"
 
 # gRPC server options, see `standalone.example.toml`.
-[grpc_options]
+[grpc]
 addr = "127.0.0.1:4001"
 runtime_size = 8
 
 # MySQL server options, see `standalone.example.toml`.
-[mysql_options]
+[mysql]
+enable = true
 addr = "127.0.0.1:4002"
 runtime_size = 2
 
 # MySQL server TLS options, see `standalone.example.toml`.
-[mysql_options.tls]
+[mysql.tls]
 mode = "disable"
 cert_path = ""
 key_path = ""
 
 # PostgresSQL server options, see `standalone.example.toml`.
-[postgres_options]
+[postgres]
+enable = true
 addr = "127.0.0.1:4003"
 runtime_size = 2
 
 # PostgresSQL server TLS options, see `standalone.example.toml`.
-[postgres_options.tls]
+[postgres.tls]
 mode = "disable"
 cert_path = ""
 key_path = ""
 
 # OpenTSDB protocol options, see `standalone.example.toml`.
-[opentsdb_options]
+[opentsdb]
+enable = true
 addr = "127.0.0.1:4242"
 runtime_size = 2
 
 # InfluxDB protocol options, see `standalone.example.toml`.
-[influxdb_options]
+[influxdb]
 enable = true
 
 # Prometheus remote storage options, see `standalone.example.toml`.
-[prom_store_options]
+[prom_store]
 enable = true
 
-# Prometheus protocol options, see `standalone.example.toml`.
-[prometheus_options]
-addr = "127.0.0.1:4004"
-
 # Metasrv client options, see `datanode.example.toml`.
-[meta_client_options]
+[meta_client]
 metasrv_addrs = ["127.0.0.1:3002"]
-timeout_millis = 3000
+timeout = "3s"
 # DDL timeouts options.
-ddl_timeout_millis = 10000
-connect_timeout_millis = 5000
+ddl_timeout = "10s"
+connect_timeout = "1s"
 tcp_nodelay = true
 
 # Log options, see `standalone.example.toml`

config/metasrv.example.toml

@@ -6,8 +6,6 @@ bind_addr = "127.0.0.1:3002"
 server_addr = "127.0.0.1:3002"
 # Etcd server address, "127.0.0.1:2379" by default.
 store_addr = "127.0.0.1:2379"
-# Datanode lease in seconds, 15 seconds by default.
-datanode_lease_secs = 15
 # Datanode selector type.
 # - "LeaseBased" (default value).
 # - "LoadBased"
@@ -26,7 +24,7 @@ enable_telemetry = true
 # Procedure storage options.
 [procedure]
 # Procedure max retry time.
-max_retry_times = 3
+max_retry_times = 12
 # Initial retry delay of procedures, increases exponentially
 retry_delay = "500ms"
 
@@ -34,6 +32,6 @@ retry_delay = "500ms"
 # [datanode]
 # # Datanode client options.
 # [datanode.client_options]
-# timeout_millis = 10000
-# connect_timeout_millis = 10000
+# timeout = "10s"
+# connect_timeout = "10s"
 # tcp_nodelay = true

config/standalone.example.toml

@@ -1,12 +1,10 @@
 # Node running mode, "standalone" or "distributed".
 mode = "standalone"
-# Whether to use in-memory catalog, `false` by default.
-enable_memory_catalog = false
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
 
 # HTTP server options.
-[http_options]
+[http]
 # Server address, "127.0.0.1:4000" by default.
 addr = "127.0.0.1:4000"
 # HTTP request timeout, 30s by default.
@@ -16,21 +14,23 @@ timeout = "30s"
 body_limit = "64MB"
 
 # gRPC server options.
-[grpc_options]
+[grpc]
 # Server address, "127.0.0.1:4001" by default.
 addr = "127.0.0.1:4001"
 # The number of server worker threads, 8 by default.
 runtime_size = 8
 
 # MySQL server options.
-[mysql_options]
+[mysql]
+# Whether to enable
+enable = true
 # Server address, "127.0.0.1:4002" by default.
 addr = "127.0.0.1:4002"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # MySQL server TLS options.
-[mysql_options.tls]
+[mysql.tls]
 # TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
 # - "disable" (default value)
 # - "prefer"
@@ -44,14 +44,16 @@ cert_path = ""
 key_path = ""
 
 # PostgresSQL server options.
-[postgres_options]
+[postgres]
+# Whether to enable
+enable = true
 # Server address, "127.0.0.1:4003" by default.
 addr = "127.0.0.1:4003"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # PostgresSQL server TLS options, see `[mysql_options.tls]` section.
-[postgres_options.tls]
+[postgres.tls]
 # TLS mode.
 mode = "disable"
 # certificate file path.
@@ -60,31 +62,26 @@ cert_path = ""
 key_path = ""
 
 # OpenTSDB protocol options.
-[opentsdb_options]
+[opentsdb]
+# Whether to enable
+enable = true
 # OpenTSDB telnet API server address, "127.0.0.1:4242" by default.
 addr = "127.0.0.1:4242"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # InfluxDB protocol options.
-[influxdb_options]
+[influxdb]
 # Whether to enable InfluxDB protocol in HTTP API, true by default.
 enable = true
 
 # Prometheus remote storage options
-[prom_store_options]
+[prom_store]
 # Whether to enable Prometheus remote write and read in HTTP API, true by default.
 enable = true
 
-# Prometheus protocol options
-[prometheus_options]
-# Prometheus API server address, "127.0.0.1:4004" by default.
-addr = "127.0.0.1:4004"
-
 # WAL options.
 [wal]
-# WAL data directory
-# dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
 file_size = "256MB"
 # WAL purge threshold.
@@ -96,6 +93,20 @@ read_batch_size = 128
 # Whether to sync log file after every write.
 sync_write = false
 
+# Metadata storage options.
+[metadata_store]
+# Kv file size in bytes.
+file_size = "256MB"
+# Kv purge threshold.
+purge_threshold = "4GB"
+
+# Procedure storage options.
+[procedure]
+# Procedure max retry time.
+max_retry_times = 3
+# Initial retry delay of procedures, increases exponentially
+retry_delay = "500ms"
+
 # Storage options.
 [storage]
 # The working home directory.
@@ -104,6 +115,10 @@ data_home = "/tmp/greptimedb/"
 type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
+# Cache configuration for object storage such as 'S3' etc.
+# cache_path = "/path/local_cache"
+# The local file cache capacity in bytes.
+# cache_capacity = "256MB"
 
 # Compaction options.
 [storage.compaction]
@@ -121,8 +136,6 @@ max_purge_tasks = 32
 checkpoint_margin = 10
 # Region manifest logs and checkpoints gc execution duration
 gc_duration = '10m'
-# Whether to try creating a manifest checkpoint on region opening
-checkpoint_on_startup = false
 
 # Storage flush options
 [storage.flush]
@@ -137,13 +150,6 @@ auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
 
-# Procedure storage options.
-[procedure]
-# Procedure max retry time.
-max_retry_times = 3
-# Initial retry delay of procedures, increases exponentially
-retry_delay = "500ms"
-
 # Log options
 # [logging]
 # Specify logs directory.

docker/buildx/centos/Dockerfile

@@ -2,6 +2,7 @@ FROM centos:7 as builder
 
 ARG CARGO_PROFILE
 ARG FEATURES
+ARG OUTPUT_DIR
 
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
@@ -13,7 +14,8 @@ RUN yum install -y epel-release  \
     openssl-devel  \
     centos-release-scl  \
     rh-python38  \
-    rh-python38-python-devel
+    rh-python38-python-devel \
+    which
 
 # Install protoc
 RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
@@ -35,17 +37,18 @@ RUN --mount=target=.,rw \
 # Export the binary to the clean image.
 FROM centos:7 as base
 
-ARG CARGO_PROFILE
+ARG OUTPUT_DIR
 
 RUN yum install -y epel-release \
     openssl \
     openssl-devel  \
     centos-release-scl  \
     rh-python38  \
-    rh-python38-python-devel
+    rh-python38-python-devel \
+    which
 
 WORKDIR /greptime
-COPY --from=builder /out/target/${CARGO_PROFILE}/greptime /greptime/bin/
+COPY --from=builder /out/target/${OUTPUT_DIR}/greptime /greptime/bin/
 ENV PATH /greptime/bin/:$PATH
 
 ENTRYPOINT ["greptime"]

docker/buildx/ubuntu/Dockerfile

@@ -1,11 +1,17 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:20.04 as builder
 
 ARG CARGO_PROFILE
 ARG FEATURES
+ARG OUTPUT_DIR
 
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
+# Add PPA for Python 3.10.
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa -y
+
 # Install dependencies.
 RUN --mount=type=cache,target=/var/cache/apt \
     apt-get update && apt-get install -y \
@@ -25,7 +31,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-mo
 ENV PATH /root/.cargo/bin/:$PATH
 
 # Build the project in release mode.
-RUN --mount=target=.,rw \
+RUN --mount=target=. \
     --mount=type=cache,target=/root/.cargo/registry \
     make build \
     CARGO_PROFILE=${CARGO_PROFILE} \
@@ -36,7 +42,7 @@ RUN --mount=target=.,rw \
 # TODO(zyy17): Maybe should use the more secure container image.
 FROM ubuntu:22.04 as base
 
-ARG CARGO_PROFILE
+ARG OUTPUT_DIR
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get \
     -y install ca-certificates \
@@ -50,7 +56,7 @@ COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt
 RUN python3 -m pip install -r /etc/greptime/requirements.txt
 
 WORKDIR /greptime
-COPY --from=builder /out/target/${CARGO_PROFILE}/greptime /greptime/bin/
+COPY --from=builder /out/target/${OUTPUT_DIR}/greptime /greptime/bin/
 ENV PATH /greptime/bin/:$PATH
 
 ENTRYPOINT ["greptime"]

docker/dev-builder/android/Dockerfile

@@ -0,0 +1,41 @@
+FROM --platform=linux/amd64 saschpe/android-ndk:34-jdk17.0.8_7-ndk25.2.9519653-cmake3.22.1
+
+ENV LANG en_US.utf8
+WORKDIR /greptimedb
+
+# Rename libunwind to libgcc
+RUN cp ${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/14.0.7/lib/linux/aarch64/libunwind.a ${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/14.0.7/lib/linux/aarch64/libgcc.a
+
+# Install dependencies.
+RUN apt-get update && apt-get install -y \
+    libssl-dev \
+    protobuf-compiler \
+    curl \
+    git \
+    build-essential \
+    pkg-config \
+    python3 \
+    python3-dev \
+    python3-pip \
+    && pip3 install --upgrade pip \
+    && pip3 install pyarrow
+
+# Trust workdir
+RUN git config --global --add safe.directory /greptimedb
+
+# Install Rust.
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+
+# Add android toolchains
+ARG RUST_TOOLCHAIN
+RUN rustup toolchain install ${RUST_TOOLCHAIN}
+RUN rustup target add aarch64-linux-android
+
+# Install cargo-ndk
+RUN cargo install cargo-ndk
+ENV ANDROID_NDK_HOME $NDK_ROOT
+
+# Builder entrypoint.
+CMD ["cargo", "ndk", "--platform", "23", "-t", "aarch64-linux-android", "build", "--bin", "greptime", "--profile", "release", "--no-default-features"]

docker/dev-builder/centos/Dockerfile

@@ -0,0 +1,29 @@
+FROM centos:7 as builder
+
+ENV LANG en_US.utf8
+
+# Install dependencies
+RUN ulimit -n 1024000 && yum groupinstall -y 'Development Tools'
+RUN yum install -y epel-release  \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel \
+    which
+
+# Install protoc
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
+RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
+
+# Install Rust
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH
+
+# Install Rust toolchains.
+ARG RUST_TOOLCHAIN
+RUN rustup toolchain install ${RUST_TOOLCHAIN}
+
+# Install nextest.
+RUN cargo install cargo-nextest --locked

docker/dev-builder/ubuntu/Dockerfile

@@ -1,8 +1,13 @@
-FROM ubuntu:22.04
+FROM ubuntu:20.04
 
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
+# Add PPA for Python 3.10.
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa -y
+
 # Install dependencies.
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     libssl-dev \

docs/benchmarks/tsbs/v0.4.0.md

@@ -0,0 +1,61 @@
+# TSBS benchmark - v0.4.0
+
+## Environment
+
+### Local
+|        |                                    |
+| ------ | ---------------------------------- |
+| CPU    | AMD Ryzen 7 7735HS (8 core 3.2GHz) |
+| Memory | 32GB                               |
+| Disk   | SOLIDIGM SSDPFKNU010TZ             |
+| OS     | Ubuntu 22.04.2 LTS                 |
+
+### Aliyun amd64
+
+|         |                |
+| ------- | -------------- |
+| Machine | ecs.g7.4xlarge |
+| CPU     | 16 core        |
+| Memory  | 64GB           |
+| Disk    | 100G           |
+| OS      | Ubuntu  22.04  |
+
+### Aliyun arm64
+
+|         |                   |
+| ------- | ----------------- |
+| Machine | ecs.g8y.4xlarge   |
+| CPU     | 16 core           |
+| Memory  | 64GB              |
+| Disk    | 100G              |
+| OS      | Ubuntu  22.04 ARM |
+
+
+## Write performance
+
+| Environment        | Ingest rate（rows/s） |
+| ------------------ | --------------------- |
+| Local              | 365280.60             |
+| Aliyun g7.4xlarge  | 341368.72             |
+| Aliyun g8y.4xlarge | 320907.29             |
+
+
+## Query performance
+
+| Query type            | Local (ms) | Aliyun g7.4xlarge (ms) | Aliyun g8y.4xlarge (ms) |
+| --------------------- | ---------- | ---------------------- | ----------------------- |
+| cpu-max-all-1         | 50.70      | 31.46                  | 47.61                   |
+| cpu-max-all-8         | 262.16     | 129.26                 | 152.43                  |
+| double-groupby-1      | 2512.71    | 1408.19                | 1586.10                 |
+| double-groupby-5      | 3896.15    | 2304.29                | 2585.29                 |
+| double-groupby-all    | 5404.67    | 3337.61                | 3773.91                 |
+| groupby-orderby-limit | 3786.98    | 2065.72                | 2312.57                 |
+| high-cpu-1            | 71.96      | 37.29                  | 54.01                   |
+| high-cpu-all          | 9468.75    | 7595.69                | 8467.46                 |
+| lastpoint             | 13379.43   | 11253.76               | 12949.40                |
+| single-groupby-1-1-1  | 20.72      | 12.16                  | 13.35                   |
+| single-groupby-1-1-12 | 28.53      | 15.67                  | 21.62                   |
+| single-groupby-1-8-1  | 72.23      | 37.90                  | 43.52                   |
+| single-groupby-5-1-1  | 26.75      | 15.59                  | 17.48                   |
+| single-groupby-5-1-12 | 45.41      | 22.90                  | 31.96                   |
+| single-groupby-5-8-1  | 107.96     | 59.76                  | 69.58                   |

docs/rfcs/2023-08-13-metadata-txn.md

@@ -0,0 +1,90 @@
+---
+Feature Name: Update Metadata in single transaction
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/1715
+Date: 2023-08-13
+Author: "Feng Yangsen <fengys1996@gmail.com>, Xu Wenkang <wenymedia@gmail.com>"
+---
+
+# Summary
+Update Metadata in single transaction.
+
+# Motivation
+Currently, multiple transactions are involved during the procedure. This implementation is inefficient, and it's hard to make data consistent. Therefore, We can update multiple metadata in a single transaction.
+
+# Details 
+Now we have the following table metadata keys:
+
+**TableInfo** 
+```rust
+// __table_info/{table_id}
+pub struct TableInfoKey {
+    table_id: TableId,
+}
+
+pub struct TableInfoValue {
+    pub table_info: RawTableInfo,
+    version: u64,
+}
+```
+
+**TableRoute** 
+```rust
+// __table_route/{table_id}
+pub struct NextTableRouteKey {
+    table_id: TableId,
+}
+
+pub struct TableRoute {
+    pub region_routes: Vec<RegionRoute>,
+}
+```
+**DatanodeTable**
+```rust
+// __table_route/{datanode_id}/{table_id}
+pub struct DatanodeTableKey {
+    datanode_id: DatanodeId,
+    table_id: TableId,
+}
+
+pub struct DatanodeTableValue {
+    pub table_id: TableId,
+    pub regions: Vec<RegionNumber>,
+    version: u64,
+}
+```
+
+**TableNameKey**
+```rust
+// __table_name/{CatalogName}/{SchemaName}/{TableName}
+pub struct TableNameKey<'a> {
+    pub catalog: &'a str,
+    pub schema: &'a str,
+    pub table: &'a str,
+}
+
+pub struct TableNameValue {
+    table_id: TableId,
+}
+```
+
+These table metadata only updates in the following operations.
+
+## Region Failover
+It needs to update `TableRoute` key and `DatanodeTable` keys. If the `TableRoute` equals the Snapshot of `TableRoute` submitting the Failover task, then we can safely update these keys.
+
+After submitting Failover tasks to acquire locks for execution, the `TableRoute` may be updated by another task. After acquiring the lock, we can get the latest `TableRoute` again and then execute it if needed.
+
+## Create Table DDL
+Creates all of the above keys. `TableRoute`, `TableInfo`, should be empty.
+
+The **TableNameKey**'s lock will be held by the procedure framework.
+## Drop Table DDL
+
+`TableInfoKey` and `NextTableRouteKey` will be added with  `__removed-` prefix, and the other above keys will be deleted.  The transaction will not compare any keys.
+## Alter Table DDL
+
+1. Rename table, updates `TableInfo` and `TableName`. Compares `TableInfo`, and the new `TableNameKey` should be empty, and TableInfo should equal the Snapshot when submitting DDL.
+
+The old and new **TableNameKey**'s lock will be held by the procedure framework.
+
+2. Alter table, updates `TableInfo`. `TableInfo` should equal the Snapshot when submitting DDL.

src/api/Cargo.toml

@@ -7,6 +7,7 @@ license.workspace = true
 [dependencies]
 common-base = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-time = { workspace = true }
 datatypes = { workspace = true }
 greptime-proto.workspace = true
@@ -16,3 +17,6 @@ tonic.workspace = true
 
 [build-dependencies]
 tonic-build = "0.9"
+
+[dev-dependencies]
+paste = "1.0"

src/api/src/error.rs

@@ -16,14 +16,16 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
 use snafu::Location;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unknown proto column datatype: {}", datatype))]
     UnknownColumnDataType { datatype: i32, location: Location },
@@ -34,22 +36,14 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display(
-        "Failed to convert column default constraint, column: {}, source: {}",
-        column,
-        source
-    ))]
+    #[snafu(display("Failed to convert column default constraint, column: {}", column))]
     ConvertColumnDefaultConstraint {
         column: String,
         location: Location,
         source: datatypes::error::Error,
     },
 
-    #[snafu(display(
-        "Invalid column default constraint, column: {}, source: {}",
-        column,
-        source
-    ))]
+    #[snafu(display("Invalid column default constraint, column: {}", column))]
     InvalidColumnDefaultConstraint {
         column: String,
         location: Location,

src/api/src/v1/column_def.rs

@@ -12,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
+use std::collections::HashMap;
+
+use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, COMMENT_KEY};
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
@@ -20,7 +22,7 @@ use crate::helper::ColumnDataTypeWrapper;
 use crate::v1::ColumnDef;
 
 pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
-    let data_type = ColumnDataTypeWrapper::try_new(column_def.datatype)?;
+    let data_type = ColumnDataTypeWrapper::try_new(column_def.data_type)?;
 
     let constraint = if column_def.default_constraint.is_empty() {
         None
@@ -34,9 +36,17 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
         )
     };
 
-    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
-        .with_default_constraint(constraint)
-        .context(error::InvalidColumnDefaultConstraintSnafu {
-            column: &column_def.name,
-        })
+    let mut metadata = HashMap::new();
+    if !column_def.comment.is_empty() {
+        metadata.insert(COMMENT_KEY.to_string(), column_def.comment.clone());
+    }
+
+    Ok(
+        ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
+            .with_default_constraint(constraint)
+            .context(error::InvalidColumnDefaultConstraintSnafu {
+                column: &column_def.name,
+            })?
+            .with_metadata(metadata),
+    )
 }

src/auth/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "auth"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[features]
+default = []
+testing = []
+
+[dependencies]
+api.workspace = true
+async-trait.workspace = true
+common-error.workspace = true
+common-macro.workspace = true
+digest = "0.10"
+hex = { version = "0.4" }
+secrecy = { version = "0.8", features = ["serde", "alloc"] }
+sha1 = "0.10"
+snafu.workspace = true
+sql.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+common-test-util.workspace = true

src/auth/src/common.rs

@@ -0,0 +1,147 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use digest::Digest;
+use secrecy::SecretString;
+use sha1::Sha1;
+use snafu::{ensure, OptionExt};
+
+use crate::error::{IllegalParamSnafu, InvalidConfigSnafu, Result, UserPasswordMismatchSnafu};
+use crate::user_info::DefaultUserInfo;
+use crate::user_provider::static_user_provider::{StaticUserProvider, STATIC_USER_PROVIDER};
+use crate::{UserInfoRef, UserProviderRef};
+
+pub(crate) const DEFAULT_USERNAME: &str = "greptime";
+
+/// construct a [`UserInfo`](crate::user_info::UserInfo) impl with name
+/// use default username `greptime` if None is provided
+pub fn userinfo_by_name(username: Option<String>) -> UserInfoRef {
+    DefaultUserInfo::with_name(username.unwrap_or_else(|| DEFAULT_USERNAME.to_string()))
+}
+
+pub fn user_provider_from_option(opt: &String) -> Result<UserProviderRef> {
+    let (name, content) = opt.split_once(':').context(InvalidConfigSnafu {
+        value: opt.to_string(),
+        msg: "UserProviderOption must be in format `<option>:<value>`",
+    })?;
+    match name {
+        STATIC_USER_PROVIDER => {
+            let provider =
+                StaticUserProvider::try_from(content).map(|p| Arc::new(p) as UserProviderRef)?;
+            Ok(provider)
+        }
+        _ => InvalidConfigSnafu {
+            value: name.to_string(),
+            msg: "Invalid UserProviderOption",
+        }
+        .fail(),
+    }
+}
+
+type Username<'a> = &'a str;
+type HostOrIp<'a> = &'a str;
+
+#[derive(Debug, Clone)]
+pub enum Identity<'a> {
+    UserId(Username<'a>, Option<HostOrIp<'a>>),
+}
+
+pub type HashedPassword<'a> = &'a [u8];
+pub type Salt<'a> = &'a [u8];
+
+/// Authentication information sent by the client.
+pub enum Password<'a> {
+    PlainText(SecretString),
+    MysqlNativePassword(HashedPassword<'a>, Salt<'a>),
+    PgMD5(HashedPassword<'a>, Salt<'a>),
+}
+
+pub fn auth_mysql(
+    auth_data: HashedPassword,
+    salt: Salt,
+    username: &str,
+    save_pwd: &[u8],
+) -> Result<()> {
+    ensure!(
+        auth_data.len() == 20,
+        IllegalParamSnafu {
+            msg: "Illegal mysql password length"
+        }
+    );
+    // ref: https://github.com/mysql/mysql-server/blob/a246bad76b9271cb4333634e954040a970222e0a/sql/auth/password.cc#L62
+    let hash_stage_2 = double_sha1(save_pwd);
+    let tmp = sha1_two(salt, &hash_stage_2);
+    // xor auth_data and tmp
+    let mut xor_result = [0u8; 20];
+    for i in 0..20 {
+        xor_result[i] = auth_data[i] ^ tmp[i];
+    }
+    let candidate_stage_2 = sha1_one(&xor_result);
+    if candidate_stage_2 == hash_stage_2 {
+        Ok(())
+    } else {
+        UserPasswordMismatchSnafu {
+            username: username.to_string(),
+        }
+        .fail()
+    }
+}
+
+fn sha1_two(input_1: &[u8], input_2: &[u8]) -> Vec<u8> {
+    let mut hasher = Sha1::new();
+    hasher.update(input_1);
+    hasher.update(input_2);
+    hasher.finalize().to_vec()
+}
+
+fn sha1_one(data: &[u8]) -> Vec<u8> {
+    let mut hasher = Sha1::new();
+    hasher.update(data);
+    hasher.finalize().to_vec()
+}
+
+fn double_sha1(data: &[u8]) -> Vec<u8> {
+    sha1_one(&sha1_one(data))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_sha() {
+        let sha_1_answer: Vec<u8> = vec![
+            124, 74, 141, 9, 202, 55, 98, 175, 97, 229, 149, 32, 148, 61, 194, 100, 148, 248, 148,
+            27,
+        ];
+        let sha_1 = sha1_one("123456".as_bytes());
+        assert_eq!(sha_1, sha_1_answer);
+
+        let double_sha1_answer: Vec<u8> = vec![
+            107, 180, 131, 126, 183, 67, 41, 16, 94, 228, 86, 141, 218, 125, 198, 126, 210, 202,
+            42, 217,
+        ];
+        let double_sha1 = double_sha1("123456".as_bytes());
+        assert_eq!(double_sha1, double_sha1_answer);
+
+        let sha1_2_answer: Vec<u8> = vec![
+            132, 115, 215, 211, 99, 186, 164, 206, 168, 152, 217, 192, 117, 47, 240, 252, 142, 244,
+            37, 204,
+        ];
+        let sha1_2 = sha1_two("123456".as_bytes(), "654321".as_bytes());
+        assert_eq!(sha1_2, sha1_2_answer);
+    }
+}

src/auth/src/error.rs

@@ -0,0 +1,93 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_error::ext::{BoxedError, ErrorExt};
+use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
+use snafu::{Location, Snafu};
+
+#[derive(Snafu)]
+#[snafu(visibility(pub))]
+#[stack_trace_debug]
+pub enum Error {
+    #[snafu(display("Invalid config value: {}, {}", value, msg))]
+    InvalidConfig { value: String, msg: String },
+
+    #[snafu(display("Illegal param: {}", msg))]
+    IllegalParam { msg: String },
+
+    #[snafu(display("Internal state error: {}", msg))]
+    InternalState { msg: String },
+
+    #[snafu(display("IO error"))]
+    Io {
+        #[snafu(source)]
+        error: std::io::Error,
+        location: Location,
+    },
+
+    #[snafu(display("Auth failed"))]
+    AuthBackend {
+        location: Location,
+        source: BoxedError,
+    },
+
+    #[snafu(display("User not found, username: {}", username))]
+    UserNotFound { username: String },
+
+    #[snafu(display("Unsupported password type: {}", password_type))]
+    UnsupportedPasswordType { password_type: String },
+
+    #[snafu(display("Username and password does not match, username: {}", username))]
+    UserPasswordMismatch { username: String },
+
+    #[snafu(display(
+        "Access denied for user '{}' to database '{}-{}'",
+        username,
+        catalog,
+        schema
+    ))]
+    AccessDenied {
+        catalog: String,
+        schema: String,
+        username: String,
+    },
+
+    #[snafu(display("User is not authorized to perform this action"))]
+    PermissionDenied { location: Location },
+}
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        match self {
+            Error::InvalidConfig { .. } => StatusCode::InvalidArguments,
+            Error::IllegalParam { .. } => StatusCode::InvalidArguments,
+            Error::InternalState { .. } => StatusCode::Unexpected,
+            Error::Io { .. } => StatusCode::Internal,
+            Error::AuthBackend { .. } => StatusCode::Internal,
+
+            Error::UserNotFound { .. } => StatusCode::UserNotFound,
+            Error::UnsupportedPasswordType { .. } => StatusCode::UnsupportedPasswordType,
+            Error::UserPasswordMismatch { .. } => StatusCode::UserPasswordMismatch,
+            Error::AccessDenied { .. } => StatusCode::AccessDenied,
+            Error::PermissionDenied { .. } => StatusCode::PermissionDenied,
+        }
+    }
+
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+}
+
+pub type Result<T> = std::result::Result<T, Error>;

src/auth/src/lib.rs

@@ -0,0 +1,34 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod common;
+pub mod error;
+mod permission;
+mod user_info;
+mod user_provider;
+
+#[cfg(feature = "testing")]
+pub mod tests;
+
+pub use common::{
+    auth_mysql, user_provider_from_option, userinfo_by_name, HashedPassword, Identity, Password,
+};
+pub use permission::{PermissionChecker, PermissionReq, PermissionResp};
+pub use user_info::UserInfo;
+pub use user_provider::UserProvider;
+
+/// pub type alias
+pub type UserInfoRef = std::sync::Arc<dyn UserInfo>;
+pub type UserProviderRef = std::sync::Arc<dyn UserProvider>;
+pub type PermissionCheckerRef = std::sync::Arc<dyn PermissionChecker>;

src/auth/src/permission.rs

@@ -0,0 +1,64 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+
+use api::v1::greptime_request::Request;
+use sql::statements::statement::Statement;
+
+use crate::error::{PermissionDeniedSnafu, Result};
+use crate::{PermissionCheckerRef, UserInfoRef};
+
+#[derive(Debug, Clone)]
+pub enum PermissionReq<'a> {
+    GrpcRequest(&'a Request),
+    SqlStatement(&'a Statement),
+    PromQuery,
+    Opentsdb,
+    LineProtocol,
+    PromStoreWrite,
+    PromStoreRead,
+    Otlp,
+}
+
+#[derive(Debug)]
+pub enum PermissionResp {
+    Allow,
+    Reject,
+}
+
+pub trait PermissionChecker: Send + Sync {
+    fn check_permission(
+        &self,
+        user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> Result<PermissionResp>;
+}
+
+impl PermissionChecker for Option<&PermissionCheckerRef> {
+    fn check_permission(
+        &self,
+        user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> Result<PermissionResp> {
+        match self {
+            Some(checker) => match checker.check_permission(user_info, req) {
+                Ok(PermissionResp::Reject) => PermissionDeniedSnafu.fail(),
+                Ok(PermissionResp::Allow) => Ok(PermissionResp::Allow),
+                Err(e) => Err(e),
+            },
+            None => Ok(PermissionResp::Allow),
+        }
+    }
+}

src/auth/src/tests.rs

@@ -11,14 +11,14 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 use secrecy::ExposeSecret;
-use servers::auth::user_provider::auth_mysql;
-use servers::auth::{
-    AccessDeniedSnafu, Identity, Password, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu,
-    UserPasswordMismatchSnafu, UserProvider,
+
+use crate::error::{
+    AccessDeniedSnafu, Result, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu,
+    UserPasswordMismatchSnafu,
 };
-use session::context::UserInfo;
+use crate::user_info::DefaultUserInfo;
+use crate::{auth_mysql, Identity, Password, UserInfoRef, UserProvider};
 
 pub struct DatabaseAuthInfo<'a> {
     pub catalog: &'a str,
@@ -56,17 +56,13 @@ impl UserProvider for MockUserProvider {
         "mock_user_provider"
     }
 
-    async fn authenticate(
-        &self,
-        id: Identity<'_>,
-        password: Password<'_>,
-    ) -> servers::auth::Result<UserInfo> {
+    async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfoRef> {
         match id {
             Identity::UserId(username, _host) => match password {
                 Password::PlainText(password) => {
                     if username == "greptime" {
                         if password.expose_secret() == "greptime" {
-                            Ok(UserInfo::new("greptime"))
+                            Ok(DefaultUserInfo::with_name("greptime"))
                         } else {
                             UserPasswordMismatchSnafu {
                                 username: username.to_string(),
@@ -82,7 +78,7 @@ impl UserProvider for MockUserProvider {
                 }
                 Password::MysqlNativePassword(auth_data, salt) => {
                     auth_mysql(auth_data, salt, username, "greptime".as_bytes())
-                        .map(|_| UserInfo::new(username))
+                        .map(|_| DefaultUserInfo::with_name(username))
                 }
                 _ => UnsupportedPasswordTypeSnafu {
                     password_type: "mysql_native_password",
@@ -92,12 +88,7 @@ impl UserProvider for MockUserProvider {
         }
     }
 
-    async fn authorize(
-        &self,
-        catalog: &str,
-        schema: &str,
-        user_info: &UserInfo,
-    ) -> servers::auth::Result<()> {
+    async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfoRef) -> Result<()> {
         if catalog == self.catalog && schema == self.schema && user_info.username() == self.username
         {
             Ok(())
@@ -114,6 +105,8 @@ impl UserProvider for MockUserProvider {
 
 #[tokio::test]
 async fn test_auth_by_plain_text() {
+    use crate::error;
+
     let user_provider = MockUserProvider::default();
     assert_eq!("mock_user_provider", user_provider.name());
 
@@ -137,7 +130,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UnsupportedPasswordType { .. }
+        error::Error::UnsupportedPasswordType { .. }
     ));
 
     // auth failed, err: user not exist.
@@ -150,7 +143,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UserNotFound { .. }
+        error::Error::UserNotFound { .. }
     ));
 
     // auth failed, err: wrong password
@@ -163,7 +156,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UserPasswordMismatch { .. }
+        error::Error::UserPasswordMismatch { .. }
     ))
 }
 
@@ -176,8 +169,8 @@ async fn test_schema_validate() {
         username: "test_user",
     });
 
-    let right_user = UserInfo::new("test_user");
-    let wrong_user = UserInfo::default();
+    let right_user = DefaultUserInfo::with_name("test_user");
+    let wrong_user = DefaultUserInfo::with_name("greptime");
 
     // check catalog
     let re = validator

src/auth/src/user_info.rs

@@ -0,0 +1,47 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use crate::UserInfoRef;
+
+pub trait UserInfo: Debug + Sync + Send {
+    fn as_any(&self) -> &dyn Any;
+    fn username(&self) -> &str;
+}
+
+#[derive(Debug)]
+pub(crate) struct DefaultUserInfo {
+    username: String,
+}
+
+impl DefaultUserInfo {
+    pub(crate) fn with_name(username: impl Into<String>) -> UserInfoRef {
+        Arc::new(Self {
+            username: username.into(),
+        })
+    }
+}
+
+impl UserInfo for DefaultUserInfo {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn username(&self) -> &str {
+        self.username.as_str()
+    }
+}

src/auth/src/user_provider.rs

@@ -0,0 +1,46 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) mod static_user_provider;
+
+use crate::common::{Identity, Password};
+use crate::error::Result;
+use crate::UserInfoRef;
+
+#[async_trait::async_trait]
+pub trait UserProvider: Send + Sync {
+    fn name(&self) -> &str;
+
+    /// Checks whether a user is valid and allowed to access the database.
+    async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfoRef>;
+
+    /// Checks whether a connection request
+    /// from a certain user to a certain catalog/schema is legal.
+    /// This method should be called after [authenticate()](UserProvider::authenticate()).
+    async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfoRef) -> Result<()>;
+
+    /// Combination of [authenticate()](UserProvider::authenticate()) and [authorize()](UserProvider::authorize()).
+    /// In most cases it's preferred for both convenience and performance.
+    async fn auth(
+        &self,
+        id: Identity<'_>,
+        password: Password<'_>,
+        catalog: &str,
+        schema: &str,
+    ) -> Result<UserInfoRef> {
+        let user_info = self.authenticate(id, password).await?;
+        self.authorize(catalog, schema, &user_info).await?;
+        Ok(user_info)
+    }
+}

src/auth/src/user_provider/static_user_provider.rs

@@ -19,20 +19,17 @@ use std::io::BufRead;
 use std::path::Path;
 
 use async_trait::async_trait;
-use digest;
-use digest::Digest;
 use secrecy::ExposeSecret;
-use session::context::UserInfo;
-use sha1::Sha1;
 use snafu::{ensure, OptionExt, ResultExt};
 
-use crate::auth::{
-    Error, HashedPassword, Identity, IllegalParamSnafu, InvalidConfigSnafu, IoSnafu, Password,
-    Result, Salt, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu, UserPasswordMismatchSnafu,
-    UserProvider,
+use crate::error::{
+    Error, IllegalParamSnafu, InvalidConfigSnafu, IoSnafu, Result, UnsupportedPasswordTypeSnafu,
+    UserNotFoundSnafu, UserPasswordMismatchSnafu,
 };
+use crate::user_info::DefaultUserInfo;
+use crate::{auth_mysql, Identity, Password, UserInfoRef, UserProvider};
 
-pub const STATIC_USER_PROVIDER: &str = "static_user_provider";
+pub(crate) const STATIC_USER_PROVIDER: &str = "static_user_provider";
 
 impl TryFrom<&str> for StaticUserProvider {
     type Error = Error;
@@ -91,7 +88,7 @@ impl TryFrom<&str> for StaticUserProvider {
     }
 }
 
-pub struct StaticUserProvider {
+pub(crate) struct StaticUserProvider {
     users: HashMap<String, Vec<u8>>,
 }
 
@@ -105,7 +102,7 @@ impl UserProvider for StaticUserProvider {
         &self,
         input_id: Identity<'_>,
         input_pwd: Password<'_>,
-    ) -> Result<UserInfo> {
+    ) -> Result<UserInfoRef> {
         match input_id {
             Identity::UserId(username, _) => {
                 ensure!(
@@ -127,7 +124,7 @@ impl UserProvider for StaticUserProvider {
                             }
                         );
                         return if save_pwd == pwd.expose_secret().as_bytes() {
-                            Ok(UserInfo::new(username))
+                            Ok(DefaultUserInfo::with_name(username))
                         } else {
                             UserPasswordMismatchSnafu {
                                 username: username.to_string(),
@@ -136,14 +133,8 @@ impl UserProvider for StaticUserProvider {
                         };
                     }
                     Password::MysqlNativePassword(auth_data, salt) => {
-                        ensure!(
-                            auth_data.len() == 20,
-                            IllegalParamSnafu {
-                                msg: "Illegal MySQL native password format, length != 20"
-                            }
-                        );
                         auth_mysql(auth_data, salt, username, save_pwd)
-                            .map(|_| UserInfo::new(username))
+                            .map(|_| DefaultUserInfo::with_name(username))
                     }
                     Password::PgMD5(_, _) => UnsupportedPasswordTypeSnafu {
                         password_type: "pg_md5",
@@ -154,88 +145,28 @@ impl UserProvider for StaticUserProvider {
         }
     }
 
-    async fn authorize(&self, _catalog: &str, _schema: &str, _user_info: &UserInfo) -> Result<()> {
+    async fn authorize(
+        &self,
+        _catalog: &str,
+        _schema: &str,
+        _user_info: &UserInfoRef,
+    ) -> Result<()> {
         // default allow all
         Ok(())
     }
 }
 
-pub fn auth_mysql(
-    auth_data: HashedPassword,
-    salt: Salt,
-    username: &str,
-    save_pwd: &[u8],
-) -> Result<()> {
-    // ref: https://github.com/mysql/mysql-server/blob/a246bad76b9271cb4333634e954040a970222e0a/sql/auth/password.cc#L62
-    let hash_stage_2 = double_sha1(save_pwd);
-    let tmp = sha1_two(salt, &hash_stage_2);
-    // xor auth_data and tmp
-    let mut xor_result = [0u8; 20];
-    for i in 0..20 {
-        xor_result[i] = auth_data[i] ^ tmp[i];
-    }
-    let candidate_stage_2 = sha1_one(&xor_result);
-    if candidate_stage_2 == hash_stage_2 {
-        Ok(())
-    } else {
-        UserPasswordMismatchSnafu {
-            username: username.to_string(),
-        }
-        .fail()
-    }
-}
-
-fn sha1_two(input_1: &[u8], input_2: &[u8]) -> Vec<u8> {
-    let mut hasher = Sha1::new();
-    hasher.update(input_1);
-    hasher.update(input_2);
-    hasher.finalize().to_vec()
-}
-
-fn sha1_one(data: &[u8]) -> Vec<u8> {
-    let mut hasher = Sha1::new();
-    hasher.update(data);
-    hasher.finalize().to_vec()
-}
-
-fn double_sha1(data: &[u8]) -> Vec<u8> {
-    sha1_one(&sha1_one(data))
-}
-
 #[cfg(test)]
 pub mod test {
     use std::fs::File;
     use std::io::{LineWriter, Write};
 
     use common_test_util::temp_dir::create_temp_dir;
-    use session::context::UserInfo;
 
-    use crate::auth::user_provider::{double_sha1, sha1_one, sha1_two, StaticUserProvider};
-    use crate::auth::{Identity, Password, UserProvider};
-
-    #[test]
-    fn test_sha() {
-        let sha_1_answer: Vec<u8> = vec![
-            124, 74, 141, 9, 202, 55, 98, 175, 97, 229, 149, 32, 148, 61, 194, 100, 148, 248, 148,
-            27,
-        ];
-        let sha_1 = sha1_one("123456".as_bytes());
-        assert_eq!(sha_1, sha_1_answer);
-
-        let double_sha1_answer: Vec<u8> = vec![
-            107, 180, 131, 126, 183, 67, 41, 16, 94, 228, 86, 141, 218, 125, 198, 126, 210, 202,
-            42, 217,
-        ];
-        let double_sha1 = double_sha1("123456".as_bytes());
-        assert_eq!(double_sha1, double_sha1_answer);
-
-        let sha1_2_answer: Vec<u8> = vec![
-            132, 115, 215, 211, 99, 186, 164, 206, 168, 152, 217, 192, 117, 47, 240, 252, 142, 244,
-            37, 204,
-        ];
-        let sha1_2 = sha1_two("123456".as_bytes(), "654321".as_bytes());
-        assert_eq!(sha1_2, sha1_2_answer);
-    }
+    use crate::user_info::DefaultUserInfo;
+    use crate::user_provider::static_user_provider::StaticUserProvider;
+    use crate::user_provider::{Identity, Password};
+    use crate::UserProvider;
 
     async fn test_authenticate(provider: &dyn UserProvider, username: &str, password: &str) {
         let re = provider
@@ -249,9 +180,10 @@ pub mod test {
 
     #[tokio::test]
     async fn test_authorize() {
+        let user_info = DefaultUserInfo::with_name("root");
         let provider = StaticUserProvider::try_from("cmd:root=123456,admin=654321").unwrap();
         provider
-            .authorize("catalog", "schema", &UserInfo::new("root"))
+            .authorize("catalog", "schema", &user_info)
             .await
             .unwrap();
     }

src/auth/tests/mod.rs

@@ -0,0 +1,61 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(assert_matches)]
+use std::assert_matches::assert_matches;
+use std::sync::Arc;
+
+use api::v1::greptime_request::Request;
+use auth::error::Error::InternalState;
+use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq, PermissionResp, UserInfoRef};
+use sql::statements::show::{ShowDatabases, ShowKind};
+use sql::statements::statement::Statement;
+
+struct DummyPermissionChecker;
+
+impl PermissionChecker for DummyPermissionChecker {
+    fn check_permission(
+        &self,
+        _user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> auth::error::Result<PermissionResp> {
+        match req {
+            PermissionReq::GrpcRequest(_) => Ok(PermissionResp::Allow),
+            PermissionReq::SqlStatement(_) => Ok(PermissionResp::Reject),
+            _ => Err(InternalState {
+                msg: "testing".to_string(),
+            }),
+        }
+    }
+}
+
+#[test]
+fn test_permission_checker() {
+    let checker: PermissionCheckerRef = Arc::new(DummyPermissionChecker);
+
+    let grpc_result = checker.check_permission(
+        None,
+        PermissionReq::GrpcRequest(&Request::Query(Default::default())),
+    );
+    assert_matches!(grpc_result, Ok(PermissionResp::Allow));
+
+    let sql_result = checker.check_permission(
+        None,
+        PermissionReq::SqlStatement(&Statement::ShowDatabases(ShowDatabases::new(ShowKind::All))),
+    );
+    assert_matches!(sql_result, Ok(PermissionResp::Reject));
+
+    let err_result = checker.check_permission(None, PermissionReq::Opentsdb);
+    assert_matches!(err_result, Err(InternalState { msg }) if msg == "testing");
+}

src/catalog/Cargo.toml

@@ -16,6 +16,7 @@ async-trait = "0.1"
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
@@ -30,8 +31,9 @@ futures-util.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 metrics.workspace = true
-moka = { version = "0.11", features = ["future"] }
+moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
+partition.workspace = true
 regex.workspace = true
 serde.workspace = true
 serde_json = "1.0"
@@ -46,7 +48,6 @@ catalog = { workspace = true, features = ["testing"] }
 chrono.workspace = true
 common-test-util = { workspace = true }
 log-store = { workspace = true }
-mito = { workspace = true, features = ["test"] }
 object-store = { workspace = true }
 storage = { workspace = true }
 tokio.workspace = true

src/catalog/src/error.rs

@@ -17,41 +17,48 @@ use std::fmt::Debug;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
+use table::metadata::TableId;
 use tokio::task::JoinError;
 
-use crate::DeregisterTableRequest;
-
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display(
-        "Failed to re-compile script due to internal error, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to list catalogs"))]
+    ListCatalogs {
+        location: Location,
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to list {}'s schemas", catalog))]
+    ListSchemas {
+        location: Location,
+        catalog: String,
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to re-compile script due to internal error"))]
     CompileScriptInternal {
         location: Location,
         source: BoxedError,
     },
-    #[snafu(display("Failed to open system catalog table, source: {}", source))]
+    #[snafu(display("Failed to open system catalog table"))]
     OpenSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to create system catalog table, source: {}", source))]
+    #[snafu(display("Failed to create system catalog table"))]
     CreateSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display(
-        "Failed to create table, table info: {}, source: {}",
-        table_info,
-        source
-    ))]
+    #[snafu(display("Failed to create table, table info: {}", table_info))]
     CreateTable {
         table_info: String,
         location: Location,
@@ -85,13 +92,14 @@ pub enum Error {
     #[snafu(display("Catalog value is not present"))]
     EmptyValue { location: Location },
 
-    #[snafu(display("Failed to deserialize value, source: {}", source))]
+    #[snafu(display("Failed to deserialize value"))]
     ValueDeserialize {
-        source: serde_json::error::Error,
+        #[snafu(source)]
+        error: serde_json::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
+    #[snafu(display("Table engine not found: {}", engine_name))]
     TableEngineNotFound {
         engine_name: String,
         location: Location,
@@ -129,15 +137,18 @@ pub enum Error {
     #[snafu(display("Operation {} not supported", op))]
     NotSupported { op: String, location: Location },
 
-    #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
+    #[snafu(display("Failed to open table {table_id}"))]
     OpenTable {
-        table_info: String,
+        table_id: TableId,
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to open table in parallel, source: {}", source))]
-    ParallelOpenTable { source: JoinError },
+    #[snafu(display("Failed to open table in parallel"))]
+    ParallelOpenTable {
+        #[snafu(source)]
+        error: JoinError,
+    },
 
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
     TableNotFound {
@@ -151,72 +162,52 @@ pub enum Error {
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to create recordbatch, source: {}", source))]
+    #[snafu(display("Failed to create recordbatch"))]
     CreateRecordBatch {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display(
-        "Failed to insert table creation record to system catalog, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to insert table creation record to system catalog"))]
     InsertCatalogRecord {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display(
-        "Failed to deregister table, request: {:?}, source: {}",
-        request,
-        source
-    ))]
-    DeregisterTable {
-        request: DeregisterTableRequest,
-        location: Location,
-        source: table::error::Error,
-    },
-
-    #[snafu(display("Illegal catalog manager state: {}", msg))]
-    IllegalManagerState { location: Location, msg: String },
-
-    #[snafu(display("Failed to scan system catalog table, source: {}", source))]
+    #[snafu(display("Failed to scan system catalog table"))]
     SystemCatalogTableScan {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("{source}"))]
+    #[snafu(display(""))]
     Internal {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display(
-        "Failed to upgrade weak catalog manager reference. location: {}",
-        location
-    ))]
+    #[snafu(display("Failed to upgrade weak catalog manager reference"))]
     UpgradeWeakCatalogManagerRef { location: Location },
 
-    #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
+    #[snafu(display("Failed to execute system catalog table scan"))]
     SystemCatalogTableScanExec {
         location: Location,
         source: common_query::error::Error,
     },
 
-    #[snafu(display("Cannot parse catalog value, source: {}", source))]
+    #[snafu(display("Cannot parse catalog value"))]
     InvalidCatalogValue {
         location: Location,
         source: common_catalog::error::Error,
     },
 
-    #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
+    #[snafu(display("Failed to perform metasrv operation"))]
     MetaSrv {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Invalid table info in catalog, source: {}", source))]
+    #[snafu(display("Invalid table info in catalog"))]
     InvalidTableInfoInCatalog {
         location: Location,
         source: datatypes::error::Error,
@@ -225,17 +216,14 @@ pub enum Error {
     #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
     QueryAccessDenied { catalog: String, schema: String },
 
-    #[snafu(display("Invalid system table definition: {err_msg}"))]
-    InvalidSystemTableDef { err_msg: String, location: Location },
-
-    #[snafu(display("{}: {}", msg, source))]
+    #[snafu(display(""))]
     Datafusion {
-        msg: String,
-        source: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         location: Location,
     },
 
-    #[snafu(display("Table schema mismatch, source: {}", source))]
+    #[snafu(display("Table schema mismatch"))]
     TableSchemaMismatch {
         location: Location,
         source: table::error::Error,
@@ -244,7 +232,7 @@ pub enum Error {
     #[snafu(display("A generic error has occurred, msg: {}", msg))]
     Generic { msg: String, location: Location },
 
-    #[snafu(display("Table metadata manager error: {}", source))]
+    #[snafu(display("Table metadata manager error"))]
     TableMetadataManager {
         source: common_meta::error::Error,
         location: Location,
@@ -259,10 +247,8 @@ impl ErrorExt for Error {
             Error::InvalidKey { .. }
             | Error::SchemaNotFound { .. }
             | Error::TableNotFound { .. }
-            | Error::IllegalManagerState { .. }
             | Error::CatalogNotFound { .. }
             | Error::InvalidEntryType { .. }
-            | Error::InvalidSystemTableDef { .. }
             | Error::ParallelOpenTable { .. } => StatusCode::Unexpected,
 
             Error::SystemCatalog { .. }
@@ -284,12 +270,15 @@ impl ErrorExt for Error {
                 StatusCode::InvalidArguments
             }
 
+            Error::ListCatalogs { source, .. } | Error::ListSchemas { source, .. } => {
+                source.status_code()
+            }
+
             Error::OpenSystemCatalog { source, .. }
             | Error::CreateSystemCatalog { source, .. }
             | Error::InsertCatalogRecord { source, .. }
             | Error::OpenTable { source, .. }
             | Error::CreateTable { source, .. }
-            | Error::DeregisterTable { source, .. }
             | Error::TableSchemaMismatch { source, .. } => source.status_code(),
 
             Error::MetaSrv { source, .. } => source.status_code(),

src/catalog/src/information_schema.rs

@@ -15,30 +15,27 @@
 mod columns;
 mod tables;
 
-use std::any::Any;
 use std::collections::HashMap;
 use std::sync::{Arc, Weak};
 
-use async_trait::async_trait;
-use common_catalog::consts::{
-    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME,
-    INFORMATION_SCHEMA_TABLES_TABLE_ID,
-};
+use common_catalog::consts::INFORMATION_SCHEMA_NAME;
 use common_error::ext::BoxedError;
 use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
 use datatypes::schema::SchemaRef;
 use futures_util::StreamExt;
 use snafu::ResultExt;
+use store_api::data_source::DataSource;
 use store_api::storage::{ScanRequest, TableId};
-use table::data_source::DataSource;
 use table::error::{SchemaConversionSnafu, TablesRecordBatchSnafu};
-use table::metadata::{TableIdent, TableInfoBuilder, TableMetaBuilder, TableType};
-use table::{Result as TableResult, Table, TableRef};
+use table::metadata::{
+    FilterPushDownType, TableInfoBuilder, TableInfoRef, TableMetaBuilder, TableType,
+};
+use table::thin_table::{ThinTable, ThinTableAdapter};
+use table::TableRef;
 
 use self::columns::InformationSchemaColumns;
 use crate::error::Result;
 use crate::information_schema::tables::InformationSchemaTables;
-use crate::table_factory::TableFactory;
 use crate::CatalogManager;
 
 pub const TABLES: &str = "tables";
@@ -63,192 +60,117 @@ impl InformationSchemaProvider {
         catalog_name: String,
         catalog_manager: Weak<dyn CatalogManager>,
     ) -> HashMap<String, TableRef> {
+        let provider = Self::new(catalog_name, catalog_manager);
+
         let mut schema = HashMap::new();
-
-        schema.insert(
-            TABLES.to_string(),
-            Arc::new(InformationTable::new(
-                catalog_name.clone(),
-                INFORMATION_SCHEMA_TABLES_TABLE_ID,
-                TABLES.to_string(),
-                Arc::new(InformationSchemaTables::new(
-                    catalog_name.clone(),
-                    catalog_manager.clone(),
-                )),
-            )) as _,
-        );
-        schema.insert(
-            COLUMNS.to_string(),
-            Arc::new(InformationTable::new(
-                catalog_name.clone(),
-                INFORMATION_SCHEMA_COLUMNS_TABLE_ID,
-                COLUMNS.to_string(),
-                Arc::new(InformationSchemaColumns::new(catalog_name, catalog_manager)),
-            )) as _,
-        );
-
+        schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap());
+        schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap());
         schema
     }
 
-    pub fn table(&self, name: &str) -> Result<Option<TableRef>> {
-        let (stream_builder, table_id) = match name.to_ascii_lowercase().as_ref() {
-            TABLES => (
-                Arc::new(InformationSchemaTables::new(
-                    self.catalog_name.clone(),
-                    self.catalog_manager.clone(),
-                )) as _,
-                INFORMATION_SCHEMA_TABLES_TABLE_ID,
-            ),
-            COLUMNS => (
-                Arc::new(InformationSchemaColumns::new(
-                    self.catalog_name.clone(),
-                    self.catalog_manager.clone(),
-                )) as _,
-                INFORMATION_SCHEMA_COLUMNS_TABLE_ID,
-            ),
-            _ => {
-                return Ok(None);
-            }
-        };
+    pub fn table(&self, name: &str) -> Option<TableRef> {
+        self.information_table(name).map(|table| {
+            let table_info = Self::table_info(self.catalog_name.clone(), &table);
+            let filter_pushdown = FilterPushDownType::Unsupported;
+            let thin_table = ThinTable::new(table_info, filter_pushdown);
 
-        Ok(Some(Arc::new(InformationTable::new(
-            self.catalog_name.clone(),
-            table_id,
-            name.to_string(),
-            stream_builder,
-        ))))
+            let data_source = Arc::new(InformationTableDataSource::new(table));
+            Arc::new(ThinTableAdapter::new(thin_table, data_source)) as _
+        })
     }
 
-    pub fn table_factory(&self, name: &str) -> Result<Option<TableFactory>> {
-        let (stream_builder, table_id) = match name.to_ascii_lowercase().as_ref() {
-            TABLES => (
-                Arc::new(InformationSchemaTables::new(
-                    self.catalog_name.clone(),
-                    self.catalog_manager.clone(),
-                )) as _,
-                INFORMATION_SCHEMA_TABLES_TABLE_ID,
-            ),
-            COLUMNS => (
-                Arc::new(InformationSchemaColumns::new(
-                    self.catalog_name.clone(),
-                    self.catalog_manager.clone(),
-                )) as _,
-                INFORMATION_SCHEMA_COLUMNS_TABLE_ID,
-            ),
-            _ => {
-                return Ok(None);
-            }
-        };
-        let data_source = Arc::new(InformationTable::new(
-            self.catalog_name.clone(),
-            table_id,
-            name.to_string(),
-            stream_builder,
-        ));
-
-        Ok(Some(Arc::new(move || data_source.clone())))
-    }
-}
-
-// TODO(ruihang): make it a more generic trait:
-// https://github.com/GreptimeTeam/greptimedb/pull/1639#discussion_r1205001903
-pub trait InformationStreamBuilder: Send + Sync {
-    fn to_stream(&self) -> Result<SendableRecordBatchStream>;
-
-    fn schema(&self) -> SchemaRef;
-}
-
-pub struct InformationTable {
-    catalog_name: String,
-    table_id: TableId,
-    name: String,
-    stream_builder: Arc<dyn InformationStreamBuilder>,
-}
-
-impl InformationTable {
-    pub fn new(
-        catalog_name: String,
-        table_id: TableId,
-        name: String,
-        stream_builder: Arc<dyn InformationStreamBuilder>,
-    ) -> Self {
-        Self {
-            catalog_name,
-            table_id,
-            name,
-            stream_builder,
+    fn information_table(&self, name: &str) -> Option<InformationTableRef> {
+        match name.to_ascii_lowercase().as_str() {
+            TABLES => Some(Arc::new(InformationSchemaTables::new(
+                self.catalog_name.clone(),
+                self.catalog_manager.clone(),
+            )) as _),
+            COLUMNS => Some(Arc::new(InformationSchemaColumns::new(
+                self.catalog_name.clone(),
+                self.catalog_manager.clone(),
+            )) as _),
+            _ => None,
         }
     }
-}
 
-#[async_trait]
-impl Table for InformationTable {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.stream_builder.schema()
-    }
-
-    fn table_info(&self) -> table::metadata::TableInfoRef {
+    fn table_info(catalog_name: String, table: &InformationTableRef) -> TableInfoRef {
         let table_meta = TableMetaBuilder::default()
-            .schema(self.stream_builder.schema())
+            .schema(table.schema())
             .primary_key_indices(vec![])
             .next_column_id(0)
             .build()
             .unwrap();
-        Arc::new(
-            TableInfoBuilder::default()
-                .ident(TableIdent {
-                    table_id: self.table_id,
-                    version: 0,
-                })
-                .name(self.name.clone())
-                .catalog_name(self.catalog_name.clone())
-                .schema_name(INFORMATION_SCHEMA_NAME.to_string())
-                .meta(table_meta)
-                .table_type(TableType::Temporary)
-                .build()
-                .unwrap(),
-        )
+        let table_info = TableInfoBuilder::default()
+            .table_id(table.table_id())
+            .name(table.table_name().to_owned())
+            .catalog_name(catalog_name)
+            .schema_name(INFORMATION_SCHEMA_NAME.to_owned())
+            .meta(table_meta)
+            .table_type(table.table_type())
+            .build()
+            .unwrap();
+        Arc::new(table_info)
     }
+}
+
+trait InformationTable {
+    fn table_id(&self) -> TableId;
+
+    fn table_name(&self) -> &'static str;
+
+    fn schema(&self) -> SchemaRef;
+
+    fn to_stream(&self) -> Result<SendableRecordBatchStream>;
 
     fn table_type(&self) -> TableType {
         TableType::Temporary
     }
+}
 
-    async fn scan_to_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
-        self.get_stream(request)
+type InformationTableRef = Arc<dyn InformationTable + Send + Sync>;
+
+struct InformationTableDataSource {
+    table: InformationTableRef,
+}
+
+impl InformationTableDataSource {
+    fn new(table: InformationTableRef) -> Self {
+        Self { table }
+    }
+
+    fn try_project(&self, projection: &[usize]) -> std::result::Result<SchemaRef, BoxedError> {
+        let schema = self
+            .table
+            .schema()
+            .try_project(projection)
+            .context(SchemaConversionSnafu)
+            .map_err(BoxedError::new)?;
+        Ok(Arc::new(schema))
     }
 }
 
-impl DataSource for InformationTable {
-    fn get_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
+impl DataSource for InformationTableDataSource {
+    fn get_stream(
+        &self,
+        request: ScanRequest,
+    ) -> std::result::Result<SendableRecordBatchStream, BoxedError> {
         let projection = request.projection;
-        let projected_schema = if let Some(projection) = &projection {
-            Arc::new(
-                self.schema()
-                    .try_project(projection)
-                    .context(SchemaConversionSnafu)?,
-            )
-        } else {
-            self.schema()
+        let projected_schema = match &projection {
+            Some(projection) => self.try_project(projection)?,
+            None => self.table.schema(),
         };
+
         let stream = self
-            .stream_builder
+            .table
             .to_stream()
             .map_err(BoxedError::new)
-            .context(TablesRecordBatchSnafu)?
-            .map(move |batch| {
-                batch.and_then(|batch| {
-                    if let Some(projection) = &projection {
-                        batch.try_project(projection)
-                    } else {
-                        Ok(batch)
-                    }
-                })
+            .context(TablesRecordBatchSnafu)
+            .map_err(BoxedError::new)?
+            .map(move |batch| match &projection {
+                Some(p) => batch.and_then(|b| b.try_project(p)),
+                None => batch,
             });
+
         let stream = RecordBatchStreamAdaptor {
             schema: projected_schema,
             stream: Box::pin(stream),

src/catalog/src/information_schema/columns.rs

@@ -16,8 +16,8 @@ use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::{
-    INFORMATION_SCHEMA_NAME, SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_PRIMARY_KEY,
-    SEMANTIC_TYPE_TIME_INDEX,
+    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME, SEMANTIC_TYPE_FIELD,
+    SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
 };
 use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
@@ -31,9 +31,10 @@ use datatypes::scalars::ScalarVectorBuilder;
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::vectors::{StringVectorBuilder, VectorRef};
 use snafu::{OptionExt, ResultExt};
+use store_api::storage::TableId;
 
 use super::tables::InformationSchemaTables;
-use super::{InformationStreamBuilder, COLUMNS, TABLES};
+use super::{InformationTable, COLUMNS, TABLES};
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
@@ -81,7 +82,15 @@ impl InformationSchemaColumns {
     }
 }
 
-impl InformationStreamBuilder for InformationSchemaColumns {
+impl InformationTable for InformationSchemaColumns {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_COLUMNS_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        COLUMNS
+    }
+
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -149,7 +158,7 @@ impl InformationSchemaColumnsBuilder {
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
-                .schema_exist(&catalog_name, &schema_name)
+                .schema_exists(&catalog_name, &schema_name)
                 .await?
             {
                 continue;

src/catalog/src/information_schema/tables.rs

@@ -30,13 +30,14 @@ use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder};
 use snafu::{OptionExt, ResultExt};
+use store_api::storage::TableId;
 use table::metadata::TableType;
 
 use super::{COLUMNS, TABLES};
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
-use crate::information_schema::InformationStreamBuilder;
+use crate::information_schema::InformationTable;
 use crate::CatalogManager;
 
 pub(super) struct InformationSchemaTables {
@@ -74,7 +75,15 @@ impl InformationSchemaTables {
     }
 }
 
-impl InformationStreamBuilder for InformationSchemaTables {
+impl InformationTable for InformationSchemaTables {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_TABLES_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        TABLES
+    }
+
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -145,7 +154,7 @@ impl InformationSchemaTablesBuilder {
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
-                .schema_exist(&catalog_name, &schema_name)
+                .schema_exists(&catalog_name, &schema_name)
                 .await?
             {
                 continue;

src/catalog/src/kvbackend.rs

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod immutable;
-mod procedure;
-#[cfg(test)]
-mod tests;
+pub use client::{CachedMetaKvBackend, MetaKvBackend};
 
-use table::metadata::TableVersion;
+mod client;
+mod manager;
 
-const INIT_TABLE_VERSION: TableVersion = 0;
+#[cfg(feature = "testing")]
+pub mod mock;
+pub use manager::KvBackendCatalogManager;

src/catalog/src/kvbackend/client.rs

@@ -18,8 +18,9 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use common_error::ext::BoxedError;
+use common_meta::cache_invalidator::KvCacheInvalidator;
 use common_meta::error::Error::{CacheNotGet, GetKvCache};
-use common_meta::error::{CacheNotGetSnafu, Error, MetaSrvSnafu, Result};
+use common_meta::error::{CacheNotGetSnafu, Error, ExternalSnafu, Result};
 use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
 use common_meta::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
@@ -28,12 +29,11 @@ use common_meta::rpc::store::{
     RangeRequest, RangeResponse,
 };
 use common_meta::rpc::KeyValue;
-use common_telemetry::timer;
+use common_telemetry::{debug, timer};
 use meta_client::client::MetaClient;
 use moka::future::{Cache, CacheBuilder};
 use snafu::{OptionExt, ResultExt};
 
-use super::KvCacheInvalidator;
 use crate::metrics::{METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET};
 
 const CACHE_MAX_CAPACITY: u64 = 10000;
@@ -197,7 +197,8 @@ impl KvBackend for CachedMetaKvBackend {
 #[async_trait::async_trait]
 impl KvCacheInvalidator for CachedMetaKvBackend {
     async fn invalidate_key(&self, key: &[u8]) {
-        self.cache.invalidate(key).await
+        self.cache.invalidate(key).await;
+        debug!("invalidated cache key: {}", String::from_utf8_lossy(key));
     }
 }
 
@@ -251,7 +252,7 @@ impl KvBackend for MetaKvBackend {
             .range(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
@@ -260,7 +261,7 @@ impl KvBackend for MetaKvBackend {
             .range(RangeRequest::new().with_key(key))
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)?;
+            .context(ExternalSnafu)?;
         Ok(response.take_kvs().get_mut(0).map(|kv| KeyValue {
             key: kv.take_key(),
             value: kv.take_value(),
@@ -272,7 +273,7 @@ impl KvBackend for MetaKvBackend {
             .batch_put(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn put(&self, req: PutRequest) -> Result<PutResponse> {
@@ -280,7 +281,7 @@ impl KvBackend for MetaKvBackend {
             .put(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
@@ -288,7 +289,7 @@ impl KvBackend for MetaKvBackend {
             .delete_range(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
@@ -296,7 +297,7 @@ impl KvBackend for MetaKvBackend {
             .batch_delete(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
@@ -304,7 +305,7 @@ impl KvBackend for MetaKvBackend {
             .batch_get(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn compare_and_put(
@@ -315,7 +316,7 @@ impl KvBackend for MetaKvBackend {
             .compare_and_put(request)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
@@ -323,7 +324,7 @@ impl KvBackend for MetaKvBackend {
             .move_value(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     fn as_any(&self) -> &dyn Any {

src/catalog/src/kvbackend/manager.rs

@@ -0,0 +1,292 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::collections::BTreeSet;
+use std::sync::{Arc, Weak};
+
+use common_catalog::consts::{DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, NUMBERS_TABLE_ID};
+use common_error::ext::BoxedError;
+use common_meta::cache_invalidator::{CacheInvalidator, CacheInvalidatorRef, Context};
+use common_meta::datanode_manager::DatanodeManagerRef;
+use common_meta::error::Result as MetaResult;
+use common_meta::key::catalog_name::CatalogNameKey;
+use common_meta::key::schema_name::SchemaNameKey;
+use common_meta::key::table_name::TableNameKey;
+use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
+use common_meta::kv_backend::KvBackendRef;
+use common_meta::table_name::TableName;
+use futures_util::TryStreamExt;
+use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
+use snafu::prelude::*;
+use table::dist_table::DistTable;
+use table::metadata::TableId;
+use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+use table::TableRef;
+
+use crate::error::{
+    self as catalog_err, ListCatalogsSnafu, ListSchemasSnafu, Result as CatalogResult,
+    TableMetadataManagerSnafu,
+};
+use crate::information_schema::{InformationSchemaProvider, COLUMNS, TABLES};
+use crate::CatalogManager;
+
+/// Access all existing catalog, schema and tables.
+///
+/// The result comes from two source, all the user tables are presented in
+/// a kv-backend which persists the metadata of a table. And system tables
+/// comes from `SystemCatalog`, which is static and read-only.
+#[derive(Clone)]
+pub struct KvBackendCatalogManager {
+    // TODO(LFC): Maybe use a real implementation for Standalone mode.
+    // Now we use `NoopKvCacheInvalidator` for Standalone mode. In Standalone mode, the KV backend
+    // is implemented by RaftEngine. Maybe we need a cache for it?
+    cache_invalidator: CacheInvalidatorRef,
+    partition_manager: PartitionRuleManagerRef,
+    table_metadata_manager: TableMetadataManagerRef,
+    datanode_manager: DatanodeManagerRef,
+    /// A sub-CatalogManager that handles system tables
+    system_catalog: SystemCatalog,
+}
+
+#[async_trait::async_trait]
+impl CacheInvalidator for KvBackendCatalogManager {
+    async fn invalidate_table_name(&self, ctx: &Context, table_name: TableName) -> MetaResult<()> {
+        self.cache_invalidator
+            .invalidate_table_name(ctx, table_name)
+            .await
+    }
+
+    async fn invalidate_table_id(&self, ctx: &Context, table_id: TableId) -> MetaResult<()> {
+        self.cache_invalidator
+            .invalidate_table_id(ctx, table_id)
+            .await
+    }
+}
+
+impl KvBackendCatalogManager {
+    pub fn new(
+        backend: KvBackendRef,
+        cache_invalidator: CacheInvalidatorRef,
+        datanode_manager: DatanodeManagerRef,
+    ) -> Arc<Self> {
+        Arc::new_cyclic(|me| Self {
+            partition_manager: Arc::new(PartitionRuleManager::new(backend.clone())),
+            table_metadata_manager: Arc::new(TableMetadataManager::new(backend)),
+            cache_invalidator,
+            datanode_manager,
+            system_catalog: SystemCatalog {
+                catalog_manager: me.clone(),
+            },
+        })
+    }
+
+    pub fn partition_manager(&self) -> PartitionRuleManagerRef {
+        self.partition_manager.clone()
+    }
+
+    pub fn table_metadata_manager_ref(&self) -> &TableMetadataManagerRef {
+        &self.table_metadata_manager
+    }
+
+    pub fn datanode_manager(&self) -> DatanodeManagerRef {
+        self.datanode_manager.clone()
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for KvBackendCatalogManager {
+    async fn catalog_names(&self) -> CatalogResult<Vec<String>> {
+        let stream = self
+            .table_metadata_manager
+            .catalog_manager()
+            .catalog_names()
+            .await;
+
+        let keys = stream
+            .try_collect::<Vec<_>>()
+            .await
+            .map_err(BoxedError::new)
+            .context(ListCatalogsSnafu)?;
+
+        Ok(keys)
+    }
+
+    async fn schema_names(&self, catalog: &str) -> CatalogResult<Vec<String>> {
+        let stream = self
+            .table_metadata_manager
+            .schema_manager()
+            .schema_names(catalog)
+            .await;
+        let mut keys = stream
+            .try_collect::<BTreeSet<_>>()
+            .await
+            .map_err(BoxedError::new)
+            .context(ListSchemasSnafu { catalog })?
+            .into_iter()
+            .collect::<Vec<_>>();
+
+        keys.extend_from_slice(&self.system_catalog.schema_names());
+
+        Ok(keys)
+    }
+
+    async fn table_names(&self, catalog: &str, schema: &str) -> CatalogResult<Vec<String>> {
+        let mut tables = self
+            .table_metadata_manager
+            .table_name_manager()
+            .tables(catalog, schema)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .into_iter()
+            .map(|(k, _)| k)
+            .collect::<Vec<String>>();
+        tables.extend_from_slice(&self.system_catalog.table_names(schema));
+
+        Ok(tables)
+    }
+
+    async fn catalog_exists(&self, catalog: &str) -> CatalogResult<bool> {
+        self.table_metadata_manager
+            .catalog_manager()
+            .exists(CatalogNameKey::new(catalog))
+            .await
+            .context(TableMetadataManagerSnafu)
+    }
+
+    async fn schema_exists(&self, catalog: &str, schema: &str) -> CatalogResult<bool> {
+        if self.system_catalog.schema_exist(schema) {
+            return Ok(true);
+        }
+
+        self.table_metadata_manager
+            .schema_manager()
+            .exists(SchemaNameKey::new(catalog, schema))
+            .await
+            .context(TableMetadataManagerSnafu)
+    }
+
+    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> CatalogResult<bool> {
+        if self.system_catalog.table_exist(schema, table) {
+            return Ok(true);
+        }
+
+        let key = TableNameKey::new(catalog, schema, table);
+        self.table_metadata_manager
+            .table_name_manager()
+            .get(key)
+            .await
+            .context(TableMetadataManagerSnafu)
+            .map(|x| x.is_some())
+    }
+
+    async fn table(
+        &self,
+        catalog: &str,
+        schema: &str,
+        table_name: &str,
+    ) -> CatalogResult<Option<TableRef>> {
+        if let Some(table) = self.system_catalog.table(catalog, schema, table_name) {
+            return Ok(Some(table));
+        }
+
+        let key = TableNameKey::new(catalog, schema, table_name);
+        let Some(table_name_value) = self
+            .table_metadata_manager
+            .table_name_manager()
+            .get(key)
+            .await
+            .context(TableMetadataManagerSnafu)?
+        else {
+            return Ok(None);
+        };
+        let table_id = table_name_value.table_id();
+
+        let Some(table_info_value) = self
+            .table_metadata_manager
+            .table_info_manager()
+            .get(table_id)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .map(|v| v.into_inner())
+        else {
+            return Ok(None);
+        };
+        let table_info = Arc::new(
+            table_info_value
+                .table_info
+                .try_into()
+                .context(catalog_err::InvalidTableInfoInCatalogSnafu)?,
+        );
+        Ok(Some(DistTable::table(table_info)))
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+// TODO: This struct can hold a static map of all system tables when
+// the upper layer (e.g., procedure) can inform the catalog manager
+// a new catalog is created.
+/// Existing system tables:
+/// - public.numbers
+/// - information_schema.tables
+/// - information_schema.columns
+#[derive(Clone)]
+struct SystemCatalog {
+    catalog_manager: Weak<KvBackendCatalogManager>,
+}
+
+impl SystemCatalog {
+    fn schema_names(&self) -> Vec<String> {
+        vec![INFORMATION_SCHEMA_NAME.to_string()]
+    }
+
+    fn table_names(&self, schema: &str) -> Vec<String> {
+        if schema == INFORMATION_SCHEMA_NAME {
+            vec![TABLES.to_string(), COLUMNS.to_string()]
+        } else if schema == DEFAULT_SCHEMA_NAME {
+            vec![NUMBERS_TABLE_NAME.to_string()]
+        } else {
+            vec![]
+        }
+    }
+
+    fn schema_exist(&self, schema: &str) -> bool {
+        schema == INFORMATION_SCHEMA_NAME
+    }
+
+    fn table_exist(&self, schema: &str, table: &str) -> bool {
+        if schema == INFORMATION_SCHEMA_NAME {
+            table == TABLES || table == COLUMNS
+        } else if schema == DEFAULT_SCHEMA_NAME {
+            table == NUMBERS_TABLE_NAME
+        } else {
+            false
+        }
+    }
+
+    fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Option<TableRef> {
+        if schema == INFORMATION_SCHEMA_NAME {
+            let information_schema_provider =
+                InformationSchemaProvider::new(catalog.to_string(), self.catalog_manager.clone());
+            information_schema_provider.table(table_name)
+        } else if schema == DEFAULT_SCHEMA_NAME && table_name == NUMBERS_TABLE_NAME {
+            Some(NumbersTable::table(NUMBERS_TABLE_ID))
+        } else {
+            None
+        }
+    }
+}

src/catalog/src/kvbackend/mock.rs

@@ -55,14 +55,14 @@ impl TableEngine for MockTableEngine {
 
         let data = vec![Arc::new(StringVector::from(vec!["a", "b", "c"])) as _];
         let record_batch = RecordBatch::new(schema, data).unwrap();
-        let table: TableRef = Arc::new(MemTable::new_with_catalog(
+        let table = MemTable::new_with_catalog(
             &request.table_name,
             record_batch,
             table_id,
             request.catalog_name,
             request.schema_name,
             vec![0],
-        )) as Arc<_>;
+        );
 
         let mut tables = self.tables.write().unwrap();
         let _ = tables.insert(table_id, table.clone() as TableRef);

src/catalog/src/lib.rs

@@ -17,80 +17,38 @@
 #![feature(try_blocks)]
 
 use std::any::Any;
-use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use api::v1::meta::{RegionStat, TableIdent, TableName};
-use common_telemetry::{info, warn};
-use snafu::ResultExt;
-use table::engine::{EngineContext, TableEngineRef};
+use futures::future::BoxFuture;
 use table::metadata::TableId;
 use table::requests::CreateTableRequest;
 use table::TableRef;
 
-use crate::error::{CreateTableSnafu, Result};
+use crate::error::Result;
 
 pub mod error;
 pub mod information_schema;
-pub mod local;
+pub mod kvbackend;
+pub mod memory;
 mod metrics;
-pub mod remote;
-pub mod system;
-pub mod table_factory;
 pub mod table_source;
-pub mod tables;
 
 #[async_trait::async_trait]
 pub trait CatalogManager: Send + Sync {
     fn as_any(&self) -> &dyn Any;
 
-    /// Starts a catalog manager.
-    async fn start(&self) -> Result<()>;
-
-    /// Registers a catalog to catalog manager, returns whether the catalog exist before.
-    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool>;
-
-    /// Register a schema with catalog name and schema name. Retuens whether the
-    /// schema registered.
-    ///
-    /// # Errors
-    ///
-    /// This method will/should fail if catalog not exist
-    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool>;
-
-    /// Deregisters a database within given catalog/schema to catalog manager
-    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool>;
-
-    /// Registers a table within given catalog/schema to catalog manager,
-    /// returns whether the table registered.
-    ///
-    /// # Errors
-    ///
-    /// This method will/should fail if catalog or schema not exist
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
-
-    /// Deregisters a table within given catalog/schema to catalog manager
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()>;
-
-    /// Rename a table to [RenameTableRequest::new_table_name], returns whether the table is renamed.
-    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool>;
-
-    /// Register a system table, should be called before starting the manager.
-    async fn register_system_table(&self, request: RegisterSystemTableRequest)
-        -> error::Result<()>;
-
     async fn catalog_names(&self) -> Result<Vec<String>>;
 
     async fn schema_names(&self, catalog: &str) -> Result<Vec<String>>;
 
     async fn table_names(&self, catalog: &str, schema: &str) -> Result<Vec<String>>;
 
-    async fn catalog_exist(&self, catalog: &str) -> Result<bool>;
+    async fn catalog_exists(&self, catalog: &str) -> Result<bool>;
 
-    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool>;
+    async fn schema_exists(&self, catalog: &str, schema: &str) -> Result<bool>;
 
-    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
+    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
 
     /// Returns the table by catalog, schema and table name.
     async fn table(
@@ -104,7 +62,8 @@ pub trait CatalogManager: Send + Sync {
 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
 
 /// Hook called after system table opening.
-pub type OpenSystemTableHook = Arc<dyn Fn(TableRef) -> Result<()> + Send + Sync>;
+pub type OpenSystemTableHook =
+    Box<dyn Fn(TableRef) -> BoxFuture<'static, Result<()>> + Send + Sync>;
 
 /// Register system table request:
 /// - When system table is already created and registered, the hook will be called
@@ -163,116 +122,3 @@ pub struct RegisterSchemaRequest {
     pub catalog: String,
     pub schema: String,
 }
-
-pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
-    manager: &'a M,
-    engine: TableEngineRef,
-    sys_table_requests: &'a mut Vec<RegisterSystemTableRequest>,
-) -> Result<()> {
-    for req in sys_table_requests.drain(..) {
-        let catalog_name = &req.create_table_request.catalog_name;
-        let schema_name = &req.create_table_request.schema_name;
-        let table_name = &req.create_table_request.table_name;
-        let table_id = req.create_table_request.id;
-
-        let table = manager.table(catalog_name, schema_name, table_name).await?;
-        let table = if let Some(table) = table {
-            table
-        } else {
-            let table = engine
-                .create_table(&EngineContext::default(), req.create_table_request.clone())
-                .await
-                .with_context(|_| CreateTableSnafu {
-                    table_info: common_catalog::format_full_table_name(
-                        catalog_name,
-                        schema_name,
-                        table_name,
-                    ),
-                })?;
-            let _ = manager
-                .register_table(RegisterTableRequest {
-                    catalog: catalog_name.clone(),
-                    schema: schema_name.clone(),
-                    table_name: table_name.clone(),
-                    table_id,
-                    table: table.clone(),
-                })
-                .await?;
-            info!("Created and registered system table: {table_name}");
-            table
-        };
-        if let Some(hook) = req.open_hook {
-            (hook)(table)?;
-        }
-    }
-    Ok(())
-}
-
-/// The stat of regions in the datanode node.
-/// The number of regions can be got from len of vec.
-///
-/// Ignores any errors occurred during iterating regions. The intention of this method is to
-/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
-/// "try our best" job.
-pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
-    let mut region_number: u64 = 0;
-    let mut region_stats = Vec::new();
-
-    let Ok(catalog_names) = catalog_manager.catalog_names().await else {
-        return (region_number, region_stats);
-    };
-    for catalog_name in catalog_names {
-        let Ok(schema_names) = catalog_manager.schema_names(&catalog_name).await else {
-            continue;
-        };
-        for schema_name in schema_names {
-            let Ok(table_names) = catalog_manager
-                .table_names(&catalog_name, &schema_name)
-                .await
-            else {
-                continue;
-            };
-            for table_name in table_names {
-                let Ok(Some(table)) = catalog_manager
-                    .table(&catalog_name, &schema_name, &table_name)
-                    .await
-                else {
-                    continue;
-                };
-
-                let table_info = table.table_info();
-                let region_numbers = &table_info.meta.region_numbers;
-                region_number += region_numbers.len() as u64;
-
-                let engine = &table_info.meta.engine;
-                let table_id = table_info.ident.table_id;
-
-                match table.region_stats() {
-                    Ok(stats) => {
-                        let stats = stats.into_iter().map(|stat| RegionStat {
-                            region_id: stat.region_id,
-                            table_ident: Some(TableIdent {
-                                table_id,
-                                table_name: Some(TableName {
-                                    catalog_name: catalog_name.clone(),
-                                    schema_name: schema_name.clone(),
-                                    table_name: table_name.clone(),
-                                }),
-                                engine: engine.clone(),
-                            }),
-                            approximate_bytes: stat.disk_usage_bytes as i64,
-                            attrs: HashMap::from([("engine_name".to_owned(), engine.clone())]),
-                            ..Default::default()
-                        });
-
-                        region_stats.extend(stats);
-                    }
-                    Err(e) => {
-                        warn!("Failed to get region status, err: {:?}", e);
-                    }
-                };
-            }
-        }
-    }
-    (region_number, region_stats)
-}

src/catalog/src/local/manager.rs

@@ -1,634 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::Arc;
-
-use common_catalog::consts::{
-    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
-    MITO_ENGINE, NUMBERS_TABLE_ID, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID,
-    SYSTEM_CATALOG_TABLE_NAME,
-};
-use common_catalog::format_full_table_name;
-use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use common_telemetry::{error, info};
-use datatypes::prelude::ScalarVector;
-use datatypes::vectors::{BinaryVector, UInt8Vector};
-use futures_util::lock::Mutex;
-use metrics::increment_gauge;
-use snafu::{ensure, OptionExt, ResultExt};
-use table::engine::manager::TableEngineManagerRef;
-use table::engine::EngineContext;
-use table::metadata::TableId;
-use table::requests::OpenTableRequest;
-use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
-use table::table::TableIdProvider;
-use table::TableRef;
-
-use crate::error::{
-    self, CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu,
-    Result, SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu,
-    SystemCatalogTypeMismatchSnafu, TableEngineNotFoundSnafu, TableExistsSnafu, TableNotExistSnafu,
-    TableNotFoundSnafu, UnimplementedSnafu,
-};
-use crate::local::memory::MemoryCatalogManager;
-use crate::system::{
-    decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
-    VALUE_INDEX,
-};
-use crate::tables::SystemCatalog;
-use crate::{
-    handle_system_table_request, CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest,
-    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
-};
-
-/// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
-pub struct LocalCatalogManager {
-    system: Arc<SystemCatalog>,
-    catalogs: Arc<MemoryCatalogManager>,
-    engine_manager: TableEngineManagerRef,
-    next_table_id: AtomicU32,
-    init_lock: Mutex<bool>,
-    register_lock: Mutex<()>,
-    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
-}
-
-impl LocalCatalogManager {
-    /// Create a new [CatalogManager] with given user catalogs and mito engine
-    pub async fn try_new(engine_manager: TableEngineManagerRef) -> Result<Self> {
-        let engine = engine_manager
-            .engine(MITO_ENGINE)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: MITO_ENGINE,
-            })?;
-        let table = SystemCatalogTable::new(engine.clone()).await?;
-        let memory_catalog_manager = crate::local::memory::new_memory_catalog_manager()?;
-        let system_catalog = Arc::new(SystemCatalog::new(table));
-        Ok(Self {
-            system: system_catalog,
-            catalogs: memory_catalog_manager,
-            engine_manager,
-            next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
-            init_lock: Mutex::new(false),
-            register_lock: Mutex::new(()),
-            system_table_requests: Mutex::new(Vec::default()),
-        })
-    }
-
-    /// Scan all entries from system catalog table
-    pub async fn init(&self) -> Result<()> {
-        self.init_system_catalog().await?;
-        let system_records = self.system.information_schema.system.records().await?;
-        let entries = self.collect_system_catalog_entries(system_records).await?;
-        let max_table_id = self.handle_system_catalog_entries(entries).await?;
-
-        info!(
-            "All system catalog entries processed, max table id: {}",
-            max_table_id
-        );
-        self.next_table_id
-            .store((max_table_id + 1).max(MIN_USER_TABLE_ID), Ordering::Relaxed);
-        *self.init_lock.lock().await = true;
-
-        // Processing system table hooks
-        let mut sys_table_requests = self.system_table_requests.lock().await;
-        let engine = self
-            .engine_manager
-            .engine(MITO_ENGINE)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: MITO_ENGINE,
-            })?;
-
-        handle_system_table_request(self, engine, &mut sys_table_requests).await?;
-        Ok(())
-    }
-
-    async fn init_system_catalog(&self) -> Result<()> {
-        // register default catalog and default schema
-        self.catalogs
-            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())?;
-        self.catalogs.register_schema_sync(RegisterSchemaRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-        })?;
-
-        // register SystemCatalogTable
-        self.catalogs
-            .register_catalog_sync(SYSTEM_CATALOG_NAME.to_string())?;
-        self.catalogs.register_schema_sync(RegisterSchemaRequest {
-            catalog: SYSTEM_CATALOG_NAME.to_string(),
-            schema: INFORMATION_SCHEMA_NAME.to_string(),
-        })?;
-        let register_table_req = RegisterTableRequest {
-            catalog: SYSTEM_CATALOG_NAME.to_string(),
-            schema: INFORMATION_SCHEMA_NAME.to_string(),
-            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
-            table_id: SYSTEM_CATALOG_TABLE_ID,
-            table: self.system.information_schema.system.clone(),
-        };
-        self.catalogs.register_table(register_table_req).await?;
-
-        // Add numbers table for test
-        let numbers_table = Arc::new(NumbersTable::default());
-        let register_number_table_req = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: NUMBERS_TABLE_NAME.to_string(),
-            table_id: NUMBERS_TABLE_ID,
-            table: numbers_table,
-        };
-
-        self.catalogs
-            .register_table(register_number_table_req)
-            .await?;
-
-        Ok(())
-    }
-
-    /// Collect stream of system catalog entries to `Vec<Entry>`
-    async fn collect_system_catalog_entries(
-        &self,
-        stream: SendableRecordBatchStream,
-    ) -> Result<Vec<Entry>> {
-        let record_batch = common_recordbatch::util::collect(stream)
-            .await
-            .context(ReadSystemCatalogSnafu)?;
-        let rbs = record_batch
-            .into_iter()
-            .map(Self::record_batch_to_entry)
-            .collect::<Result<Vec<_>>>()?;
-        Ok(rbs.into_iter().flat_map(Vec::into_iter).collect::<_>())
-    }
-
-    /// Convert `RecordBatch` to a vector of `Entry`.
-    fn record_batch_to_entry(rb: RecordBatch) -> Result<Vec<Entry>> {
-        ensure!(
-            rb.num_columns() >= 6,
-            SystemCatalogSnafu {
-                msg: format!("Length mismatch: {}", rb.num_columns())
-            }
-        );
-
-        let entry_type = rb
-            .column(ENTRY_TYPE_INDEX)
-            .as_any()
-            .downcast_ref::<UInt8Vector>()
-            .with_context(|| SystemCatalogTypeMismatchSnafu {
-                data_type: rb.column(ENTRY_TYPE_INDEX).data_type(),
-            })?;
-
-        let key = rb
-            .column(KEY_INDEX)
-            .as_any()
-            .downcast_ref::<BinaryVector>()
-            .with_context(|| SystemCatalogTypeMismatchSnafu {
-                data_type: rb.column(KEY_INDEX).data_type(),
-            })?;
-
-        let value = rb
-            .column(VALUE_INDEX)
-            .as_any()
-            .downcast_ref::<BinaryVector>()
-            .with_context(|| SystemCatalogTypeMismatchSnafu {
-                data_type: rb.column(VALUE_INDEX).data_type(),
-            })?;
-
-        let mut res = Vec::with_capacity(rb.num_rows());
-        for ((t, k), v) in entry_type
-            .iter_data()
-            .zip(key.iter_data())
-            .zip(value.iter_data())
-        {
-            let entry = decode_system_catalog(t, k, v)?;
-            res.push(entry);
-        }
-        Ok(res)
-    }
-
-    /// Processes records from system catalog table and returns the max table id persisted
-    /// in system catalog table.
-    async fn handle_system_catalog_entries(&self, entries: Vec<Entry>) -> Result<TableId> {
-        let entries = Self::sort_entries(entries);
-        let mut max_table_id = 0;
-        for entry in entries {
-            match entry {
-                Entry::Catalog(c) => {
-                    self.catalogs
-                        .register_catalog_sync(c.catalog_name.clone())?;
-                    info!("Register catalog: {}", c.catalog_name);
-                }
-                Entry::Schema(s) => {
-                    let req = RegisterSchemaRequest {
-                        catalog: s.catalog_name.clone(),
-                        schema: s.schema_name.clone(),
-                    };
-                    let _ = self.catalogs.register_schema_sync(req)?;
-                    info!("Registered schema: {:?}", s);
-                }
-                Entry::Table(t) => {
-                    max_table_id = max_table_id.max(t.table_id);
-                    if t.is_deleted {
-                        continue;
-                    }
-                    self.open_and_register_table(&t).await?;
-                    info!("Registered table: {:?}", t);
-                }
-            }
-        }
-        Ok(max_table_id)
-    }
-
-    /// Sort catalog entries to ensure catalog entries comes first, then schema entries,
-    /// and table entries is the last.
-    fn sort_entries(mut entries: Vec<Entry>) -> Vec<Entry> {
-        entries.sort();
-        entries
-    }
-
-    async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
-        self.check_catalog_schema_exist(&t.catalog_name, &t.schema_name)
-            .await?;
-
-        let context = EngineContext {};
-        let open_request = OpenTableRequest {
-            catalog_name: t.catalog_name.clone(),
-            schema_name: t.schema_name.clone(),
-            table_name: t.table_name.clone(),
-            table_id: t.table_id,
-            region_numbers: vec![0],
-        };
-        let engine = self
-            .engine_manager
-            .engine(&t.engine)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: &t.engine,
-            })?;
-
-        let table_ref = engine
-            .open_table(&context, open_request)
-            .await
-            .with_context(|_| OpenTableSnafu {
-                table_info: format!(
-                    "{}.{}.{}, id: {}",
-                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
-                ),
-            })?
-            .with_context(|| TableNotFoundSnafu {
-                table_info: format!(
-                    "{}.{}.{}, id: {}",
-                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
-                ),
-            })?;
-
-        let register_request = RegisterTableRequest {
-            catalog: t.catalog_name.clone(),
-            schema: t.schema_name.clone(),
-            table_name: t.table_name.clone(),
-            table_id: t.table_id,
-            table: table_ref,
-        };
-        let _ = self.catalogs.register_table(register_request).await?;
-
-        Ok(())
-    }
-
-    async fn check_state(&self) -> Result<()> {
-        let started = self.init_lock.lock().await;
-        ensure!(
-            *started,
-            IllegalManagerStateSnafu {
-                msg: "Catalog manager not started",
-            }
-        );
-        Ok(())
-    }
-
-    async fn check_catalog_schema_exist(
-        &self,
-        catalog_name: &str,
-        schema_name: &str,
-    ) -> Result<()> {
-        if !self.catalogs.catalog_exist(catalog_name).await? {
-            return CatalogNotFoundSnafu { catalog_name }.fail()?;
-        }
-        if !self
-            .catalogs
-            .schema_exist(catalog_name, schema_name)
-            .await?
-        {
-            return SchemaNotFoundSnafu {
-                catalog: catalog_name,
-                schema: schema_name,
-            }
-            .fail()?;
-        }
-        Ok(())
-    }
-}
-
-#[async_trait::async_trait]
-impl TableIdProvider for LocalCatalogManager {
-    async fn next_table_id(&self) -> table::Result<TableId> {
-        Ok(self.next_table_id.fetch_add(1, Ordering::Relaxed))
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogManager for LocalCatalogManager {
-    /// Start [LocalCatalogManager] to load all information from system catalog table.
-    /// Make sure table engine is initialized before starting [MemoryCatalogManager].
-    async fn start(&self) -> Result<()> {
-        self.init().await
-    }
-
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        self.check_state().await?;
-
-        let catalog_name = request.catalog.clone();
-        let schema_name = request.schema.clone();
-
-        self.check_catalog_schema_exist(&catalog_name, &schema_name)
-            .await?;
-
-        {
-            let _lock = self.register_lock.lock().await;
-            if let Some(existing) = self
-                .catalogs
-                .table(&request.catalog, &request.schema, &request.table_name)
-                .await?
-            {
-                if existing.table_info().ident.table_id != request.table_id {
-                    error!(
-                        "Unexpected table register request: {:?}, existing: {:?}",
-                        request,
-                        existing.table_info()
-                    );
-                    return TableExistsSnafu {
-                        table: format_full_table_name(
-                            &catalog_name,
-                            &schema_name,
-                            &request.table_name,
-                        ),
-                    }
-                    .fail();
-                }
-                // Try to register table with same table id, just ignore.
-                Ok(false)
-            } else {
-                // table does not exist
-                let engine = request.table.table_info().meta.engine.to_string();
-                let table_name = request.table_name.clone();
-                let table_id = request.table_id;
-                let _ = self.catalogs.register_table(request).await?;
-                let _ = self
-                    .system
-                    .register_table(
-                        catalog_name.clone(),
-                        schema_name.clone(),
-                        table_name,
-                        table_id,
-                        engine,
-                    )
-                    .await?;
-                increment_gauge!(
-                    crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-                    1.0,
-                    &[crate::metrics::db_label(&catalog_name, &schema_name)],
-                );
-                Ok(true)
-            }
-        }
-    }
-
-    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        self.check_state().await?;
-
-        let catalog_name = &request.catalog;
-        let schema_name = &request.schema;
-
-        self.check_catalog_schema_exist(catalog_name, schema_name)
-            .await?;
-        ensure!(
-            self.catalogs
-                .table(catalog_name, schema_name, &request.new_table_name)
-                .await?
-                .is_none(),
-            TableExistsSnafu {
-                table: &request.new_table_name
-            }
-        );
-
-        let _lock = self.register_lock.lock().await;
-        let old_table = self
-            .catalogs
-            .table(catalog_name, schema_name, &request.table_name)
-            .await?
-            .context(TableNotExistSnafu {
-                table: &request.table_name,
-            })?;
-
-        let engine = old_table.table_info().meta.engine.to_string();
-        // rename table in system catalog
-        let _ = self
-            .system
-            .register_table(
-                catalog_name.clone(),
-                schema_name.clone(),
-                request.new_table_name.clone(),
-                request.table_id,
-                engine,
-            )
-            .await?;
-
-        self.catalogs.rename_table(request).await
-    }
-
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
-        self.check_state().await?;
-
-        {
-            let _ = self.register_lock.lock().await;
-
-            let DeregisterTableRequest {
-                catalog,
-                schema,
-                table_name,
-            } = &request;
-            let table_id = self
-                .catalogs
-                .table(catalog, schema, table_name)
-                .await?
-                .with_context(|| error::TableNotExistSnafu {
-                    table: format_full_table_name(catalog, schema, table_name),
-                })?
-                .table_info()
-                .ident
-                .table_id;
-
-            self.system.deregister_table(&request, table_id).await?;
-            self.catalogs.deregister_table(request).await
-        }
-    }
-
-    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        self.check_state().await?;
-
-        let catalog_name = &request.catalog;
-        let schema_name = &request.schema;
-
-        if !self.catalogs.catalog_exist(catalog_name).await? {
-            return CatalogNotFoundSnafu { catalog_name }.fail()?;
-        }
-
-        {
-            let _lock = self.register_lock.lock().await;
-            ensure!(
-                !self
-                    .catalogs
-                    .schema_exist(catalog_name, schema_name)
-                    .await?,
-                SchemaExistsSnafu {
-                    schema: schema_name,
-                }
-            );
-            let _ = self
-                .system
-                .register_schema(request.catalog.clone(), schema_name.clone())
-                .await?;
-            self.catalogs.register_schema_sync(request)
-        }
-    }
-
-    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
-        UnimplementedSnafu {
-            operation: "deregister schema",
-        }
-        .fail()
-    }
-
-    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
-        let catalog_name = request.create_table_request.catalog_name.clone();
-        let schema_name = request.create_table_request.schema_name.clone();
-
-        let mut sys_table_requests = self.system_table_requests.lock().await;
-        sys_table_requests.push(request);
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&catalog_name, &schema_name)],
-        );
-        Ok(())
-    }
-
-    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
-        self.catalogs.schema_exist(catalog, schema).await
-    }
-
-    async fn table(
-        &self,
-        catalog_name: &str,
-        schema_name: &str,
-        table_name: &str,
-    ) -> Result<Option<TableRef>> {
-        self.catalogs
-            .table(catalog_name, schema_name, table_name)
-            .await
-    }
-
-    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
-        if catalog.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
-            Ok(true)
-        } else {
-            self.catalogs.catalog_exist(catalog).await
-        }
-    }
-
-    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
-        self.catalogs.table_exist(catalog, schema, table).await
-    }
-
-    async fn catalog_names(&self) -> Result<Vec<String>> {
-        self.catalogs.catalog_names().await
-    }
-
-    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
-        self.catalogs.schema_names(catalog_name).await
-    }
-
-    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
-        self.catalogs.table_names(catalog_name, schema_name).await
-    }
-
-    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
-        self.catalogs.clone().register_catalog(name).await
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::assert_matches::assert_matches;
-
-    use mito::engine::MITO_ENGINE;
-
-    use super::*;
-    use crate::system::{CatalogEntry, SchemaEntry};
-
-    #[test]
-    fn test_sort_entry() {
-        let vec = vec![
-            Entry::Table(TableEntry {
-                catalog_name: "C1".to_string(),
-                schema_name: "S1".to_string(),
-                table_name: "T1".to_string(),
-                table_id: 1,
-                engine: MITO_ENGINE.to_string(),
-                is_deleted: false,
-            }),
-            Entry::Catalog(CatalogEntry {
-                catalog_name: "C2".to_string(),
-            }),
-            Entry::Schema(SchemaEntry {
-                catalog_name: "C1".to_string(),
-                schema_name: "S1".to_string(),
-            }),
-            Entry::Schema(SchemaEntry {
-                catalog_name: "C2".to_string(),
-                schema_name: "S2".to_string(),
-            }),
-            Entry::Catalog(CatalogEntry {
-                catalog_name: "".to_string(),
-            }),
-            Entry::Table(TableEntry {
-                catalog_name: "C1".to_string(),
-                schema_name: "S1".to_string(),
-                table_name: "T2".to_string(),
-                table_id: 2,
-                engine: MITO_ENGINE.to_string(),
-                is_deleted: false,
-            }),
-        ];
-        let res = LocalCatalogManager::sort_entries(vec);
-        assert_matches!(res[0], Entry::Catalog(..));
-        assert_matches!(res[1], Entry::Catalog(..));
-        assert_matches!(res[2], Entry::Schema(..));
-        assert_matches!(res[3], Entry::Schema(..));
-        assert_matches!(res[4], Entry::Table(..));
-        assert_matches!(res[5], Entry::Table(..));
-    }
-}

src/catalog/src/local/memory.rs

@@ -1,606 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::collections::hash_map::Entry;
-use std::collections::HashMap;
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::{Arc, RwLock, Weak};
-
-use common_catalog::consts::{
-    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
-};
-use metrics::{decrement_gauge, increment_gauge};
-use snafu::OptionExt;
-use table::metadata::TableId;
-use table::table::TableIdProvider;
-use table::TableRef;
-
-use crate::error::{
-    CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, TableNotFoundSnafu,
-};
-use crate::information_schema::InformationSchemaProvider;
-use crate::{
-    CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest, RegisterSchemaRequest,
-    RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
-};
-
-type SchemaEntries = HashMap<String, HashMap<String, TableRef>>;
-
-/// Simple in-memory list of catalogs
-pub struct MemoryCatalogManager {
-    /// Collection of catalogs containing schemas and ultimately Tables
-    pub catalogs: RwLock<HashMap<String, SchemaEntries>>,
-    pub table_id: AtomicU32,
-}
-
-#[async_trait::async_trait]
-impl TableIdProvider for MemoryCatalogManager {
-    async fn next_table_id(&self) -> table::error::Result<TableId> {
-        Ok(self.table_id.fetch_add(1, Ordering::Relaxed))
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogManager for MemoryCatalogManager {
-    async fn start(&self) -> Result<()> {
-        self.table_id.store(MIN_USER_TABLE_ID, Ordering::Relaxed);
-        Ok(())
-    }
-
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        self.register_table_sync(request)
-    }
-
-    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let schema = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?
-            .get_mut(&request.schema)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?;
-
-        // check old and new table names
-        if !schema.contains_key(&request.table_name) {
-            return TableNotFoundSnafu {
-                table_info: request.table_name.to_string(),
-            }
-            .fail()?;
-        }
-        if schema.contains_key(&request.new_table_name) {
-            return TableExistsSnafu {
-                table: &request.new_table_name,
-            }
-            .fail();
-        }
-
-        let table = schema.remove(&request.table_name).unwrap();
-        let _ = schema.insert(request.new_table_name, table);
-
-        Ok(true)
-    }
-
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let schema = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?
-            .get_mut(&request.schema)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?;
-        let result = schema.remove(&request.table_name);
-        if result.is_some() {
-            decrement_gauge!(
-                crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-                1.0,
-                &[crate::metrics::db_label(&request.catalog, &request.schema)],
-            );
-        }
-        Ok(())
-    }
-
-    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        self.register_schema_sync(request)
-    }
-
-    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let schemas = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?;
-        let table_count = schemas
-            .remove(&request.schema)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?
-            .len();
-        decrement_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            table_count as f64,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
-
-        decrement_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
-        Ok(true)
-    }
-
-    async fn register_system_table(&self, _request: RegisterSystemTableRequest) -> Result<()> {
-        // TODO(ruihang): support register system table request
-        Ok(())
-    }
-
-    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
-        Ok(self
-            .catalogs
-            .read()
-            .unwrap()
-            .get(catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: catalog,
-            })?
-            .contains_key(schema))
-    }
-
-    async fn table(
-        &self,
-        catalog: &str,
-        schema: &str,
-        table_name: &str,
-    ) -> Result<Option<TableRef>> {
-        let result = try {
-            self.catalogs
-                .read()
-                .unwrap()
-                .get(catalog)?
-                .get(schema)?
-                .get(table_name)
-                .cloned()?
-        };
-        Ok(result)
-    }
-
-    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
-        Ok(self.catalogs.read().unwrap().get(catalog).is_some())
-    }
-
-    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
-        let catalogs = self.catalogs.read().unwrap();
-        Ok(catalogs
-            .get(catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: catalog,
-            })?
-            .get(schema)
-            .with_context(|| SchemaNotFoundSnafu { catalog, schema })?
-            .contains_key(table))
-    }
-
-    async fn catalog_names(&self) -> Result<Vec<String>> {
-        Ok(self.catalogs.read().unwrap().keys().cloned().collect())
-    }
-
-    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
-        Ok(self
-            .catalogs
-            .read()
-            .unwrap()
-            .get(catalog_name)
-            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
-            .keys()
-            .cloned()
-            .collect())
-    }
-
-    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
-        Ok(self
-            .catalogs
-            .read()
-            .unwrap()
-            .get(catalog_name)
-            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
-            .get(schema_name)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: catalog_name,
-                schema: schema_name,
-            })?
-            .keys()
-            .cloned()
-            .collect())
-    }
-
-    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
-        self.register_catalog_sync(name)
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-impl MemoryCatalogManager {
-    /// Create a manager with some default setups
-    /// (e.g. default catalog/schema and information schema)
-    pub fn with_default_setup() -> Arc<Self> {
-        let manager = Arc::new(Self {
-            table_id: AtomicU32::new(MIN_USER_TABLE_ID),
-            catalogs: Default::default(),
-        });
-
-        // Safety: default catalog/schema is registered in order so no CatalogNotFound error will occur
-        manager
-            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())
-            .unwrap();
-        manager
-            .register_schema_sync(RegisterSchemaRequest {
-                catalog: DEFAULT_CATALOG_NAME.to_string(),
-                schema: DEFAULT_SCHEMA_NAME.to_string(),
-            })
-            .unwrap();
-
-        manager
-    }
-
-    /// Registers a catalog and return the catalog already exist
-    pub fn register_catalog_if_absent(&self, name: String) -> bool {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let entry = catalogs.entry(name);
-        match entry {
-            Entry::Occupied(_) => true,
-            Entry::Vacant(v) => {
-                let _ = v.insert(HashMap::new());
-                false
-            }
-        }
-    }
-
-    pub fn register_catalog_sync(self: &Arc<Self>, name: String) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-
-        match catalogs.entry(name.clone()) {
-            Entry::Vacant(e) => {
-                let catalog = self.create_catalog_entry(name);
-                e.insert(catalog);
-                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
-                Ok(true)
-            }
-            Entry::Occupied(_) => Ok(false),
-        }
-    }
-
-    pub fn register_schema_sync(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let catalog = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?;
-
-        match catalog.entry(request.schema) {
-            Entry::Vacant(e) => {
-                e.insert(HashMap::new());
-                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
-                Ok(true)
-            }
-            Entry::Occupied(_) => Ok(false),
-        }
-    }
-
-    pub fn register_table_sync(&self, request: RegisterTableRequest) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let schema = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?
-            .get_mut(&request.schema)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?;
-
-        if schema.contains_key(&request.table_name) {
-            return TableExistsSnafu {
-                table: &request.table_name,
-            }
-            .fail();
-        }
-        schema.insert(request.table_name, request.table);
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
-        Ok(true)
-    }
-
-    fn create_catalog_entry(self: &Arc<Self>, catalog: String) -> SchemaEntries {
-        let information_schema = InformationSchemaProvider::build(
-            catalog,
-            Arc::downgrade(self) as Weak<dyn CatalogManager>,
-        );
-        let mut catalog = HashMap::new();
-        catalog.insert(INFORMATION_SCHEMA_NAME.to_string(), information_schema);
-        catalog
-    }
-
-    #[cfg(any(test, feature = "testing"))]
-    pub fn new_with_table(table: TableRef) -> Arc<Self> {
-        let manager = Self::with_default_setup();
-        let request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table.table_info().name.clone(),
-            table_id: table.table_info().ident.table_id,
-            table,
-        };
-        let _ = manager.register_table_sync(request).unwrap();
-        manager
-    }
-}
-
-/// Create a memory catalog list contains a numbers table for test
-pub fn new_memory_catalog_manager() -> Result<Arc<MemoryCatalogManager>> {
-    Ok(MemoryCatalogManager::with_default_setup())
-}
-
-#[cfg(test)]
-mod tests {
-    use common_catalog::consts::*;
-    use common_error::ext::ErrorExt;
-    use common_error::status_code::StatusCode;
-    use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
-
-    use super::*;
-
-    #[tokio::test]
-    async fn test_new_memory_catalog_list() {
-        let catalog_list = new_memory_catalog_manager().unwrap();
-
-        let register_request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: NUMBERS_TABLE_NAME.to_string(),
-            table_id: NUMBERS_TABLE_ID,
-            table: Arc::new(NumbersTable::default()),
-        };
-
-        let _ = catalog_list.register_table(register_request).await.unwrap();
-        let table = catalog_list
-            .table(
-                DEFAULT_CATALOG_NAME,
-                DEFAULT_SCHEMA_NAME,
-                NUMBERS_TABLE_NAME,
-            )
-            .await
-            .unwrap();
-        let _ = table.unwrap();
-        assert!(catalog_list
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "not_exists")
-            .await
-            .unwrap()
-            .is_none());
-    }
-
-    #[tokio::test]
-    async fn test_mem_manager_rename_table() {
-        let catalog = MemoryCatalogManager::with_default_setup();
-        let table_name = "test_table";
-        assert!(!catalog
-            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap());
-        // register test table
-        let table_id = 2333;
-        let register_request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            table_id,
-            table: Arc::new(NumbersTable::new(table_id)),
-        };
-        assert!(catalog.register_table(register_request).await.unwrap());
-        assert!(catalog
-            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap());
-
-        // rename test table
-        let new_table_name = "test_table_renamed";
-        let rename_request = RenameTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            new_table_name: new_table_name.to_string(),
-            table_id,
-        };
-        let _ = catalog.rename_table(rename_request).await.unwrap();
-
-        // test old table name not exist
-        assert!(!catalog
-            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap());
-
-        // test new table name exists
-        assert!(catalog
-            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap());
-        let registered_table = catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(registered_table.table_info().ident.table_id, table_id);
-
-        let dup_register_request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: new_table_name.to_string(),
-            table_id: table_id + 1,
-            table: Arc::new(NumbersTable::new(table_id + 1)),
-        };
-        let result = catalog.register_table(dup_register_request).await;
-        let err = result.err().unwrap();
-        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
-    }
-
-    #[tokio::test]
-    async fn test_catalog_rename_table() {
-        let catalog = MemoryCatalogManager::with_default_setup();
-        let table_name = "num";
-        let table_id = 2333;
-        let table: TableRef = Arc::new(NumbersTable::new(table_id));
-
-        // register table
-        let register_table_req = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            table_id,
-            table,
-        };
-        assert!(catalog.register_table(register_table_req).await.unwrap());
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap()
-            .is_some());
-
-        // rename table
-        let new_table_name = "numbers_new";
-        let rename_table_req = RenameTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            new_table_name: new_table_name.to_string(),
-            table_id,
-        };
-        assert!(catalog.rename_table(rename_table_req).await.unwrap());
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap()
-            .is_none());
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap()
-            .is_some());
-
-        let registered_table = catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(registered_table.table_info().ident.table_id, table_id);
-    }
-
-    #[test]
-    pub fn test_register_if_absent() {
-        let list = MemoryCatalogManager::with_default_setup();
-        assert!(!list.register_catalog_if_absent("test_catalog".to_string(),));
-        assert!(list.register_catalog_if_absent("test_catalog".to_string()));
-    }
-
-    #[tokio::test]
-    pub async fn test_catalog_deregister_table() {
-        let catalog = MemoryCatalogManager::with_default_setup();
-        let table_name = "foo_table";
-
-        let register_table_req = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            table_id: 2333,
-            table: Arc::new(NumbersTable::default()),
-        };
-        let _ = catalog.register_table(register_table_req).await.unwrap();
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap()
-            .is_some());
-
-        let deregister_table_req = DeregisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-        };
-        catalog
-            .deregister_table(deregister_table_req)
-            .await
-            .unwrap();
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap()
-            .is_none());
-    }
-
-    #[tokio::test]
-    async fn test_catalog_deregister_schema() {
-        let catalog = MemoryCatalogManager::with_default_setup();
-
-        // Registers a catalog, a schema, and a table.
-        let catalog_name = "foo_catalog".to_string();
-        let schema_name = "foo_schema".to_string();
-        let table_name = "foo_table".to_string();
-        let schema = RegisterSchemaRequest {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-        };
-        let table = RegisterTableRequest {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-            table_name,
-            table_id: 0,
-            table: Arc::new(NumbersTable::default()),
-        };
-        catalog
-            .clone()
-            .register_catalog(catalog_name.clone())
-            .await
-            .unwrap();
-        catalog.register_schema(schema).await.unwrap();
-        catalog.register_table(table).await.unwrap();
-
-        let request = DeregisterSchemaRequest {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-        };
-
-        assert!(catalog.deregister_schema(request).await.unwrap());
-        assert!(!catalog
-            .schema_exist(&catalog_name, &schema_name)
-            .await
-            .unwrap());
-    }
-}

src/catalog/src/memory.rs

@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod config;
-pub mod engine;
-pub mod error;
-mod manifest;
-mod metrics;
-pub mod table;
+pub mod manager;
+
+pub use manager::{new_memory_catalog_manager, MemoryCatalogManager};

src/catalog/src/memory/manager.rs

@@ -0,0 +1,369 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock, Weak};
+
+use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
+use metrics::{decrement_gauge, increment_gauge};
+use snafu::OptionExt;
+use table::TableRef;
+
+use crate::error::{CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu};
+use crate::information_schema::InformationSchemaProvider;
+use crate::{CatalogManager, DeregisterTableRequest, RegisterSchemaRequest, RegisterTableRequest};
+
+type SchemaEntries = HashMap<String, HashMap<String, TableRef>>;
+
+/// Simple in-memory list of catalogs
+#[derive(Clone)]
+pub struct MemoryCatalogManager {
+    /// Collection of catalogs containing schemas and ultimately Tables
+    catalogs: Arc<RwLock<HashMap<String, SchemaEntries>>>,
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for MemoryCatalogManager {
+    async fn schema_exists(&self, catalog: &str, schema: &str) -> Result<bool> {
+        self.schema_exist_sync(catalog, schema)
+    }
+
+    async fn table(
+        &self,
+        catalog: &str,
+        schema: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        let result = try {
+            self.catalogs
+                .read()
+                .unwrap()
+                .get(catalog)?
+                .get(schema)?
+                .get(table_name)
+                .cloned()?
+        };
+        Ok(result)
+    }
+
+    async fn catalog_exists(&self, catalog: &str) -> Result<bool> {
+        self.catalog_exist_sync(catalog)
+    }
+
+    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+        let catalogs = self.catalogs.read().unwrap();
+        Ok(catalogs
+            .get(catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .get(schema)
+            .with_context(|| SchemaNotFoundSnafu { catalog, schema })?
+            .contains_key(table))
+    }
+
+    async fn catalog_names(&self) -> Result<Vec<String>> {
+        Ok(self.catalogs.read().unwrap().keys().cloned().collect())
+    }
+
+    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog_name)
+            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
+            .keys()
+            .cloned()
+            .collect())
+    }
+
+    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog_name)
+            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
+            .get(schema_name)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            })?
+            .keys()
+            .cloned()
+            .collect())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl MemoryCatalogManager {
+    pub fn new() -> Arc<Self> {
+        Arc::new(Self {
+            catalogs: Default::default(),
+        })
+    }
+
+    /// Creates a manager with some default setups
+    /// (e.g. default catalog/schema and information schema)
+    pub fn with_default_setup() -> Arc<Self> {
+        let manager = Arc::new(Self {
+            catalogs: Default::default(),
+        });
+
+        // Safety: default catalog/schema is registered in order so no CatalogNotFound error will occur
+        manager.register_catalog_sync(DEFAULT_CATALOG_NAME).unwrap();
+        manager
+            .register_schema_sync(RegisterSchemaRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+            })
+            .unwrap();
+
+        manager
+    }
+
+    fn schema_exist_sync(&self, catalog: &str, schema: &str) -> Result<bool> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .contains_key(schema))
+    }
+
+    fn catalog_exist_sync(&self, catalog: &str) -> Result<bool> {
+        Ok(self.catalogs.read().unwrap().get(catalog).is_some())
+    }
+
+    /// Registers a catalog if it does not exist and returns false if the schema exists.
+    pub fn register_catalog_sync(&self, name: &str) -> Result<bool> {
+        let name = name.to_string();
+
+        let mut catalogs = self.catalogs.write().unwrap();
+
+        match catalogs.entry(name.clone()) {
+            Entry::Vacant(e) => {
+                let arc_self = Arc::new(self.clone());
+                let catalog = arc_self.create_catalog_entry(name);
+                e.insert(catalog);
+                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
+                Ok(true)
+            }
+            Entry::Occupied(_) => Ok(false),
+        }
+    }
+
+    pub fn deregister_table_sync(&self, request: DeregisterTableRequest) -> Result<()> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .get_mut(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+        let result = schema.remove(&request.table_name);
+        if result.is_some() {
+            decrement_gauge!(
+                crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+                1.0,
+                &[crate::metrics::db_label(&request.catalog, &request.schema)],
+            );
+        }
+        Ok(())
+    }
+
+    /// Registers a schema if it does not exist.
+    /// It returns an error if the catalog does not exist,
+    /// and returns false if the schema exists.
+    pub fn register_schema_sync(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?;
+
+        match catalog.entry(request.schema) {
+            Entry::Vacant(e) => {
+                e.insert(HashMap::new());
+                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
+                Ok(true)
+            }
+            Entry::Occupied(_) => Ok(false),
+        }
+    }
+
+    /// Registers a schema and returns an error if the catalog or schema does not exist.
+    pub fn register_table_sync(&self, request: RegisterTableRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .get_mut(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+
+        if schema.contains_key(&request.table_name) {
+            return TableExistsSnafu {
+                table: &request.table_name,
+            }
+            .fail();
+        }
+        schema.insert(request.table_name, request.table);
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
+        Ok(true)
+    }
+
+    fn create_catalog_entry(self: &Arc<Self>, catalog: String) -> SchemaEntries {
+        let information_schema = InformationSchemaProvider::build(
+            catalog,
+            Arc::downgrade(self) as Weak<dyn CatalogManager>,
+        );
+        let mut catalog = HashMap::new();
+        catalog.insert(INFORMATION_SCHEMA_NAME.to_string(), information_schema);
+        catalog
+    }
+
+    #[cfg(any(test, feature = "testing"))]
+    pub fn new_with_table(table: TableRef) -> Arc<Self> {
+        let manager = Self::with_default_setup();
+        let catalog = &table.table_info().catalog_name;
+        let schema = &table.table_info().schema_name;
+
+        if !manager.catalog_exist_sync(catalog).unwrap() {
+            manager.register_catalog_sync(catalog).unwrap();
+        }
+
+        if !manager.schema_exist_sync(catalog, schema).unwrap() {
+            manager
+                .register_schema_sync(RegisterSchemaRequest {
+                    catalog: catalog.to_string(),
+                    schema: schema.to_string(),
+                })
+                .unwrap();
+        }
+
+        let request = RegisterTableRequest {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table_name: table.table_info().name.clone(),
+            table_id: table.table_info().ident.table_id,
+            table,
+        };
+        let _ = manager.register_table_sync(request).unwrap();
+        manager
+    }
+}
+
+/// Create a memory catalog list contains a numbers table for test
+pub fn new_memory_catalog_manager() -> Result<Arc<MemoryCatalogManager>> {
+    Ok(MemoryCatalogManager::with_default_setup())
+}
+
+#[cfg(test)]
+mod tests {
+    use common_catalog::consts::*;
+    use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_new_memory_catalog_list() {
+        let catalog_list = new_memory_catalog_manager().unwrap();
+
+        let register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: NUMBERS_TABLE_NAME.to_string(),
+            table_id: NUMBERS_TABLE_ID,
+            table: NumbersTable::table(NUMBERS_TABLE_ID),
+        };
+
+        catalog_list.register_table_sync(register_request).unwrap();
+        let table = catalog_list
+            .table(
+                DEFAULT_CATALOG_NAME,
+                DEFAULT_SCHEMA_NAME,
+                NUMBERS_TABLE_NAME,
+            )
+            .await
+            .unwrap();
+        let _ = table.unwrap();
+        assert!(catalog_list
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "not_exists")
+            .await
+            .unwrap()
+            .is_none());
+    }
+
+    #[test]
+    pub fn test_register_catalog_sync() {
+        let list = MemoryCatalogManager::with_default_setup();
+        assert!(list.register_catalog_sync("test_catalog").unwrap());
+        assert!(!list.register_catalog_sync("test_catalog").unwrap());
+    }
+
+    #[tokio::test]
+    pub async fn test_catalog_deregister_table() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+        let table_name = "foo_table";
+
+        let register_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id: 2333,
+            table: NumbersTable::table(2333),
+        };
+        catalog.register_table_sync(register_table_req).unwrap();
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_some());
+
+        let deregister_table_req = DeregisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+        };
+        catalog.deregister_table_sync(deregister_table_req).unwrap();
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_none());
+    }
+}

src/catalog/src/remote/manager.rs

@@ -1,434 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use common_catalog::consts::MITO_ENGINE;
-use common_meta::helper::{CatalogKey, SchemaKey};
-use common_meta::ident::TableIdent;
-use common_meta::key::datanode_table::DatanodeTableValue;
-use common_meta::key::TableMetadataManagerRef;
-use common_meta::kv_backend::KvBackendRef;
-use common_telemetry::{error, info, warn};
-use metrics::increment_gauge;
-use snafu::{ensure, OptionExt, ResultExt};
-use table::engine::manager::TableEngineManagerRef;
-use table::engine::EngineContext;
-use table::requests::OpenTableRequest;
-use table::TableRef;
-use tokio::sync::Mutex;
-
-use crate::error::{
-    OpenTableSnafu, ParallelOpenTableSnafu, Result, TableEngineNotFoundSnafu, TableExistsSnafu,
-    TableMetadataManagerSnafu, TableNotFoundSnafu, UnimplementedSnafu,
-};
-use crate::local::MemoryCatalogManager;
-use crate::remote::region_alive_keeper::RegionAliveKeepers;
-use crate::{
-    handle_system_table_request, CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest,
-    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
-};
-
-/// Catalog manager based on metasrv.
-pub struct RemoteCatalogManager {
-    node_id: u64,
-    backend: KvBackendRef,
-    engine_manager: TableEngineManagerRef,
-    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
-    region_alive_keepers: Arc<RegionAliveKeepers>,
-    memory_catalog_manager: Arc<MemoryCatalogManager>,
-    table_metadata_manager: TableMetadataManagerRef,
-}
-
-impl RemoteCatalogManager {
-    pub fn new(
-        engine_manager: TableEngineManagerRef,
-        node_id: u64,
-        backend: KvBackendRef,
-        region_alive_keepers: Arc<RegionAliveKeepers>,
-        table_metadata_manager: TableMetadataManagerRef,
-    ) -> Self {
-        Self {
-            engine_manager,
-            node_id,
-            backend,
-            system_table_requests: Default::default(),
-            region_alive_keepers,
-            memory_catalog_manager: MemoryCatalogManager::with_default_setup(),
-            table_metadata_manager,
-        }
-    }
-
-    async fn initiate_catalogs(&self) -> Result<()> {
-        let tables = self
-            .table_metadata_manager
-            .datanode_table_manager()
-            .tables(self.node_id)
-            .await
-            .context(TableMetadataManagerSnafu)?;
-
-        let joins = tables
-            .into_iter()
-            .map(|datanode_table_value| {
-                let engine_manager = self.engine_manager.clone();
-                let memory_catalog_manager = self.memory_catalog_manager.clone();
-                let table_metadata_manager = self.table_metadata_manager.clone();
-                common_runtime::spawn_bg(async move {
-                    let table_id = datanode_table_value.table_id;
-                    if let Err(e) = open_and_register_table(
-                        engine_manager,
-                        datanode_table_value,
-                        memory_catalog_manager,
-                        table_metadata_manager,
-                    )
-                    .await
-                    {
-                        // Note that we don't return error here if table opened failed. This is because
-                        // we don't want those broken tables to impede the startup of Datanode.
-                        // However, this could be changed in the future.
-                        error!(e; "Failed to open or register table, id = {table_id}")
-                    }
-                })
-            })
-            .collect::<Vec<_>>();
-
-        let _ = futures::future::try_join_all(joins)
-            .await
-            .context(ParallelOpenTableSnafu)?;
-        Ok(())
-    }
-
-    fn build_schema_key(&self, catalog_name: String, schema_name: String) -> SchemaKey {
-        SchemaKey {
-            catalog_name,
-            schema_name,
-        }
-    }
-}
-
-async fn open_and_register_table(
-    engine_manager: TableEngineManagerRef,
-    datanode_table_value: DatanodeTableValue,
-    memory_catalog_manager: Arc<MemoryCatalogManager>,
-    table_metadata_manager: TableMetadataManagerRef,
-) -> Result<()> {
-    let context = EngineContext {};
-
-    let table_id = datanode_table_value.table_id;
-    let region_numbers = datanode_table_value.regions;
-
-    let table_info_value = table_metadata_manager
-        .table_info_manager()
-        .get(table_id)
-        .await
-        .context(TableMetadataManagerSnafu)?
-        .context(TableNotFoundSnafu {
-            table_info: format!("table id: {table_id}"),
-        })?;
-    let table_info = &table_info_value.table_info;
-    let catalog_name = table_info.catalog_name.clone();
-    let schema_name = table_info.schema_name.clone();
-    let table_name = table_info.name.clone();
-
-    let request = OpenTableRequest {
-        catalog_name: catalog_name.clone(),
-        schema_name: schema_name.clone(),
-        table_name: table_name.clone(),
-        table_id,
-        region_numbers: region_numbers.clone(),
-    };
-    let engine =
-        engine_manager
-            .engine(&table_info.meta.engine)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: &table_info.meta.engine,
-            })?;
-
-    let table_ident = TableIdent {
-        catalog: catalog_name,
-        schema: schema_name,
-        table: table_name,
-        table_id,
-        engine: table_info.meta.engine.clone(),
-    };
-
-    let table = engine
-        .open_table(&context, request)
-        .await
-        .with_context(|_| OpenTableSnafu {
-            table_info: table_ident.to_string(),
-        })?
-        .with_context(|| TableNotFoundSnafu {
-            table_info: table_ident.to_string(),
-        })?;
-    info!("Successfully opened table, {table_ident}");
-
-    if !memory_catalog_manager
-        .catalog_exist(&table_ident.catalog)
-        .await?
-    {
-        memory_catalog_manager.register_catalog_sync(table_ident.catalog.clone())?;
-    }
-
-    if !memory_catalog_manager
-        .schema_exist(&table_ident.catalog, &table_ident.schema)
-        .await?
-    {
-        memory_catalog_manager.register_schema_sync(RegisterSchemaRequest {
-            catalog: table_ident.catalog.clone(),
-            schema: table_ident.schema.clone(),
-        })?;
-    }
-
-    let request = RegisterTableRequest {
-        catalog: table_ident.catalog.clone(),
-        schema: table_ident.schema.clone(),
-        table_name: table_ident.table.clone(),
-        table_id,
-        table,
-    };
-    let registered = memory_catalog_manager.register_table_sync(request)?;
-    ensure!(
-        registered,
-        TableExistsSnafu {
-            table: table_ident.to_string(),
-        }
-    );
-    info!("Successfully registered table, {table_ident}");
-    Ok(())
-}
-
-#[async_trait]
-impl CatalogManager for RemoteCatalogManager {
-    async fn start(&self) -> Result<()> {
-        self.initiate_catalogs().await?;
-
-        let mut system_table_requests = self.system_table_requests.lock().await;
-        let engine = self
-            .engine_manager
-            .engine(MITO_ENGINE)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: MITO_ENGINE,
-            })?;
-        handle_system_table_request(self, engine, &mut system_table_requests).await?;
-        info!("All system table opened");
-        Ok(())
-    }
-
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        let table = request.table.clone();
-
-        let registered = self.memory_catalog_manager.register_table_sync(request)?;
-
-        if registered {
-            let table_info = table.table_info();
-            let table_ident = TableIdent {
-                catalog: table_info.catalog_name.clone(),
-                schema: table_info.schema_name.clone(),
-                table: table_info.name.clone(),
-                table_id: table_info.table_id(),
-                engine: table_info.meta.engine.clone(),
-            };
-            self.region_alive_keepers
-                .register_table(table_ident, table)
-                .await?;
-        }
-
-        Ok(registered)
-    }
-
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
-        let Some(table) = self
-            .memory_catalog_manager
-            .table(&request.catalog, &request.schema, &request.table_name)
-            .await?
-        else {
-            return Ok(());
-        };
-
-        let table_info = table.table_info();
-        let table_ident = TableIdent {
-            catalog: request.catalog.clone(),
-            schema: request.schema.clone(),
-            table: request.table_name.clone(),
-            table_id: table_info.ident.table_id,
-            engine: table_info.meta.engine.clone(),
-        };
-        if let Some(keeper) = self
-            .region_alive_keepers
-            .deregister_table(&table_ident)
-            .await
-        {
-            warn!(
-                "Table {} is deregistered from region alive keepers",
-                keeper.table_ident(),
-            );
-        }
-
-        self.memory_catalog_manager.deregister_table(request).await
-    }
-
-    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        self.memory_catalog_manager.register_schema_sync(request)
-    }
-
-    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
-        UnimplementedSnafu {
-            operation: "deregister schema",
-        }
-        .fail()
-    }
-
-    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        self.memory_catalog_manager.rename_table(request).await
-    }
-
-    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
-        let catalog_name = request.create_table_request.catalog_name.clone();
-        let schema_name = request.create_table_request.schema_name.clone();
-
-        let mut requests = self.system_table_requests.lock().await;
-        requests.push(request);
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&catalog_name, &schema_name)],
-        );
-        Ok(())
-    }
-
-    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
-        if !self.catalog_exist(catalog).await? {
-            return Ok(false);
-        }
-
-        if self
-            .memory_catalog_manager
-            .schema_exist(catalog, schema)
-            .await?
-        {
-            return Ok(true);
-        }
-
-        let key = self
-            .build_schema_key(catalog.to_string(), schema.to_string())
-            .to_string();
-        let remote_schema_exists = self
-            .backend
-            .get(key.as_bytes())
-            .await
-            .context(TableMetadataManagerSnafu)?
-            .is_some();
-
-        // Create schema locally if remote schema exists. Since local schema is managed by memory
-        // catalog manager, creating a local schema is relatively cheap (just a HashMap).
-        // Besides, if this method ("schema_exist) is called, it's very likely that someone wants to
-        // create a table in this schema. We should create the schema now.
-        if remote_schema_exists
-            && self
-                .memory_catalog_manager
-                .register_schema(RegisterSchemaRequest {
-                    catalog: catalog.to_string(),
-                    schema: schema.to_string(),
-                })
-                .await?
-        {
-            info!("register schema '{catalog}/{schema}' on demand");
-        }
-
-        Ok(remote_schema_exists)
-    }
-
-    async fn table(
-        &self,
-        catalog_name: &str,
-        schema_name: &str,
-        table_name: &str,
-    ) -> Result<Option<TableRef>> {
-        self.memory_catalog_manager
-            .table(catalog_name, schema_name, table_name)
-            .await
-    }
-
-    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
-        if self.memory_catalog_manager.catalog_exist(catalog).await? {
-            return Ok(true);
-        }
-
-        let key = CatalogKey {
-            catalog_name: catalog.to_string(),
-        };
-
-        let remote_catalog_exists = self
-            .backend
-            .get(key.to_string().as_bytes())
-            .await
-            .context(TableMetadataManagerSnafu)?
-            .is_some();
-
-        // Create catalog locally if remote catalog exists. Since local catalog is managed by memory
-        // catalog manager, creating a local catalog is relatively cheap (just a HashMap).
-        // Besides, if this method ("catalog_exist) is called, it's very likely that someone wants to
-        // create a table in this catalog. We should create the catalog now.
-        if remote_catalog_exists
-            && self
-                .memory_catalog_manager
-                .clone()
-                .register_catalog(catalog.to_string())
-                .await?
-        {
-            info!("register catalog '{catalog}' on demand");
-        }
-
-        Ok(remote_catalog_exists)
-    }
-
-    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
-        if !self.catalog_exist(catalog).await? {
-            return Ok(false);
-        }
-
-        if !self.schema_exist(catalog, schema).await? {
-            return Ok(false);
-        }
-
-        self.memory_catalog_manager
-            .table_exist(catalog, schema, table)
-            .await
-    }
-
-    async fn catalog_names(&self) -> Result<Vec<String>> {
-        self.memory_catalog_manager.catalog_names().await
-    }
-
-    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
-        self.memory_catalog_manager.schema_names(catalog_name).await
-    }
-
-    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
-        self.memory_catalog_manager
-            .table_names(catalog_name, schema_name)
-            .await
-    }
-
-    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
-        self.memory_catalog_manager.register_catalog_sync(name)
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}

src/catalog/src/remote/region_alive_keeper.rs

@@ -1,826 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::future::Future;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use common_meta::error::InvalidProtoMsgSnafu;
-use common_meta::heartbeat::handler::{
-    HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
-};
-use common_meta::ident::TableIdent;
-use common_meta::RegionIdent;
-use common_telemetry::{debug, error, info, warn};
-use snafu::{OptionExt, ResultExt};
-use store_api::storage::RegionNumber;
-use table::engine::manager::TableEngineManagerRef;
-use table::engine::{CloseTableResult, EngineContext, TableEngineRef};
-use table::requests::CloseTableRequest;
-use table::TableRef;
-use tokio::sync::{mpsc, oneshot, Mutex};
-use tokio::task::JoinHandle;
-use tokio::time::{Duration, Instant};
-
-use crate::error::{Result, TableEngineNotFoundSnafu};
-
-/// [RegionAliveKeepers] manages all [RegionAliveKeeper] in a scope of tables.
-pub struct RegionAliveKeepers {
-    table_engine_manager: TableEngineManagerRef,
-    keepers: Arc<Mutex<HashMap<TableIdent, Arc<RegionAliveKeeper>>>>,
-    heartbeat_interval_millis: u64,
-    started: AtomicBool,
-
-    /// The epoch when [RegionAliveKeepers] is created. It's used to get a monotonically non-decreasing
-    /// elapsed time when submitting heartbeats to Metasrv (because [Instant] is monotonically
-    /// non-decreasing). The heartbeat request will carry the duration since this epoch, and the
-    /// duration acts like an "invariant point" for region's keep alive lease.
-    epoch: Instant,
-}
-
-impl RegionAliveKeepers {
-    pub fn new(
-        table_engine_manager: TableEngineManagerRef,
-        heartbeat_interval_millis: u64,
-    ) -> Self {
-        Self {
-            table_engine_manager,
-            keepers: Arc::new(Mutex::new(HashMap::new())),
-            heartbeat_interval_millis,
-            started: AtomicBool::new(false),
-            epoch: Instant::now(),
-        }
-    }
-
-    pub async fn find_keeper(&self, table_ident: &TableIdent) -> Option<Arc<RegionAliveKeeper>> {
-        self.keepers.lock().await.get(table_ident).cloned()
-    }
-
-    pub async fn register_table(&self, table_ident: TableIdent, table: TableRef) -> Result<()> {
-        let keeper = self.find_keeper(&table_ident).await;
-        if keeper.is_some() {
-            return Ok(());
-        }
-
-        let table_engine = self
-            .table_engine_manager
-            .engine(&table_ident.engine)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: &table_ident.engine,
-            })?;
-
-        let keeper = Arc::new(RegionAliveKeeper::new(
-            table_engine,
-            table_ident.clone(),
-            self.heartbeat_interval_millis,
-        ));
-        for r in table.table_info().meta.region_numbers.iter() {
-            keeper.register_region(*r).await;
-        }
-
-        let mut keepers = self.keepers.lock().await;
-        let _ = keepers.insert(table_ident.clone(), keeper.clone());
-
-        if self.started.load(Ordering::Relaxed) {
-            keeper.start().await;
-
-            info!("RegionAliveKeeper for table {table_ident} is started!");
-        } else {
-            info!("RegionAliveKeeper for table {table_ident} is registered but not started yet!");
-        }
-        Ok(())
-    }
-
-    pub async fn deregister_table(
-        &self,
-        table_ident: &TableIdent,
-    ) -> Option<Arc<RegionAliveKeeper>> {
-        self.keepers.lock().await.remove(table_ident).map(|x| {
-            info!("Deregister RegionAliveKeeper for table {table_ident}");
-            x
-        })
-    }
-
-    pub async fn register_region(&self, region_ident: &RegionIdent) {
-        let table_ident = &region_ident.table_ident;
-        let Some(keeper) = self.find_keeper(table_ident).await else {
-            // Alive keeper could be affected by lagging msg, just warn and ignore.
-            warn!("Alive keeper for region {region_ident} is not found!");
-            return;
-        };
-        keeper.register_region(region_ident.region_number).await
-    }
-
-    pub async fn deregister_region(&self, region_ident: &RegionIdent) {
-        let table_ident = &region_ident.table_ident;
-        let Some(keeper) = self.find_keeper(table_ident).await else {
-            // Alive keeper could be affected by lagging msg, just warn and ignore.
-            warn!("Alive keeper for region {region_ident} is not found!");
-            return;
-        };
-        let _ = keeper.deregister_region(region_ident.region_number).await;
-    }
-
-    pub async fn start(&self) {
-        let keepers = self.keepers.lock().await;
-        for keeper in keepers.values() {
-            keeper.start().await;
-        }
-        self.started.store(true, Ordering::Relaxed);
-
-        info!(
-            "RegionAliveKeepers for tables {:?} are started!",
-            keepers.keys().map(|x| x.to_string()).collect::<Vec<_>>(),
-        );
-    }
-
-    pub fn epoch(&self) -> Instant {
-        self.epoch
-    }
-}
-
-#[async_trait]
-impl HeartbeatResponseHandler for RegionAliveKeepers {
-    fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
-        !ctx.response.region_leases.is_empty()
-    }
-
-    async fn handle(
-        &self,
-        ctx: &mut HeartbeatResponseHandlerContext,
-    ) -> common_meta::error::Result<HandleControl> {
-        let leases = ctx.response.region_leases.drain(..).collect::<Vec<_>>();
-        for lease in leases {
-            let table_ident: TableIdent = match lease
-                .table_ident
-                .context(InvalidProtoMsgSnafu {
-                    err_msg: "'table_ident' is missing in RegionLease",
-                })
-                .and_then(|x| x.try_into())
-            {
-                Ok(x) => x,
-                Err(e) => {
-                    error!(e; "");
-                    continue;
-                }
-            };
-
-            let Some(keeper) = self.keepers.lock().await.get(&table_ident).cloned() else {
-                // Alive keeper could be affected by lagging msg, just warn and ignore.
-                warn!("Alive keeper for table {table_ident} is not found!");
-                continue;
-            };
-
-            let start_instant = self.epoch + Duration::from_millis(lease.duration_since_epoch);
-            let deadline = start_instant + Duration::from_secs(lease.lease_seconds);
-            keeper.keep_lived(lease.regions, deadline).await;
-        }
-        Ok(HandleControl::Continue)
-    }
-}
-
-/// [RegionAliveKeeper] starts a countdown for each region in a table. When deadline is reached,
-/// the region will be closed.
-/// The deadline is controlled by Metasrv. It works like "lease" for regions: a Datanode submits its
-/// opened regions to Metasrv, in heartbeats. If Metasrv decides some region could be resided in this
-/// Datanode, it will "extend" the region's "lease", with a deadline for [RegionAliveKeeper] to
-/// countdown.
-pub struct RegionAliveKeeper {
-    table_engine: TableEngineRef,
-    table_ident: TableIdent,
-    countdown_task_handles: Arc<Mutex<HashMap<RegionNumber, Arc<CountdownTaskHandle>>>>,
-    heartbeat_interval_millis: u64,
-    started: AtomicBool,
-}
-
-impl RegionAliveKeeper {
-    fn new(
-        table_engine: TableEngineRef,
-        table_ident: TableIdent,
-        heartbeat_interval_millis: u64,
-    ) -> Self {
-        Self {
-            table_engine,
-            table_ident,
-            countdown_task_handles: Arc::new(Mutex::new(HashMap::new())),
-            heartbeat_interval_millis,
-            started: AtomicBool::new(false),
-        }
-    }
-
-    async fn find_handle(&self, region: &RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
-        self.countdown_task_handles
-            .lock()
-            .await
-            .get(region)
-            .cloned()
-    }
-
-    async fn register_region(&self, region: RegionNumber) {
-        if self.find_handle(&region).await.is_some() {
-            return;
-        }
-
-        let countdown_task_handles = Arc::downgrade(&self.countdown_task_handles);
-        let on_task_finished = async move {
-            if let Some(x) = countdown_task_handles.upgrade() {
-                let _ = x.lock().await.remove(&region);
-            } // Else the countdown task handles map could be dropped because the keeper is dropped.
-        };
-        let handle = Arc::new(CountdownTaskHandle::new(
-            self.table_engine.clone(),
-            self.table_ident.clone(),
-            region,
-            || on_task_finished,
-        ));
-
-        let mut handles = self.countdown_task_handles.lock().await;
-        let _ = handles.insert(region, handle.clone());
-
-        if self.started.load(Ordering::Relaxed) {
-            handle.start(self.heartbeat_interval_millis).await;
-
-            info!(
-                "Region alive countdown for region {region} in table {} is started!",
-                self.table_ident
-            );
-        } else {
-            info!(
-                "Region alive countdown for region {region} in table {} is registered but not started yet!",
-                self.table_ident
-            );
-        }
-    }
-
-    async fn deregister_region(&self, region: RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
-        self.countdown_task_handles
-            .lock()
-            .await
-            .remove(&region)
-            .map(|x| {
-                info!(
-                    "Deregister alive countdown for region {region} in table {}",
-                    self.table_ident
-                );
-                x
-            })
-    }
-
-    async fn start(&self) {
-        let handles = self.countdown_task_handles.lock().await;
-        for handle in handles.values() {
-            handle.start(self.heartbeat_interval_millis).await;
-        }
-
-        self.started.store(true, Ordering::Relaxed);
-        info!(
-            "Region alive countdowns for regions {:?} in table {} are started!",
-            handles.keys().copied().collect::<Vec<_>>(),
-            self.table_ident
-        );
-    }
-
-    async fn keep_lived(&self, designated_regions: Vec<RegionNumber>, deadline: Instant) {
-        for region in designated_regions {
-            if let Some(handle) = self.find_handle(&region).await {
-                handle.reset_deadline(deadline).await;
-            }
-            // Else the region alive keeper might be triggered by lagging messages, we can safely ignore it.
-        }
-    }
-
-    pub async fn deadline(&self, region: RegionNumber) -> Option<Instant> {
-        let mut deadline = None;
-        if let Some(handle) = self.find_handle(&region).await {
-            let (s, r) = oneshot::channel();
-            if handle.tx.send(CountdownCommand::Deadline(s)).await.is_ok() {
-                deadline = r.await.ok()
-            }
-        }
-        deadline
-    }
-
-    pub fn table_ident(&self) -> &TableIdent {
-        &self.table_ident
-    }
-}
-
-#[derive(Debug)]
-enum CountdownCommand {
-    Start(u64),
-    Reset(Instant),
-    Deadline(oneshot::Sender<Instant>),
-}
-
-struct CountdownTaskHandle {
-    tx: mpsc::Sender<CountdownCommand>,
-    handler: JoinHandle<()>,
-    table_ident: TableIdent,
-    region: RegionNumber,
-}
-
-impl CountdownTaskHandle {
-    /// Creates a new [CountdownTaskHandle] and starts the countdown task.
-    /// # Params
-    /// - `on_task_finished`: a callback to be invoked when the task is finished. Note that it will not
-    ///   be invoked if the task is cancelled (by dropping the handle). This is because we want something
-    ///   meaningful to be done when the task is finished, e.g. deregister the handle from the map.
-    ///   While dropping the handle does not necessarily mean the task is finished.
-    fn new<Fut>(
-        table_engine: TableEngineRef,
-        table_ident: TableIdent,
-        region: RegionNumber,
-        on_task_finished: impl FnOnce() -> Fut + Send + 'static,
-    ) -> Self
-    where
-        Fut: Future<Output = ()> + Send,
-    {
-        let (tx, rx) = mpsc::channel(1024);
-
-        let mut countdown_task = CountdownTask {
-            table_engine,
-            table_ident: table_ident.clone(),
-            region,
-            rx,
-        };
-        let handler = common_runtime::spawn_bg(async move {
-            countdown_task.run().await;
-            on_task_finished().await;
-        });
-
-        Self {
-            tx,
-            handler,
-            table_ident,
-            region,
-        }
-    }
-
-    async fn start(&self, heartbeat_interval_millis: u64) {
-        if let Err(e) = self
-            .tx
-            .send(CountdownCommand::Start(heartbeat_interval_millis))
-            .await
-        {
-            warn!(
-                "Failed to start region alive keeper countdown: {e}. \
-                Maybe the task is stopped due to region been closed."
-            );
-        }
-    }
-
-    async fn reset_deadline(&self, deadline: Instant) {
-        if let Err(e) = self.tx.send(CountdownCommand::Reset(deadline)).await {
-            warn!(
-                "Failed to reset region alive keeper deadline: {e}. \
-                Maybe the task is stopped due to region been closed."
-            );
-        }
-    }
-}
-
-impl Drop for CountdownTaskHandle {
-    fn drop(&mut self) {
-        debug!(
-            "Aborting region alive countdown task for region {} in table {}",
-            self.region, self.table_ident,
-        );
-        self.handler.abort();
-    }
-}
-
-struct CountdownTask {
-    table_engine: TableEngineRef,
-    table_ident: TableIdent,
-    region: RegionNumber,
-    rx: mpsc::Receiver<CountdownCommand>,
-}
-
-impl CountdownTask {
-    async fn run(&mut self) {
-        // 30 years. See `Instant::far_future`.
-        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
-
-        // Make sure the alive countdown is not gonna happen before heartbeat task is started (the
-        // "start countdown" command will be sent from heartbeat task).
-        let countdown = tokio::time::sleep_until(far_future);
-        tokio::pin!(countdown);
-
-        let region = &self.region;
-        let table_ident = &self.table_ident;
-        loop {
-            tokio::select! {
-                command = self.rx.recv() => {
-                    match command {
-                        Some(CountdownCommand::Start(heartbeat_interval_millis)) => {
-                            // Set first deadline in 4 heartbeats (roughly after 20 seconds from now if heartbeat
-                            // interval is set to default 5 seconds), to make Datanode and Metasrv more tolerable to
-                            // network or other jitters during startup.
-                            let first_deadline = Instant::now() + Duration::from_millis(heartbeat_interval_millis) * 4;
-                            countdown.set(tokio::time::sleep_until(first_deadline));
-                        },
-                        Some(CountdownCommand::Reset(deadline)) => {
-                            if countdown.deadline() < deadline {
-                                debug!(
-                                    "Reset deadline of region {region} of table {table_ident} to approximately {} seconds later",
-                                    (deadline - Instant::now()).as_secs_f32(),
-                                );
-                                countdown.set(tokio::time::sleep_until(deadline));
-                            }
-                            // Else the countdown could be either:
-                            // - not started yet;
-                            // - during startup protection;
-                            // - received a lagging heartbeat message.
-                            // All can be safely ignored.
-                        },
-                        None => {
-                            info!(
-                                "The handle of countdown task for region {region} of table {table_ident} \
-                                is dropped, RegionAliveKeeper out."
-                            );
-                            break;
-                        },
-                        Some(CountdownCommand::Deadline(tx)) => {
-                            let _ = tx.send(countdown.deadline());
-                        }
-                    }
-                }
-                () = &mut countdown => {
-                    let result = self.close_region().await;
-                    warn!(
-                        "Region {region} of table {table_ident} is closed, result: {result:?}. \
-                        RegionAliveKeeper out.",
-                    );
-                    break;
-                }
-            }
-        }
-    }
-
-    async fn close_region(&self) -> CloseTableResult {
-        let ctx = EngineContext::default();
-        let region = self.region;
-        let table_ident = &self.table_ident;
-        loop {
-            let request = CloseTableRequest {
-                catalog_name: table_ident.catalog.clone(),
-                schema_name: table_ident.schema.clone(),
-                table_name: table_ident.table.clone(),
-                table_id: table_ident.table_id,
-                region_numbers: vec![region],
-                flush: true,
-            };
-            match self.table_engine.close_table(&ctx, request).await {
-                Ok(result) => return result,
-                // If region is failed to close, immediately retry. Maybe we should panic instead?
-                Err(e) => error!(e;
-                    "Failed to close region {region} of table {table_ident}. \
-                    For the integrity of data, retry closing and retry without wait.",
-                ),
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::sync::atomic::{AtomicBool, Ordering};
-    use std::sync::Arc;
-
-    use api::v1::meta::{HeartbeatResponse, RegionLease};
-    use common_meta::heartbeat::mailbox::HeartbeatMailbox;
-    use datatypes::schema::RawSchema;
-    use table::engine::manager::MemoryTableEngineManager;
-    use table::engine::TableEngine;
-    use table::requests::{CreateTableRequest, TableOptions};
-    use table::test_util::EmptyTable;
-
-    use super::*;
-    use crate::remote::mock::MockTableEngine;
-
-    async fn prepare_keepers() -> (TableIdent, RegionAliveKeepers) {
-        let table_engine = Arc::new(MockTableEngine::default());
-        let table_engine_manager = Arc::new(MemoryTableEngineManager::new(table_engine));
-        let keepers = RegionAliveKeepers::new(table_engine_manager, 5000);
-
-        let catalog = "my_catalog";
-        let schema = "my_schema";
-        let table = "my_table";
-        let table_ident = TableIdent {
-            catalog: catalog.to_string(),
-            schema: schema.to_string(),
-            table: table.to_string(),
-            table_id: 1,
-            engine: "MockTableEngine".to_string(),
-        };
-        let table = Arc::new(EmptyTable::new(CreateTableRequest {
-            id: 1,
-            catalog_name: catalog.to_string(),
-            schema_name: schema.to_string(),
-            table_name: table.to_string(),
-            desc: None,
-            schema: RawSchema {
-                column_schemas: vec![],
-                timestamp_index: None,
-                version: 0,
-            },
-            region_numbers: vec![1, 2, 3],
-            primary_key_indices: vec![],
-            create_if_not_exists: false,
-            table_options: TableOptions::default(),
-            engine: "MockTableEngine".to_string(),
-        }));
-        keepers
-            .register_table(table_ident.clone(), table)
-            .await
-            .unwrap();
-        assert!(keepers.keepers.lock().await.contains_key(&table_ident));
-
-        (table_ident, keepers)
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_handle_heartbeat_response() {
-        let (table_ident, keepers) = prepare_keepers().await;
-
-        keepers.start().await;
-        let startup_protection_until = Instant::now() + Duration::from_secs(21);
-
-        let duration_since_epoch = (Instant::now() - keepers.epoch).as_millis() as _;
-        let lease_seconds = 100;
-        let response = HeartbeatResponse {
-            region_leases: vec![RegionLease {
-                table_ident: Some(table_ident.clone().into()),
-                regions: vec![1, 3], // Not extending region 2's lease time.
-                duration_since_epoch,
-                lease_seconds,
-            }],
-            ..Default::default()
-        };
-        let keep_alive_until = keepers.epoch
-            + Duration::from_millis(duration_since_epoch)
-            + Duration::from_secs(lease_seconds);
-
-        let (tx, _) = mpsc::channel(8);
-        let mailbox = Arc::new(HeartbeatMailbox::new(tx));
-        let mut ctx = HeartbeatResponseHandlerContext::new(mailbox, response);
-
-        assert!(keepers.handle(&mut ctx).await.unwrap() == HandleControl::Continue);
-
-        // sleep to wait for background task spawned in `handle`
-        tokio::time::sleep(Duration::from_secs(1)).await;
-
-        async fn test(
-            keeper: &Arc<RegionAliveKeeper>,
-            region_number: RegionNumber,
-            startup_protection_until: Instant,
-            keep_alive_until: Instant,
-            is_kept_live: bool,
-        ) {
-            let deadline = keeper.deadline(region_number).await.unwrap();
-            if is_kept_live {
-                assert!(deadline > startup_protection_until && deadline == keep_alive_until);
-            } else {
-                assert!(deadline <= startup_protection_until);
-            }
-        }
-
-        let keeper = &keepers
-            .keepers
-            .lock()
-            .await
-            .get(&table_ident)
-            .cloned()
-            .unwrap();
-
-        // Test region 1 and 3 is kept lived. Their deadlines are updated to desired instant.
-        test(keeper, 1, startup_protection_until, keep_alive_until, true).await;
-        test(keeper, 3, startup_protection_until, keep_alive_until, true).await;
-
-        // Test region 2 is not kept lived. It's deadline is not updated: still during startup protection period.
-        test(keeper, 2, startup_protection_until, keep_alive_until, false).await;
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_region_alive_keepers() {
-        let (table_ident, keepers) = prepare_keepers().await;
-
-        keepers
-            .register_region(&RegionIdent {
-                cluster_id: 1,
-                datanode_id: 1,
-                table_ident: table_ident.clone(),
-                region_number: 4,
-            })
-            .await;
-
-        keepers.start().await;
-        for keeper in keepers.keepers.lock().await.values() {
-            let regions = {
-                let handles = keeper.countdown_task_handles.lock().await;
-                handles.keys().copied().collect::<Vec<_>>()
-            };
-            for region in regions {
-                // assert countdown tasks are started
-                let deadline = keeper.deadline(region).await.unwrap();
-                assert!(deadline <= Instant::now() + Duration::from_secs(20));
-            }
-        }
-
-        keepers
-            .deregister_region(&RegionIdent {
-                cluster_id: 1,
-                datanode_id: 1,
-                table_ident: table_ident.clone(),
-                region_number: 1,
-            })
-            .await;
-        let mut regions = keepers
-            .find_keeper(&table_ident)
-            .await
-            .unwrap()
-            .countdown_task_handles
-            .lock()
-            .await
-            .keys()
-            .copied()
-            .collect::<Vec<_>>();
-        regions.sort();
-        assert_eq!(regions, vec![2, 3, 4]);
-
-        let keeper = keepers.deregister_table(&table_ident).await.unwrap();
-        assert!(Arc::try_unwrap(keeper).is_ok(), "keeper is not dropped");
-        assert!(keepers.keepers.lock().await.is_empty());
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_region_alive_keeper() {
-        let table_engine = Arc::new(MockTableEngine::default());
-        let table_ident = TableIdent {
-            catalog: "my_catalog".to_string(),
-            schema: "my_schema".to_string(),
-            table: "my_table".to_string(),
-            table_id: 1024,
-            engine: "mito".to_string(),
-        };
-        let keeper = RegionAliveKeeper::new(table_engine, table_ident, 1000);
-
-        let region = 1;
-        assert!(keeper.find_handle(&region).await.is_none());
-        keeper.register_region(region).await;
-        let _ = keeper.find_handle(&region).await.unwrap();
-
-        let ten_seconds_later = || Instant::now() + Duration::from_secs(10);
-
-        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
-        assert!(keeper.find_handle(&2).await.is_none());
-        assert!(keeper.find_handle(&3).await.is_none());
-
-        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
-        // assert if keeper is not started, keep_lived is of no use
-        assert!(keeper.deadline(region).await.unwrap() > far_future);
-
-        keeper.start().await;
-        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
-        // assert keep_lived works if keeper is started
-        assert!(keeper.deadline(region).await.unwrap() <= ten_seconds_later());
-
-        let handle = keeper.deregister_region(region).await.unwrap();
-        assert!(Arc::try_unwrap(handle).is_ok(), "handle is not dropped");
-        assert!(keeper.find_handle(&region).await.is_none());
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_countdown_task_handle() {
-        let table_engine = Arc::new(MockTableEngine::default());
-        let table_ident = TableIdent {
-            catalog: "my_catalog".to_string(),
-            schema: "my_schema".to_string(),
-            table: "my_table".to_string(),
-            table_id: 1024,
-            engine: "mito".to_string(),
-        };
-        let finished = Arc::new(AtomicBool::new(false));
-        let finished_clone = finished.clone();
-        let handle = CountdownTaskHandle::new(
-            table_engine.clone(),
-            table_ident.clone(),
-            1,
-            || async move { finished_clone.store(true, Ordering::Relaxed) },
-        );
-        let tx = handle.tx.clone();
-
-        // assert countdown task is running
-        tx.send(CountdownCommand::Start(5000)).await.unwrap();
-        assert!(!finished.load(Ordering::Relaxed));
-
-        drop(handle);
-        tokio::time::sleep(Duration::from_secs(1)).await;
-
-        // assert countdown task is stopped
-        assert!(tx
-            .try_send(CountdownCommand::Reset(
-                Instant::now() + Duration::from_secs(10)
-            ))
-            .is_err());
-        // assert `on_task_finished` is not called (because the task is aborted by the handle's drop)
-        assert!(!finished.load(Ordering::Relaxed));
-
-        let finished = Arc::new(AtomicBool::new(false));
-        let finished_clone = finished.clone();
-        let handle = CountdownTaskHandle::new(table_engine, table_ident, 1, || async move {
-            finished_clone.store(true, Ordering::Relaxed)
-        });
-        handle.tx.send(CountdownCommand::Start(100)).await.unwrap();
-        tokio::time::sleep(Duration::from_secs(1)).await;
-        // assert `on_task_finished` is called when task is finished normally
-        assert!(finished.load(Ordering::Relaxed));
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_countdown_task_run() {
-        let ctx = &EngineContext::default();
-        let catalog = "my_catalog";
-        let schema = "my_schema";
-        let table = "my_table";
-        let table_id = 1;
-        let request = CreateTableRequest {
-            id: table_id,
-            catalog_name: catalog.to_string(),
-            schema_name: schema.to_string(),
-            table_name: table.to_string(),
-            desc: None,
-            schema: RawSchema {
-                column_schemas: vec![],
-                timestamp_index: None,
-                version: 0,
-            },
-            region_numbers: vec![],
-            primary_key_indices: vec![],
-            create_if_not_exists: false,
-            table_options: TableOptions::default(),
-            engine: "mito".to_string(),
-        };
-
-        let table_engine = Arc::new(MockTableEngine::default());
-        let _ = table_engine.create_table(ctx, request).await.unwrap();
-
-        let table_ident = TableIdent {
-            catalog: catalog.to_string(),
-            schema: schema.to_string(),
-            table: table.to_string(),
-            table_id,
-            engine: "mito".to_string(),
-        };
-        let (tx, rx) = mpsc::channel(10);
-        let mut task = CountdownTask {
-            table_engine: table_engine.clone(),
-            table_ident,
-            region: 1,
-            rx,
-        };
-        let _handle = common_runtime::spawn_bg(async move {
-            task.run().await;
-        });
-
-        async fn deadline(tx: &mpsc::Sender<CountdownCommand>) -> Instant {
-            let (s, r) = oneshot::channel();
-            tx.send(CountdownCommand::Deadline(s)).await.unwrap();
-            r.await.unwrap()
-        }
-
-        // if countdown task is not started, its deadline is set to far future
-        assert!(deadline(&tx).await > Instant::now() + Duration::from_secs(86400 * 365 * 29));
-
-        // start countdown in 250ms * 4 = 1s
-        tx.send(CountdownCommand::Start(250)).await.unwrap();
-        // assert deadline is correctly set
-        assert!(deadline(&tx).await <= Instant::now() + Duration::from_secs(1));
-
-        // reset countdown in 1.5s
-        tx.send(CountdownCommand::Reset(
-            Instant::now() + Duration::from_millis(1500),
-        ))
-        .await
-        .unwrap();
-
-        // assert the table is closed after deadline is reached
-        assert!(table_engine.table_exists(ctx, table_id));
-        // spare 500ms for the task to close the table
-        tokio::time::sleep(Duration::from_millis(2000)).await;
-        assert!(!table_engine.table_exists(ctx, table_id));
-    }
-}

src/catalog/src/system.rs

@@ -1,616 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use common_catalog::consts::{
-    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
-    SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
-};
-use common_recordbatch::SendableRecordBatchStream;
-use common_telemetry::debug;
-use common_time::util;
-use datatypes::prelude::{ConcreteDataType, ScalarVector, VectorRef};
-use datatypes::schema::{ColumnSchema, RawSchema, SchemaRef};
-use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
-use serde::{Deserialize, Serialize};
-use snafu::{ensure, OptionExt, ResultExt};
-use store_api::storage::ScanRequest;
-use table::engine::{EngineContext, TableEngineRef};
-use table::metadata::{TableId, TableInfoRef, TableType};
-use table::requests::{
-    CreateTableRequest, DeleteRequest, InsertRequest, OpenTableRequest, TableOptions,
-};
-use table::{Result as TableResult, Table, TableRef};
-
-use crate::error::{
-    self, CreateSystemCatalogSnafu, EmptyValueSnafu, Error, InvalidEntryTypeSnafu, InvalidKeySnafu,
-    OpenSystemCatalogSnafu, Result, ValueDeserializeSnafu,
-};
-use crate::DeregisterTableRequest;
-
-pub const ENTRY_TYPE_INDEX: usize = 0;
-pub const KEY_INDEX: usize = 1;
-pub const VALUE_INDEX: usize = 3;
-
-pub struct SystemCatalogTable(TableRef);
-
-#[async_trait::async_trait]
-impl Table for SystemCatalogTable {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.0.schema()
-    }
-
-    async fn scan_to_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
-        self.0.scan_to_stream(request).await
-    }
-
-    /// Insert values into table.
-    async fn insert(&self, request: InsertRequest) -> TableResult<usize> {
-        self.0.insert(request).await
-    }
-
-    fn table_info(&self) -> TableInfoRef {
-        self.0.table_info()
-    }
-
-    fn table_type(&self) -> TableType {
-        self.0.table_type()
-    }
-
-    async fn delete(&self, request: DeleteRequest) -> TableResult<usize> {
-        self.0.delete(request).await
-    }
-
-    fn statistics(&self) -> Option<table::stats::TableStatistics> {
-        self.0.statistics()
-    }
-}
-
-impl SystemCatalogTable {
-    pub async fn new(engine: TableEngineRef) -> Result<Self> {
-        let request = OpenTableRequest {
-            catalog_name: SYSTEM_CATALOG_NAME.to_string(),
-            schema_name: INFORMATION_SCHEMA_NAME.to_string(),
-            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
-            table_id: SYSTEM_CATALOG_TABLE_ID,
-            region_numbers: vec![0],
-        };
-        let schema = build_system_catalog_schema();
-        let ctx = EngineContext::default();
-
-        if let Some(table) = engine
-            .open_table(&ctx, request)
-            .await
-            .context(OpenSystemCatalogSnafu)?
-        {
-            Ok(Self(table))
-        } else {
-            // system catalog table is not yet created, try to create
-            let request = CreateTableRequest {
-                id: SYSTEM_CATALOG_TABLE_ID,
-                catalog_name: SYSTEM_CATALOG_NAME.to_string(),
-                schema_name: INFORMATION_SCHEMA_NAME.to_string(),
-                table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
-                desc: Some("System catalog table".to_string()),
-                schema,
-                region_numbers: vec![0],
-                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX],
-                create_if_not_exists: true,
-                table_options: TableOptions::default(),
-                engine: engine.name().to_string(),
-            };
-
-            let table = engine
-                .create_table(&ctx, request)
-                .await
-                .context(CreateSystemCatalogSnafu)?;
-            Ok(Self(table))
-        }
-    }
-
-    /// Create a stream of all entries inside system catalog table
-    pub async fn records(&self) -> Result<SendableRecordBatchStream> {
-        let full_projection = None;
-        let scan_req = ScanRequest {
-            sequence: None,
-            projection: full_projection,
-            filters: vec![],
-            output_ordering: None,
-            limit: None,
-        };
-        let stream = self
-            .scan_to_stream(scan_req)
-            .await
-            .context(error::SystemCatalogTableScanSnafu)?;
-        Ok(stream)
-    }
-}
-
-/// Build system catalog table schema.
-/// A system catalog table consists of 6 columns, namely
-/// - entry_type: type of entry in current row, can be any variant of [EntryType].
-/// - key: a binary encoded key of entry, differs according to different entry type.
-/// - timestamp: currently not used.
-/// - value: JSON-encoded value of entry's metadata.
-/// - gmt_created: create time of this metadata.
-/// - gmt_modified: last updated time of this metadata.
-fn build_system_catalog_schema() -> RawSchema {
-    let cols = vec![
-        ColumnSchema::new(
-            "entry_type".to_string(),
-            ConcreteDataType::uint8_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "key".to_string(),
-            ConcreteDataType::binary_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "timestamp".to_string(),
-            ConcreteDataType::timestamp_millisecond_datatype(),
-            false,
-        )
-        .with_time_index(true),
-        ColumnSchema::new(
-            "value".to_string(),
-            ConcreteDataType::binary_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "gmt_created".to_string(),
-            ConcreteDataType::timestamp_millisecond_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "gmt_modified".to_string(),
-            ConcreteDataType::timestamp_millisecond_datatype(),
-            false,
-        ),
-    ];
-
-    RawSchema::new(cols)
-}
-
-/// Formats key string for table entry in system catalog
-#[inline]
-pub fn format_table_entry_key(catalog: &str, schema: &str, table_id: TableId) -> String {
-    format!("{catalog}.{schema}.{table_id}")
-}
-
-pub fn build_table_insert_request(
-    catalog: String,
-    schema: String,
-    table_name: String,
-    table_id: TableId,
-    engine: String,
-) -> InsertRequest {
-    let entry_key = format_table_entry_key(&catalog, &schema, table_id);
-    build_insert_request(
-        EntryType::Table,
-        entry_key.as_bytes(),
-        serde_json::to_string(&TableEntryValue {
-            table_name,
-            engine,
-            is_deleted: false,
-        })
-        .unwrap()
-        .as_bytes(),
-    )
-}
-
-pub(crate) fn build_table_deletion_request(
-    request: &DeregisterTableRequest,
-    table_id: TableId,
-) -> InsertRequest {
-    let entry_key = format_table_entry_key(&request.catalog, &request.schema, table_id);
-    build_insert_request(
-        EntryType::Table,
-        entry_key.as_bytes(),
-        serde_json::to_string(&TableEntryValue {
-            table_name: "".to_string(),
-            engine: "".to_string(),
-            is_deleted: true,
-        })
-        .unwrap()
-        .as_bytes(),
-    )
-}
-
-fn build_primary_key_columns(entry_type: EntryType, key: &[u8]) -> HashMap<String, VectorRef> {
-    HashMap::from([
-        (
-            "entry_type".to_string(),
-            Arc::new(UInt8Vector::from_slice([entry_type as u8])) as VectorRef,
-        ),
-        (
-            "key".to_string(),
-            Arc::new(BinaryVector::from_slice(&[key])) as VectorRef,
-        ),
-        (
-            "timestamp".to_string(),
-            // Timestamp in key part is intentionally left to 0
-            Arc::new(TimestampMillisecondVector::from_slice([0])) as VectorRef,
-        ),
-    ])
-}
-
-pub fn build_schema_insert_request(catalog_name: String, schema_name: String) -> InsertRequest {
-    let full_schema_name = format!("{catalog_name}.{schema_name}");
-    build_insert_request(
-        EntryType::Schema,
-        full_schema_name.as_bytes(),
-        serde_json::to_string(&SchemaEntryValue {})
-            .unwrap()
-            .as_bytes(),
-    )
-}
-
-pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) -> InsertRequest {
-    let primary_key_columns = build_primary_key_columns(entry_type, key);
-
-    let mut columns_values = HashMap::with_capacity(6);
-    columns_values.extend(primary_key_columns);
-
-    let _ = columns_values.insert(
-        "value".to_string(),
-        Arc::new(BinaryVector::from_slice(&[value])) as _,
-    );
-
-    let now = util::current_time_millis();
-    let _ = columns_values.insert(
-        "gmt_created".to_string(),
-        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
-    );
-
-    let _ = columns_values.insert(
-        "gmt_modified".to_string(),
-        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
-    );
-
-    InsertRequest {
-        catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-        schema_name: DEFAULT_SCHEMA_NAME.to_string(),
-        table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
-        columns_values,
-        region_number: 0, // system catalog table has only one region
-    }
-}
-
-pub fn decode_system_catalog(
-    entry_type: Option<u8>,
-    key: Option<&[u8]>,
-    value: Option<&[u8]>,
-) -> Result<Entry> {
-    debug!(
-        "Decode system catalog entry: {:?}, {:?}, {:?}",
-        entry_type, key, value
-    );
-    let entry_type = entry_type.context(InvalidKeySnafu { key: None })?;
-    let key = String::from_utf8_lossy(key.context(InvalidKeySnafu { key: None })?);
-
-    match EntryType::try_from(entry_type)? {
-        EntryType::Catalog => {
-            // As for catalog entry, the key is a string with format: `<catalog_name>`
-            // and the value is current not used.
-            let catalog_name = key.to_string();
-            Ok(Entry::Catalog(CatalogEntry { catalog_name }))
-        }
-        EntryType::Schema => {
-            // As for schema entry, the key is a string with format: `<catalog_name>.<schema_name>`
-            // and the value is current not used.
-            let schema_parts = key.split('.').collect::<Vec<_>>();
-            ensure!(
-                schema_parts.len() == 2,
-                InvalidKeySnafu {
-                    key: Some(key.to_string())
-                }
-            );
-            Ok(Entry::Schema(SchemaEntry {
-                catalog_name: schema_parts[0].to_string(),
-                schema_name: schema_parts[1].to_string(),
-            }))
-        }
-
-        EntryType::Table => {
-            // As for table entry, the key is a string with format: `<catalog_name>.<schema_name>.<table_id>`
-            // and the value is a JSON string with format: `{"table_name": <table_name>}`
-            let table_parts = key.split('.').collect::<Vec<_>>();
-            ensure!(
-                table_parts.len() >= 3,
-                InvalidKeySnafu {
-                    key: Some(key.to_string())
-                }
-            );
-            let value = value.context(EmptyValueSnafu)?;
-            debug!("Table meta value: {}", String::from_utf8_lossy(value));
-            let table_meta: TableEntryValue =
-                serde_json::from_slice(value).context(ValueDeserializeSnafu)?;
-            let table_id = table_parts[2].parse::<TableId>().unwrap();
-            Ok(Entry::Table(TableEntry {
-                catalog_name: table_parts[0].to_string(),
-                schema_name: table_parts[1].to_string(),
-                table_name: table_meta.table_name,
-                table_id,
-                engine: table_meta.engine,
-                is_deleted: table_meta.is_deleted,
-            }))
-        }
-    }
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub enum EntryType {
-    Catalog = 1,
-    Schema = 2,
-    Table = 3,
-}
-
-impl TryFrom<u8> for EntryType {
-    type Error = Error;
-
-    fn try_from(value: u8) -> std::result::Result<Self, Self::Error> {
-        match value {
-            b if b == Self::Catalog as u8 => Ok(Self::Catalog),
-            b if b == Self::Schema as u8 => Ok(Self::Schema),
-            b if b == Self::Table as u8 => Ok(Self::Table),
-            b => InvalidEntryTypeSnafu {
-                entry_type: Some(b),
-            }
-            .fail(),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub enum Entry {
-    Catalog(CatalogEntry),
-    Schema(SchemaEntry),
-    Table(TableEntry),
-}
-
-#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub struct CatalogEntry {
-    pub catalog_name: String,
-}
-
-#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub struct SchemaEntry {
-    pub catalog_name: String,
-    pub schema_name: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub struct SchemaEntryValue;
-
-#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub struct TableEntry {
-    pub catalog_name: String,
-    pub schema_name: String,
-    pub table_name: String,
-    pub table_id: TableId,
-    pub engine: String,
-    pub is_deleted: bool,
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub struct TableEntryValue {
-    pub table_name: String,
-
-    #[serde(default = "mito_engine")]
-    pub engine: String,
-
-    #[serde(default = "not_deleted")]
-    pub is_deleted: bool,
-}
-
-fn mito_engine() -> String {
-    MITO_ENGINE.to_string()
-}
-
-fn not_deleted() -> bool {
-    false
-}
-
-#[cfg(test)]
-mod tests {
-    use common_recordbatch::RecordBatches;
-    use common_test_util::temp_dir::{create_temp_dir, TempDir};
-    use datatypes::value::Value;
-    use log_store::NoopLogStore;
-    use mito::config::EngineConfig;
-    use mito::engine::{MitoEngine, MITO_ENGINE};
-    use object_store::ObjectStore;
-    use storage::compaction::noop::NoopCompactionScheduler;
-    use storage::config::EngineConfig as StorageEngineConfig;
-    use storage::EngineImpl;
-    use table::metadata::TableType;
-    use table::metadata::TableType::Base;
-
-    use super::*;
-
-    #[test]
-    pub fn test_decode_catalog_entry() {
-        let entry = decode_system_catalog(
-            Some(EntryType::Catalog as u8),
-            Some("some_catalog".as_bytes()),
-            None,
-        )
-        .unwrap();
-        if let Entry::Catalog(e) = entry {
-            assert_eq!("some_catalog", e.catalog_name);
-        } else {
-            panic!("Unexpected type: {entry:?}");
-        }
-    }
-
-    #[test]
-    pub fn test_decode_schema_entry() {
-        let entry = decode_system_catalog(
-            Some(EntryType::Schema as u8),
-            Some("some_catalog.some_schema".as_bytes()),
-            None,
-        )
-        .unwrap();
-
-        if let Entry::Schema(e) = entry {
-            assert_eq!("some_catalog", e.catalog_name);
-            assert_eq!("some_schema", e.schema_name);
-        } else {
-            panic!("Unexpected type: {entry:?}");
-        }
-    }
-
-    #[test]
-    pub fn test_decode_table() {
-        let entry = decode_system_catalog(
-            Some(EntryType::Table as u8),
-            Some("some_catalog.some_schema.42".as_bytes()),
-            Some("{\"table_name\":\"some_table\"}".as_bytes()),
-        )
-        .unwrap();
-
-        if let Entry::Table(e) = entry {
-            assert_eq!("some_catalog", e.catalog_name);
-            assert_eq!("some_schema", e.schema_name);
-            assert_eq!("some_table", e.table_name);
-            assert_eq!(42, e.table_id);
-        } else {
-            panic!("Unexpected type: {entry:?}");
-        }
-    }
-
-    #[test]
-    pub fn test_decode_mismatch() {
-        assert!(decode_system_catalog(
-            Some(EntryType::Table as u8),
-            Some("some_catalog.some_schema.42".as_bytes()),
-            None,
-        )
-        .is_err());
-    }
-
-    #[test]
-    pub fn test_entry_type() {
-        assert_eq!(EntryType::Catalog, EntryType::try_from(1).unwrap());
-        assert_eq!(EntryType::Schema, EntryType::try_from(2).unwrap());
-        assert_eq!(EntryType::Table, EntryType::try_from(3).unwrap());
-        assert!(EntryType::try_from(4).is_err());
-    }
-
-    pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
-        let dir = create_temp_dir("system-table-test");
-        let store_dir = dir.path().to_string_lossy();
-        let mut builder = object_store::services::Fs::default();
-        let _ = builder.root(&store_dir);
-        let object_store = ObjectStore::new(builder).unwrap().finish();
-        let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
-        let table_engine = Arc::new(MitoEngine::new(
-            EngineConfig::default(),
-            EngineImpl::new(
-                StorageEngineConfig::default(),
-                Arc::new(NoopLogStore),
-                object_store.clone(),
-                noop_compaction_scheduler,
-            )
-            .unwrap(),
-            object_store,
-        ));
-        (dir, table_engine)
-    }
-
-    #[tokio::test]
-    async fn test_system_table_type() {
-        let (_dir, table_engine) = prepare_table_engine().await;
-        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
-        assert_eq!(Base, system_table.table_type());
-    }
-
-    #[tokio::test]
-    async fn test_system_table_info() {
-        let (_dir, table_engine) = prepare_table_engine().await;
-        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
-        let info = system_table.table_info();
-        assert_eq!(TableType::Base, info.table_type);
-        assert_eq!(SYSTEM_CATALOG_TABLE_NAME, info.name);
-        assert_eq!(SYSTEM_CATALOG_TABLE_ID, info.ident.table_id);
-        assert_eq!(SYSTEM_CATALOG_NAME, info.catalog_name);
-        assert_eq!(INFORMATION_SCHEMA_NAME, info.schema_name);
-    }
-
-    #[tokio::test]
-    async fn test_system_catalog_table_records() {
-        let (_, table_engine) = prepare_table_engine().await;
-        let catalog_table = SystemCatalogTable::new(table_engine).await.unwrap();
-
-        let table_insertion = build_table_insert_request(
-            DEFAULT_CATALOG_NAME.to_string(),
-            DEFAULT_SCHEMA_NAME.to_string(),
-            "my_table".to_string(),
-            1,
-            MITO_ENGINE.to_string(),
-        );
-        let result = catalog_table.insert(table_insertion).await.unwrap();
-        assert_eq!(result, 1);
-
-        let records = catalog_table.records().await.unwrap();
-        let mut batches = RecordBatches::try_collect(records).await.unwrap().take();
-        assert_eq!(batches.len(), 1);
-        let batch = batches.remove(0);
-        assert_eq!(batch.num_rows(), 1);
-
-        let row = batch.rows().next().unwrap();
-        let Value::UInt8(entry_type) = row[0] else {
-            unreachable!()
-        };
-        let Value::Binary(key) = row[1].clone() else {
-            unreachable!()
-        };
-        let Value::Binary(value) = row[3].clone() else {
-            unreachable!()
-        };
-        let entry = decode_system_catalog(Some(entry_type), Some(&*key), Some(&*value)).unwrap();
-        let expected = Entry::Table(TableEntry {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: "my_table".to_string(),
-            table_id: 1,
-            engine: MITO_ENGINE.to_string(),
-            is_deleted: false,
-        });
-        assert_eq!(entry, expected);
-
-        let table_deletion = build_table_deletion_request(
-            &DeregisterTableRequest {
-                catalog: DEFAULT_CATALOG_NAME.to_string(),
-                schema: DEFAULT_SCHEMA_NAME.to_string(),
-                table_name: "my_table".to_string(),
-            },
-            1,
-        );
-        let result = catalog_table.insert(table_deletion).await.unwrap();
-        assert_eq!(result, 1);
-
-        let records = catalog_table.records().await.unwrap();
-        let batches = RecordBatches::try_collect(records).await.unwrap().take();
-        assert_eq!(batches.len(), 1);
-    }
-}

src/catalog/src/table_source.rs

@@ -123,7 +123,7 @@ mod tests {
     use session::context::QueryContext;
 
     use super::*;
-    use crate::local::MemoryCatalogManager;
+    use crate::memory::MemoryCatalogManager;
 
     #[test]
     fn test_validate_table_ref() {

src/catalog/src/tables.rs

@@ -1,101 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The `tables` table in system catalog keeps a record of all tables created by user.
-
-use std::sync::Arc;
-
-use common_telemetry::logging;
-use snafu::ResultExt;
-use table::metadata::TableId;
-use table::Table;
-
-use crate::error::{self, InsertCatalogRecordSnafu, Result as CatalogResult};
-use crate::system::{
-    build_schema_insert_request, build_table_deletion_request, build_table_insert_request,
-    SystemCatalogTable,
-};
-use crate::DeregisterTableRequest;
-
-pub struct InformationSchema {
-    pub system: Arc<SystemCatalogTable>,
-}
-
-pub struct SystemCatalog {
-    pub information_schema: Arc<InformationSchema>,
-}
-
-impl SystemCatalog {
-    pub(crate) fn new(system: SystemCatalogTable) -> Self {
-        let schema = InformationSchema {
-            system: Arc::new(system),
-        };
-        Self {
-            information_schema: Arc::new(schema),
-        }
-    }
-
-    pub async fn register_table(
-        &self,
-        catalog: String,
-        schema: String,
-        table_name: String,
-        table_id: TableId,
-        engine: String,
-    ) -> crate::error::Result<usize> {
-        let request = build_table_insert_request(catalog, schema, table_name, table_id, engine);
-        self.information_schema
-            .system
-            .insert(request)
-            .await
-            .context(InsertCatalogRecordSnafu)
-    }
-
-    pub(crate) async fn deregister_table(
-        &self,
-        request: &DeregisterTableRequest,
-        table_id: TableId,
-    ) -> CatalogResult<()> {
-        self.information_schema
-            .system
-            .insert(build_table_deletion_request(request, table_id))
-            .await
-            .map(|x| {
-                if x != 1 {
-                    let table = common_catalog::format_full_table_name(
-                        &request.catalog,
-                        &request.schema,
-                        &request.table_name
-                    );
-                    logging::warn!("Failed to delete table record from information_schema, unexpected returned result: {x}, table: {table}");
-                }
-            })
-            .with_context(|_| error::DeregisterTableSnafu {
-                request: request.clone(),
-            })
-    }
-
-    pub async fn register_schema(
-        &self,
-        catalog: String,
-        schema: String,
-    ) -> crate::error::Result<usize> {
-        let request = build_schema_insert_request(catalog, schema);
-        self.information_schema
-            .system
-            .insert(request)
-            .await
-            .context(InsertCatalogRecordSnafu)
-    }
-}

src/catalog/tests/local_catalog_tests.rs

@@ -1,176 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use catalog::local::LocalCatalogManager;
-    use catalog::{CatalogManager, RegisterTableRequest, RenameTableRequest};
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-    use common_telemetry::{error, info};
-    use common_test_util::temp_dir::TempDir;
-    use mito::config::EngineConfig;
-    use table::engine::manager::MemoryTableEngineManager;
-    use table::table::numbers::NumbersTable;
-    use table::TableRef;
-    use tokio::sync::Mutex;
-
-    async fn create_local_catalog_manager(
-    ) -> Result<(TempDir, LocalCatalogManager), catalog::error::Error> {
-        let (dir, object_store) =
-            mito::table::test_util::new_test_object_store("setup_mock_engine_and_table").await;
-        let mock_engine = Arc::new(mito::table::test_util::MockMitoEngine::new(
-            EngineConfig::default(),
-            mito::table::test_util::MockEngine::default(),
-            object_store,
-        ));
-        let engine_manager = Arc::new(MemoryTableEngineManager::new(mock_engine.clone()));
-        let catalog_manager = LocalCatalogManager::try_new(engine_manager).await.unwrap();
-        catalog_manager.start().await?;
-        Ok((dir, catalog_manager))
-    }
-
-    #[tokio::test]
-    async fn test_rename_table() {
-        common_telemetry::init_default_ut_logging();
-        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
-        // register table
-        let table_name = "test_table";
-        let table_id = 42;
-        let table = Arc::new(NumbersTable::new(table_id));
-        let request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            table_id,
-            table: table.clone(),
-        };
-        assert!(catalog_manager.register_table(request).await.unwrap());
-
-        // rename table
-        let new_table_name = "table_t";
-        let rename_table_req = RenameTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            new_table_name: new_table_name.to_string(),
-            table_id,
-        };
-        assert!(catalog_manager
-            .rename_table(rename_table_req)
-            .await
-            .unwrap());
-
-        let registered_table = catalog_manager
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(registered_table.table_info().ident.table_id, table_id);
-    }
-
-    #[tokio::test]
-    async fn test_duplicate_register() {
-        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
-        let request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: "test_table".to_string(),
-            table_id: 42,
-            table: Arc::new(NumbersTable::new(42)),
-        };
-        assert!(catalog_manager
-            .register_table(request.clone())
-            .await
-            .unwrap());
-
-        // register table with same table id will succeed with 0 as return val.
-        assert!(!catalog_manager.register_table(request).await.unwrap());
-
-        let err = catalog_manager
-            .register_table(RegisterTableRequest {
-                catalog: DEFAULT_CATALOG_NAME.to_string(),
-                schema: DEFAULT_SCHEMA_NAME.to_string(),
-                table_name: "test_table".to_string(),
-                table_id: 43,
-                table: Arc::new(NumbersTable::new(43)),
-            })
-            .await
-            .unwrap_err();
-        assert!(
-            err.to_string()
-                .contains("Table `greptime.public.test_table` already exists"),
-            "Actual error message: {err}",
-        );
-    }
-
-    #[test]
-    fn test_concurrent_register() {
-        common_telemetry::init_default_ut_logging();
-        let rt = Arc::new(tokio::runtime::Builder::new_multi_thread().build().unwrap());
-        let (_dir, catalog_manager) =
-            rt.block_on(async { create_local_catalog_manager().await.unwrap() });
-        let catalog_manager = Arc::new(catalog_manager);
-
-        let succeed: Arc<Mutex<Option<TableRef>>> = Arc::new(Mutex::new(None));
-
-        let mut handles = Vec::with_capacity(8);
-        for i in 0..8 {
-            let catalog = catalog_manager.clone();
-            let succeed = succeed.clone();
-            let handle = rt.spawn(async move {
-                let table_id = 42 + i;
-                let table = Arc::new(NumbersTable::new(table_id));
-                let req = RegisterTableRequest {
-                    catalog: DEFAULT_CATALOG_NAME.to_string(),
-                    schema: DEFAULT_SCHEMA_NAME.to_string(),
-                    table_name: "test_table".to_string(),
-                    table_id,
-                    table: table.clone(),
-                };
-                match catalog.register_table(req).await {
-                    Ok(res) => {
-                        if res {
-                            let mut succeed = succeed.lock().await;
-                            info!("Successfully registered table: {}", table_id);
-                            *succeed = Some(table);
-                        }
-                    }
-                    Err(_) => {
-                        error!("Failed to register table {}", table_id);
-                    }
-                }
-            });
-            handles.push(handle);
-        }
-
-        rt.block_on(async move {
-            for handle in handles {
-                handle.await.unwrap();
-            }
-            let guard = succeed.lock().await;
-            let table = guard.as_ref().unwrap();
-            let table_registered = catalog_manager
-                .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "test_table")
-                .await
-                .unwrap()
-                .unwrap();
-            assert_eq!(
-                table_registered.table_info().ident.table_id,
-                table.table_info().ident.table_id
-            );
-        });
-    }
-}

src/catalog/tests/remote_catalog_tests.rs

@@ -1,490 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#![feature(assert_matches)]
-
-#[cfg(test)]
-mod tests {
-    use std::assert_matches::assert_matches;
-    use std::collections::HashSet;
-    use std::sync::Arc;
-    use std::time::Duration;
-
-    use catalog::error::Error;
-    use catalog::remote::mock::MockTableEngine;
-    use catalog::remote::region_alive_keeper::RegionAliveKeepers;
-    use catalog::remote::{CachedMetaKvBackend, RemoteCatalogManager};
-    use catalog::{CatalogManager, RegisterSchemaRequest, RegisterTableRequest};
-    use common_catalog::consts::{
-        DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
-    };
-    use common_meta::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
-    use common_meta::ident::TableIdent;
-    use common_meta::key::TableMetadataManager;
-    use common_meta::kv_backend::memory::MemoryKvBackend;
-    use common_meta::kv_backend::KvBackend;
-    use common_meta::rpc::store::{CompareAndPutRequest, PutRequest, RangeRequest};
-    use datatypes::schema::RawSchema;
-    use table::engine::manager::{MemoryTableEngineManager, TableEngineManagerRef};
-    use table::engine::{EngineContext, TableEngineRef};
-    use table::requests::CreateTableRequest;
-    use table::test_util::EmptyTable;
-    use tokio::time::Instant;
-
-    struct TestingComponents {
-        catalog_manager: Arc<RemoteCatalogManager>,
-        table_engine_manager: TableEngineManagerRef,
-        region_alive_keepers: Arc<RegionAliveKeepers>,
-    }
-
-    impl TestingComponents {
-        fn table_engine(&self) -> TableEngineRef {
-            self.table_engine_manager.engine(MITO_ENGINE).unwrap()
-        }
-    }
-
-    #[tokio::test]
-    async fn test_backend() {
-        let backend = MemoryKvBackend::<Error>::default();
-
-        let default_catalog_key = CatalogKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-        }
-        .to_string();
-        let req = PutRequest::new()
-            .with_key(default_catalog_key.as_bytes())
-            .with_value(CatalogValue.as_bytes().unwrap());
-        backend.put(req).await.unwrap();
-
-        let schema_key = SchemaKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
-        }
-        .to_string();
-        let req = PutRequest::new()
-            .with_key(schema_key.as_bytes())
-            .with_value(SchemaValue.as_bytes().unwrap());
-        backend.put(req).await.unwrap();
-
-        let req = RangeRequest::new().with_prefix(b"__c-".to_vec());
-        let res = backend
-            .range(req)
-            .await
-            .unwrap()
-            .kvs
-            .into_iter()
-            .map(|kv| String::from_utf8_lossy(kv.key()).to_string());
-        assert_eq!(
-            vec!["__c-greptime".to_string()],
-            res.into_iter().collect::<Vec<_>>()
-        );
-    }
-
-    #[tokio::test]
-    async fn test_cached_backend() {
-        let backend = CachedMetaKvBackend::wrap(Arc::new(MemoryKvBackend::default()));
-
-        let default_catalog_key = CatalogKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-        }
-        .to_string();
-        let req = PutRequest::new()
-            .with_key(default_catalog_key.as_bytes())
-            .with_value(CatalogValue.as_bytes().unwrap());
-        backend.put(req).await.unwrap();
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        let _ = ret.unwrap();
-
-        let req = CompareAndPutRequest::new()
-            .with_key(b"__c-greptime".to_vec())
-            .with_expect(CatalogValue.as_bytes().unwrap())
-            .with_value(b"123".to_vec());
-        let _ = backend.compare_and_put(req).await.unwrap();
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert_eq!(b"123", ret.as_ref().unwrap().value.as_slice());
-
-        let req = PutRequest::new()
-            .with_key(b"__c-greptime".to_vec())
-            .with_value(b"1234".to_vec());
-        let _ = backend.put(req).await;
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert_eq!(b"1234", ret.unwrap().value.as_slice());
-
-        backend.delete(b"__c-greptime", false).await.unwrap();
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert!(ret.is_none());
-    }
-
-    async fn prepare_components(node_id: u64) -> TestingComponents {
-        let backend = Arc::new(MemoryKvBackend::default());
-
-        let req = PutRequest::new()
-            .with_key(b"__c-greptime".to_vec())
-            .with_value(b"".to_vec());
-        backend.put(req).await.unwrap();
-
-        let req = PutRequest::new()
-            .with_key(b"__s-greptime-public".to_vec())
-            .with_value(b"".to_vec());
-        backend.put(req).await.unwrap();
-
-        let cached_backend = Arc::new(CachedMetaKvBackend::wrap(backend));
-
-        let table_engine = Arc::new(MockTableEngine::default());
-        let engine_manager = Arc::new(MemoryTableEngineManager::alias(
-            MITO_ENGINE.to_string(),
-            table_engine,
-        ));
-
-        let region_alive_keepers = Arc::new(RegionAliveKeepers::new(engine_manager.clone(), 5000));
-
-        let catalog_manager = RemoteCatalogManager::new(
-            engine_manager.clone(),
-            node_id,
-            cached_backend.clone(),
-            region_alive_keepers.clone(),
-            Arc::new(TableMetadataManager::new(cached_backend)),
-        );
-        catalog_manager.start().await.unwrap();
-
-        TestingComponents {
-            catalog_manager: Arc::new(catalog_manager),
-            table_engine_manager: engine_manager,
-            region_alive_keepers,
-        }
-    }
-
-    #[tokio::test]
-    async fn test_remote_catalog_default() {
-        common_telemetry::init_default_ut_logging();
-        let node_id = 42;
-        let TestingComponents {
-            catalog_manager, ..
-        } = prepare_components(node_id).await;
-        assert_eq!(
-            vec![DEFAULT_CATALOG_NAME.to_string()],
-            catalog_manager.catalog_names().await.unwrap()
-        );
-
-        let mut schema_names = catalog_manager
-            .schema_names(DEFAULT_CATALOG_NAME)
-            .await
-            .unwrap();
-        schema_names.sort_unstable();
-        assert_eq!(
-            vec![
-                INFORMATION_SCHEMA_NAME.to_string(),
-                DEFAULT_SCHEMA_NAME.to_string()
-            ],
-            schema_names
-        );
-    }
-
-    #[tokio::test]
-    async fn test_remote_catalog_register_nonexistent() {
-        common_telemetry::init_default_ut_logging();
-        let node_id = 42;
-        let components = prepare_components(node_id).await;
-
-        // register a new table with an nonexistent catalog
-        let catalog_name = "nonexistent_catalog".to_string();
-        let schema_name = "nonexistent_schema".to_string();
-        let table_name = "fail_table".to_string();
-        // this schema has no effect
-        let table_schema = RawSchema::new(vec![]);
-        let table = components
-            .table_engine()
-            .create_table(
-                &EngineContext {},
-                CreateTableRequest {
-                    id: 1,
-                    catalog_name: catalog_name.clone(),
-                    schema_name: schema_name.clone(),
-                    table_name: table_name.clone(),
-                    desc: None,
-                    schema: table_schema,
-                    region_numbers: vec![0],
-                    primary_key_indices: vec![],
-                    create_if_not_exists: false,
-                    table_options: Default::default(),
-                    engine: MITO_ENGINE.to_string(),
-                },
-            )
-            .await
-            .unwrap();
-        let reg_req = RegisterTableRequest {
-            catalog: catalog_name,
-            schema: schema_name,
-            table_name,
-            table_id: 1,
-            table,
-        };
-        let res = components.catalog_manager.register_table(reg_req).await;
-
-        // because nonexistent_catalog does not exist yet.
-        assert_matches!(
-            res.err().unwrap(),
-            catalog::error::Error::CatalogNotFound { .. }
-        );
-    }
-
-    #[tokio::test]
-    async fn test_register_table() {
-        let node_id = 42;
-        let components = prepare_components(node_id).await;
-        let mut schema_names = components
-            .catalog_manager
-            .schema_names(DEFAULT_CATALOG_NAME)
-            .await
-            .unwrap();
-        schema_names.sort_unstable();
-        assert_eq!(
-            vec![
-                INFORMATION_SCHEMA_NAME.to_string(),
-                DEFAULT_SCHEMA_NAME.to_string(),
-            ],
-            schema_names
-        );
-
-        // register a new table with an nonexistent catalog
-        let catalog_name = DEFAULT_CATALOG_NAME.to_string();
-        let schema_name = DEFAULT_SCHEMA_NAME.to_string();
-        let table_name = "test_table".to_string();
-        let table_id = 1;
-        // this schema has no effect
-        let table_schema = RawSchema::new(vec![]);
-        let table = components
-            .table_engine()
-            .create_table(
-                &EngineContext {},
-                CreateTableRequest {
-                    id: table_id,
-                    catalog_name: catalog_name.clone(),
-                    schema_name: schema_name.clone(),
-                    table_name: table_name.clone(),
-                    desc: None,
-                    schema: table_schema,
-                    region_numbers: vec![0],
-                    primary_key_indices: vec![],
-                    create_if_not_exists: false,
-                    table_options: Default::default(),
-                    engine: MITO_ENGINE.to_string(),
-                },
-            )
-            .await
-            .unwrap();
-        let reg_req = RegisterTableRequest {
-            catalog: catalog_name,
-            schema: schema_name,
-            table_name: table_name.clone(),
-            table_id,
-            table,
-        };
-        assert!(components
-            .catalog_manager
-            .register_table(reg_req)
-            .await
-            .unwrap());
-        assert_eq!(
-            vec![table_name],
-            components
-                .catalog_manager
-                .table_names(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
-                .await
-                .unwrap()
-        );
-    }
-
-    #[tokio::test]
-    async fn test_register_catalog_schema_table() {
-        let node_id = 42;
-        let components = prepare_components(node_id).await;
-
-        let catalog_name = "test_catalog".to_string();
-        let schema_name = "nonexistent_schema".to_string();
-
-        // register catalog to catalog manager
-        assert!(components
-            .catalog_manager
-            .clone()
-            .register_catalog(catalog_name.clone())
-            .await
-            .is_ok());
-        assert_eq!(
-            HashSet::<String>::from_iter(vec![
-                DEFAULT_CATALOG_NAME.to_string(),
-                catalog_name.clone()
-            ]),
-            HashSet::from_iter(components.catalog_manager.catalog_names().await.unwrap())
-        );
-
-        let table_to_register = components
-            .table_engine()
-            .create_table(
-                &EngineContext {},
-                CreateTableRequest {
-                    id: 2,
-                    catalog_name: catalog_name.clone(),
-                    schema_name: schema_name.clone(),
-                    table_name: "".to_string(),
-                    desc: None,
-                    schema: RawSchema::new(vec![]),
-                    region_numbers: vec![0],
-                    primary_key_indices: vec![],
-                    create_if_not_exists: false,
-                    table_options: Default::default(),
-                    engine: MITO_ENGINE.to_string(),
-                },
-            )
-            .await
-            .unwrap();
-
-        let reg_req = RegisterTableRequest {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-            table_name: " fail_table".to_string(),
-            table_id: 2,
-            table: table_to_register,
-        };
-        // this register will fail since schema does not exist yet
-        assert_matches!(
-            components
-                .catalog_manager
-                .register_table(reg_req.clone())
-                .await
-                .unwrap_err(),
-            catalog::error::Error::SchemaNotFound { .. }
-        );
-
-        let register_schema_request = RegisterSchemaRequest {
-            catalog: catalog_name.to_string(),
-            schema: schema_name.to_string(),
-        };
-        assert!(components
-            .catalog_manager
-            .register_schema(register_schema_request)
-            .await
-            .expect("Register schema should not fail"));
-        assert!(components
-            .catalog_manager
-            .register_table(reg_req)
-            .await
-            .unwrap());
-
-        assert_eq!(
-            HashSet::from([schema_name.clone(), INFORMATION_SCHEMA_NAME.to_string()]),
-            components
-                .catalog_manager
-                .schema_names(&catalog_name)
-                .await
-                .unwrap()
-                .into_iter()
-                .collect()
-        )
-    }
-
-    #[tokio::test]
-    async fn test_register_table_before_and_after_region_alive_keeper_started() {
-        let components = prepare_components(42).await;
-        let catalog_manager = &components.catalog_manager;
-        let region_alive_keepers = &components.region_alive_keepers;
-
-        let table_before = TableIdent {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table: "table_before".to_string(),
-            table_id: 1,
-            engine: MITO_ENGINE.to_string(),
-        };
-        let request = RegisterTableRequest {
-            catalog: table_before.catalog.clone(),
-            schema: table_before.schema.clone(),
-            table_name: table_before.table.clone(),
-            table_id: table_before.table_id,
-            table: Arc::new(EmptyTable::new(CreateTableRequest {
-                id: table_before.table_id,
-                catalog_name: table_before.catalog.clone(),
-                schema_name: table_before.schema.clone(),
-                table_name: table_before.table.clone(),
-                desc: None,
-                schema: RawSchema::new(vec![]),
-                region_numbers: vec![0],
-                primary_key_indices: vec![],
-                create_if_not_exists: false,
-                table_options: Default::default(),
-                engine: MITO_ENGINE.to_string(),
-            })),
-        };
-        assert!(catalog_manager.register_table(request).await.unwrap());
-
-        let keeper = region_alive_keepers
-            .find_keeper(&table_before)
-            .await
-            .unwrap();
-        let deadline = keeper.deadline(0).await.unwrap();
-        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
-        // assert region alive countdown is not started
-        assert!(deadline > far_future);
-
-        region_alive_keepers.start().await;
-
-        let table_after = TableIdent {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table: "table_after".to_string(),
-            table_id: 2,
-            engine: MITO_ENGINE.to_string(),
-        };
-        let request = RegisterTableRequest {
-            catalog: table_after.catalog.clone(),
-            schema: table_after.schema.clone(),
-            table_name: table_after.table.clone(),
-            table_id: table_after.table_id,
-            table: Arc::new(EmptyTable::new(CreateTableRequest {
-                id: table_after.table_id,
-                catalog_name: table_after.catalog.clone(),
-                schema_name: table_after.schema.clone(),
-                table_name: table_after.table.clone(),
-                desc: None,
-                schema: RawSchema::new(vec![]),
-                region_numbers: vec![0],
-                primary_key_indices: vec![],
-                create_if_not_exists: false,
-                table_options: Default::default(),
-                engine: MITO_ENGINE.to_string(),
-            })),
-        };
-        assert!(catalog_manager.register_table(request).await.unwrap());
-
-        let keeper = region_alive_keepers
-            .find_keeper(&table_after)
-            .await
-            .unwrap();
-        let deadline = keeper.deadline(0).await.unwrap();
-        // assert countdown is started for the table registered after [RegionAliveKeepers] started
-        assert!(deadline <= Instant::now() + Duration::from_secs(20));
-
-        let keeper = region_alive_keepers
-            .find_keeper(&table_before)
-            .await
-            .unwrap();
-        let deadline = keeper.deadline(0).await.unwrap();
-        // assert countdown is started for the table registered before [RegionAliveKeepers] started, too
-        assert!(deadline <= Instant::now() + Duration::from_secs(20));
-    }
-}

src/client/Cargo.toml

@@ -11,10 +11,12 @@ testing = []
 api = { workspace = true }
 arrow-flight.workspace = true
 async-stream.workspace = true
+async-trait.workspace = true
 common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
@@ -22,12 +24,14 @@ common-telemetry = { workspace = true }
 common-time = { workspace = true }
 datafusion.workspace = true
 datatypes = { workspace = true }
+derive_builder.workspace = true
 enum_dispatch = "0.3"
 futures-util.workspace = true
-moka = { version = "0.9", features = ["future"] }
+moka = { workspace = true, features = ["future"] }
 parking_lot = "0.12"
 prost.workspace = true
 rand.workspace = true
+session = { workspace = true }
 snafu.workspace = true
 tokio-stream = { version = "0.1", features = ["net"] }
 tokio.workspace = true

src/client/examples/logical.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, TableId};
+use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, SemanticType, TableId};
 use client::{Client, Database};
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
 use prost::Message;
@@ -41,21 +41,27 @@ async fn run() {
         column_defs: vec![
             ColumnDef {
                 name: "timestamp".to_string(),
-                datatype: ColumnDataType::TimestampMillisecond as i32,
+                data_type: ColumnDataType::TimestampMillisecond as i32,
                 is_nullable: false,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Timestamp as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "key".to_string(),
-                datatype: ColumnDataType::Uint64 as i32,
+                data_type: ColumnDataType::Uint64 as i32,
                 is_nullable: false,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Tag as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "value".to_string(),
-                datatype: ColumnDataType::Uint64 as i32,
+                data_type: ColumnDataType::Uint64 as i32,
                 is_nullable: false,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
         ],
         time_index: "timestamp".to_string(),
@@ -63,7 +69,6 @@ async fn run() {
         create_if_not_exists: false,
         table_options: Default::default(),
         table_id: Some(TableId { id: 1024 }),
-        region_numbers: vec![0],
         engine: MITO_ENGINE.to_string(),
     };
 
@@ -73,7 +78,7 @@ async fn run() {
 
     let logical = mock_logical_plan();
     event!(Level::INFO, "plan size: {:#?}", logical.len());
-    let result = db.logical_plan(logical, None).await.unwrap();
+    let result = db.logical_plan(logical, 0).await.unwrap();
 
     event!(Level::INFO, "result: {:#?}", result);
 }

src/client/examples/stream_ingest.rs

@@ -42,14 +42,14 @@ async fn run() {
         .insert(vec![to_insert_request(weather_records_1())])
         .await
     {
-        error!("Error: {e}");
+        error!("Error: {e:?}");
     }
 
     if let Err(e) = stream_inserter
         .insert(vec![to_insert_request(weather_records_2())])
         .await
     {
-        error!("Error: {e}");
+        error!("Error: {e:?}");
     }
 
     let result = stream_inserter.finish().await;
@@ -59,7 +59,7 @@ async fn run() {
             info!("Rows written: {rows}");
         }
         Err(e) => {
-            error!("Error: {e}");
+            error!("Error: {e:?}");
         }
     };
 }
@@ -131,7 +131,7 @@ fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
         Column {
             column_name: "ts".to_owned(),
             values: Some(column::Values {
-                ts_millisecond_values: timestamp_millis,
+                timestamp_millisecond_values: timestamp_millis,
                 ..Default::default()
             }),
             semantic_type: SemanticType::Timestamp as i32,
@@ -177,6 +177,5 @@ fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
         table_name: "weather_demo".to_owned(),
         columns,
         row_count: rows as u32,
-        ..Default::default()
     }
 }

src/client/src/client.rs

@@ -17,6 +17,7 @@ use std::sync::Arc;
 use api::v1::greptime_database_client::GreptimeDatabaseClient;
 use api::v1::health_check_client::HealthCheckClient;
 use api::v1::prometheus_gateway_client::PrometheusGatewayClient;
+use api::v1::region::region_client::RegionClient as PbRegionClient;
 use api::v1::HealthCheckRequest;
 use arrow_flight::flight_service_client::FlightServiceClient;
 use common_grpc::channel_manager::ChannelManager;
@@ -82,11 +83,6 @@ impl Client {
         Default::default()
     }
 
-    pub fn with_manager(channel_manager: ChannelManager) -> Self {
-        let inner = Arc::new(Inner::with_manager(channel_manager));
-        Self { inner }
-    }
-
     pub fn with_urls<U, A>(urls: A) -> Self
     where
         U: AsRef<str>,
@@ -142,21 +138,48 @@ impl Client {
         Ok((addr, channel))
     }
 
+    fn max_grpc_recv_message_size(&self) -> usize {
+        self.inner
+            .channel_manager
+            .config()
+            .max_recv_message_size
+            .as_bytes() as usize
+    }
+
+    fn max_grpc_send_message_size(&self) -> usize {
+        self.inner
+            .channel_manager
+            .config()
+            .max_send_message_size
+            .as_bytes() as usize
+    }
+
     pub(crate) fn make_flight_client(&self) -> Result<FlightClient> {
         let (addr, channel) = self.find_channel()?;
         Ok(FlightClient {
             addr,
-            client: FlightServiceClient::new(channel),
+            client: FlightServiceClient::new(channel)
+                .max_decoding_message_size(self.max_grpc_recv_message_size())
+                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
     pub(crate) fn make_database_client(&self) -> Result<DatabaseClient> {
         let (_, channel) = self.find_channel()?;
         Ok(DatabaseClient {
-            inner: GreptimeDatabaseClient::new(channel),
+            inner: GreptimeDatabaseClient::new(channel)
+                .max_decoding_message_size(self.max_grpc_recv_message_size())
+                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
+    pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
+        let (_, channel) = self.find_channel()?;
+        Ok(PbRegionClient::new(channel)
+            .max_decoding_message_size(self.max_grpc_recv_message_size())
+            .max_encoding_message_size(self.max_grpc_send_message_size()))
+    }
+
     pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {
         let (_, channel) = self.find_channel()?;
         Ok(PrometheusGatewayClient::new(channel))

src/client/src/client_manager.rs

@@ -13,12 +13,15 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
+use std::sync::Arc;
 use std::time::Duration;
 
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
+use common_meta::datanode_manager::{Datanode, DatanodeManager};
 use common_meta::peer::Peer;
 use moka::future::{Cache, CacheBuilder};
 
+use crate::region::RegionRequester;
 use crate::Client;
 
 pub struct DatanodeClients {
@@ -40,6 +43,15 @@ impl Debug for DatanodeClients {
     }
 }
 
+#[async_trait::async_trait]
+impl DatanodeManager for DatanodeClients {
+    async fn datanode(&self, datanode: &Peer) -> Arc<dyn Datanode> {
+        let client = self.get_client(datanode).await;
+
+        Arc::new(RegionRequester::new(client))
+    }
+}
+
 impl DatanodeClients {
     pub fn new(config: ChannelConfig) -> Self {
         Self {

src/client/src/database.rs

@@ -17,20 +17,23 @@ use api::v1::ddl_request::Expr as DdlExpr;
 use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
-    AlterExpr, AuthHeader, CompactTableExpr, CreateTableExpr, DdlRequest, DeleteRequest,
-    DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest,
-    RequestHeader, TruncateTableExpr,
+    AlterExpr, AuthHeader, CreateTableExpr, DdlRequest, DeleteRequests, DropTableExpr,
+    GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest, RequestHeader,
+    RowInsertRequests, TruncateTableExpr,
 };
-use arrow_flight::{FlightData, Ticket};
+use arrow_flight::Ticket;
+use async_stream::stream;
 use common_error::ext::{BoxedError, ErrorExt};
-use common_grpc::flight::{flight_messages_to_recordbatches, FlightDecoder, FlightMessage};
+use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_query::Output;
+use common_recordbatch::error::ExternalSnafu;
+use common_recordbatch::RecordBatchStreamAdaptor;
 use common_telemetry::{logging, timer};
-use futures_util::{TryFutureExt, TryStreamExt};
+use futures_util::StreamExt;
 use prost::Message;
 use snafu::{ensure, ResultExt};
 
-use crate::error::{ConvertFlightDataSnafu, IllegalFlightMessagesSnafu, ServerSnafu};
+use crate::error::{ConvertFlightDataSnafu, Error, IllegalFlightMessagesSnafu, ServerSnafu};
 use crate::{error, from_grpc_response, metrics, Client, Result, StreamInserter};
 
 #[derive(Clone, Debug, Default)]
@@ -112,6 +115,11 @@ impl Database {
         self.handle(Request::Inserts(requests)).await
     }
 
+    pub async fn row_insert(&self, requests: RowInsertRequests) -> Result<u32> {
+        let _timer = timer!(metrics::METRIC_GRPC_INSERT);
+        self.handle(Request::RowInserts(requests)).await
+    }
+
     pub fn streaming_inserter(&self) -> Result<StreamInserter> {
         self.streaming_inserter_with_channel_size(65536)
     }
@@ -132,20 +140,20 @@ impl Database {
         Ok(stream_inserter)
     }
 
-    pub async fn delete(&self, request: DeleteRequest) -> Result<u32> {
+    pub async fn delete(&self, request: DeleteRequests) -> Result<u32> {
         let _timer = timer!(metrics::METRIC_GRPC_DELETE);
-        self.handle(Request::Delete(request)).await
+        self.handle(Request::Deletes(request)).await
     }
 
     async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = self.client.make_database_client()?.inner;
-        let request = self.to_rpc_request(request, None);
+        let request = self.to_rpc_request(request, 0);
         let response = client.handle(request).await?.into_inner();
         from_grpc_response(response)
     }
 
     #[inline]
-    fn to_rpc_request(&self, request: Request, trace_id: Option<u64>) -> GreptimeRequest {
+    fn to_rpc_request(&self, request: Request, trace_id: u64) -> GreptimeRequest {
         GreptimeRequest {
             header: Some(RequestHeader {
                 catalog: self.catalog.clone(),
@@ -153,7 +161,7 @@ impl Database {
                 authorization: self.ctx.auth_header.clone(),
                 dbname: self.dbname.clone(),
                 trace_id,
-                span_id: None,
+                span_id: 0,
             }),
             request: Some(request),
         }
@@ -165,16 +173,12 @@ impl Database {
             Request::Query(QueryRequest {
                 query: Some(Query::Sql(sql.to_string())),
             }),
-            None,
+            0,
         )
         .await
     }
 
-    pub async fn logical_plan(
-        &self,
-        logical_plan: Vec<u8>,
-        trace_id: Option<u64>,
-    ) -> Result<Output> {
+    pub async fn logical_plan(&self, logical_plan: Vec<u8>, trace_id: u64) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_LOGICAL_PLAN);
         self.do_get(
             Request::Query(QueryRequest {
@@ -202,7 +206,7 @@ impl Database {
                     step: step.to_string(),
                 })),
             }),
-            None,
+            0,
         )
         .await
     }
@@ -213,7 +217,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::CreateTable(expr)),
             }),
-            None,
+            0,
         )
         .await
     }
@@ -224,7 +228,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::Alter(expr)),
             }),
-            None,
+            0,
         )
         .await
     }
@@ -235,29 +239,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::DropTable(expr)),
             }),
-            None,
-        )
-        .await
-    }
-
-    pub async fn flush_table(&self, expr: FlushTableExpr) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_FLUSH_TABLE);
-        self.do_get(
-            Request::Ddl(DdlRequest {
-                expr: Some(DdlExpr::FlushTable(expr)),
-            }),
-            None,
-        )
-        .await
-    }
-
-    pub async fn compact_table(&self, expr: CompactTableExpr) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_COMPACT_TABLE);
-        self.do_get(
-            Request::Ddl(DdlRequest {
-                expr: Some(DdlExpr::CompactTable(expr)),
-            }),
-            None,
+            0,
         )
         .await
     }
@@ -268,12 +250,12 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::TruncateTable(expr)),
             }),
-            None,
+            0,
         )
         .await
     }
 
-    async fn do_get(&self, request: Request, trace_id: Option<u64>) -> Result<Output> {
+    async fn do_get(&self, request: Request, trace_id: u64) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
         let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
         let request = self.to_rpc_request(request, trace_id);
@@ -283,55 +265,81 @@ impl Database {
 
         let mut client = self.client.make_flight_client()?;
 
-        let flight_data: Vec<FlightData> = client
-            .mut_inner()
-            .do_get(request)
-            .and_then(|response| response.into_inner().try_collect())
-            .await
-            .map_err(|e| {
-                let tonic_code = e.code();
-                let e: error::Error = e.into();
-                let code = e.status_code();
-                let msg = e.to_string();
-                ServerSnafu { code, msg }
-                    .fail::<()>()
-                    .map_err(BoxedError::new)
-                    .context(error::FlightGetSnafu {
-                        tonic_code,
-                        addr: client.addr(),
-                    })
-                    .map_err(|error| {
-                        logging::error!(
-                            "Failed to do Flight get, addr: {}, code: {}, source: {}",
-                            client.addr(),
-                            tonic_code,
-                            error
-                        );
-                        error
-                    })
-                    .unwrap_err()
-            })?;
-
-        let decoder = &mut FlightDecoder::default();
-        let flight_messages = flight_data
-            .into_iter()
-            .map(|x| decoder.try_decode(x).context(ConvertFlightDataSnafu))
-            .collect::<Result<Vec<_>>>()?;
-
-        let output = if let Some(FlightMessage::AffectedRows(rows)) = flight_messages.get(0) {
-            ensure!(
-                flight_messages.len() == 1,
-                IllegalFlightMessagesSnafu {
-                    reason: "Expect 'AffectedRows' Flight messages to be one and only!"
-                }
+        let response = client.mut_inner().do_get(request).await.map_err(|e| {
+            let tonic_code = e.code();
+            let e: error::Error = e.into();
+            let code = e.status_code();
+            let msg = e.to_string();
+            let error = Error::FlightGet {
+                tonic_code,
+                addr: client.addr().to_string(),
+                source: BoxedError::new(ServerSnafu { code, msg }.build()),
+            };
+            logging::error!(
+                "Failed to do Flight get, addr: {}, code: {}, source: {:?}",
+                client.addr(),
+                tonic_code,
+                error
             );
-            Output::AffectedRows(*rows)
-        } else {
-            let recordbatches = flight_messages_to_recordbatches(flight_messages)
-                .context(ConvertFlightDataSnafu)?;
-            Output::RecordBatches(recordbatches)
+            error
+        })?;
+
+        let flight_data_stream = response.into_inner();
+        let mut decoder = FlightDecoder::default();
+
+        let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
+            flight_data
+                .map_err(Error::from)
+                .and_then(|data| decoder.try_decode(data).context(ConvertFlightDataSnafu))
+        });
+
+        let Some(first_flight_message) = flight_message_stream.next().await else {
+            return IllegalFlightMessagesSnafu {
+                reason: "Expect the response not to be empty",
+            }
+            .fail();
         };
-        Ok(output)
+
+        let first_flight_message = first_flight_message?;
+
+        match first_flight_message {
+            FlightMessage::AffectedRows(rows) => {
+                ensure!(
+                    flight_message_stream.next().await.is_none(),
+                    IllegalFlightMessagesSnafu {
+                        reason: "Expect 'AffectedRows' Flight messages to be the one and the only!"
+                    }
+                );
+                Ok(Output::AffectedRows(rows))
+            }
+            FlightMessage::Recordbatch(_) => IllegalFlightMessagesSnafu {
+                reason: "The first flight message cannot be a RecordBatch message",
+            }
+            .fail(),
+            FlightMessage::Schema(schema) => {
+                let stream = Box::pin(stream!({
+                    while let Some(flight_message) = flight_message_stream.next().await {
+                        let flight_message = flight_message
+                            .map_err(BoxedError::new)
+                            .context(ExternalSnafu)?;
+                        let FlightMessage::Recordbatch(record_batch) = flight_message else {
+                            yield IllegalFlightMessagesSnafu {reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"}
+                                        .fail()
+                                        .map_err(BoxedError::new)
+                                        .context(ExternalSnafu);
+                            break;
+                        };
+                        yield Ok(record_batch);
+                    }
+                }));
+                let record_batch_stream = RecordBatchStreamAdaptor {
+                    schema,
+                    stream,
+                    output_ordering: None,
+                };
+                Ok(Output::Stream(Box::pin(record_batch_stream)))
+            }
+        }
     }
 }
 
@@ -342,106 +350,11 @@ pub struct FlightContext {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
-    use api::helper::ColumnDataTypeWrapper;
     use api::v1::auth_header::AuthScheme;
-    use api::v1::{AuthHeader, Basic, Column};
-    use common_grpc::select::{null_mask, values};
-    use common_grpc_expr::column_to_vector;
-    use datatypes::prelude::{Vector, VectorRef};
-    use datatypes::vectors::{
-        BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
-        Int16Vector, Int32Vector, Int64Vector, Int8Vector, StringVector, UInt16Vector,
-        UInt32Vector, UInt64Vector, UInt8Vector,
-    };
+    use api::v1::{AuthHeader, Basic};
 
     use crate::database::FlightContext;
 
-    #[test]
-    fn test_column_to_vector() {
-        let mut column = create_test_column(Arc::new(BooleanVector::from(vec![true])));
-        column.datatype = -100;
-        let result = column_to_vector(&column, 1);
-        assert!(result.is_err());
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Column datatype error, source: Unknown proto column datatype: -100"
-        );
-
-        macro_rules! test_with_vector {
-            ($vector: expr) => {
-                let vector = Arc::new($vector);
-                let column = create_test_column(vector.clone());
-                let result = column_to_vector(&column, vector.len() as u32).unwrap();
-                assert_eq!(result, vector as VectorRef);
-            };
-        }
-
-        test_with_vector!(BooleanVector::from(vec![Some(true), None, Some(false)]));
-        test_with_vector!(Int8Vector::from(vec![Some(i8::MIN), None, Some(i8::MAX)]));
-        test_with_vector!(Int16Vector::from(vec![
-            Some(i16::MIN),
-            None,
-            Some(i16::MAX)
-        ]));
-        test_with_vector!(Int32Vector::from(vec![
-            Some(i32::MIN),
-            None,
-            Some(i32::MAX)
-        ]));
-        test_with_vector!(Int64Vector::from(vec![
-            Some(i64::MIN),
-            None,
-            Some(i64::MAX)
-        ]));
-        test_with_vector!(UInt8Vector::from(vec![Some(u8::MIN), None, Some(u8::MAX)]));
-        test_with_vector!(UInt16Vector::from(vec![
-            Some(u16::MIN),
-            None,
-            Some(u16::MAX)
-        ]));
-        test_with_vector!(UInt32Vector::from(vec![
-            Some(u32::MIN),
-            None,
-            Some(u32::MAX)
-        ]));
-        test_with_vector!(UInt64Vector::from(vec![
-            Some(u64::MIN),
-            None,
-            Some(u64::MAX)
-        ]));
-        test_with_vector!(Float32Vector::from(vec![
-            Some(f32::MIN),
-            None,
-            Some(f32::MAX)
-        ]));
-        test_with_vector!(Float64Vector::from(vec![
-            Some(f64::MIN),
-            None,
-            Some(f64::MAX)
-        ]));
-        test_with_vector!(BinaryVector::from(vec![
-            Some(b"".to_vec()),
-            None,
-            Some(b"hello".to_vec())
-        ]));
-        test_with_vector!(StringVector::from(vec![Some(""), None, Some("foo"),]));
-        test_with_vector!(DateVector::from(vec![Some(1), None, Some(3)]));
-        test_with_vector!(DateTimeVector::from(vec![Some(4), None, Some(6)]));
-    }
-
-    fn create_test_column(vector: VectorRef) -> Column {
-        let wrapper: ColumnDataTypeWrapper = vector.data_type().try_into().unwrap();
-        Column {
-            column_name: "test".to_string(),
-            semantic_type: 1,
-            values: Some(values(&[vector.clone()]).unwrap()),
-            null_mask: null_mask(&[vector.clone()], vector.len()),
-            datatype: wrapper.datatype() as i32,
-        }
-    }
-
     #[test]
     fn test_flight_ctx() {
         let mut ctx = FlightContext::default();

src/client/src/error.rs

@@ -13,34 +13,41 @@
 // limitations under the License.
 
 use std::any::Any;
-use std::str::FromStr;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_error::{INNER_ERROR_CODE, INNER_ERROR_MSG};
+use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
+use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tonic::{Code, Status};
 
-#[derive(Debug, Snafu)]
+#[derive(Snafu)]
 #[snafu(visibility(pub))]
+#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Illegal Flight messages, reason: {}", reason))]
     IllegalFlightMessages { reason: String, location: Location },
 
-    #[snafu(display("Failed to do Flight get, code: {}, source: {}", tonic_code, source))]
+    #[snafu(display("Failed to do Flight get, code: {}", tonic_code))]
     FlightGet {
         addr: String,
         tonic_code: Code,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to convert FlightData, source: {}", source))]
+    #[snafu(display("Failure occurs during handling request"))]
+    HandleRequest {
+        location: Location,
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to convert FlightData"))]
     ConvertFlightData {
         location: Location,
         source: common_grpc::Error,
     },
 
-    #[snafu(display("Column datatype error, source: {}", source))]
+    #[snafu(display("Column datatype error"))]
     ColumnDataType {
         location: Location,
         source: api::error::Error,
@@ -52,17 +59,16 @@ pub enum Error {
     #[snafu(display("Missing required field in protobuf, field: {}", field))]
     MissingField { field: String, location: Location },
 
-    #[snafu(display(
-        "Failed to create gRPC channel, peer address: {}, source: {}",
-        addr,
-        source
-    ))]
+    #[snafu(display("Failed to create gRPC channel, peer address: {}", addr))]
     CreateChannel {
         addr: String,
         location: Location,
         source: common_grpc::error::Error,
     },
 
+    #[snafu(display("Failed to request RegionServer, code: {}", code))]
+    RegionServer { code: Code, source: BoxedError },
+
     // Server error carried in Tonic Status's metadata.
     #[snafu(display("{}", msg))]
     Server { code: StatusCode, msg: String },
@@ -86,7 +92,9 @@ impl ErrorExt for Error {
             | Error::ClientStreaming { .. } => StatusCode::Internal,
 
             Error::Server { code, .. } => *code,
-            Error::FlightGet { source, .. } => source.status_code(),
+            Error::FlightGet { source, .. }
+            | Error::HandleRequest { source, .. }
+            | Error::RegionServer { source, .. } => source.status_code(),
             Error::CreateChannel { source, .. } | Error::ConvertFlightData { source, .. } => {
                 source.status_code()
             }
@@ -107,11 +115,18 @@ impl From<Status> for Error {
                 .and_then(|v| String::from_utf8(v.as_bytes().to_vec()).ok())
         }
 
-        let code = get_metadata_value(&e, INNER_ERROR_CODE)
-            .and_then(|s| StatusCode::from_str(&s).ok())
+        let code = get_metadata_value(&e, GREPTIME_ERROR_CODE)
+            .and_then(|s| {
+                if let Ok(code) = s.parse::<u32>() {
+                    StatusCode::from_u32(code)
+                } else {
+                    None
+                }
+            })
             .unwrap_or(StatusCode::Unknown);
 
-        let msg = get_metadata_value(&e, INNER_ERROR_MSG).unwrap_or(e.to_string());
+        let msg =
+            get_metadata_value(&e, GREPTIME_ERROR_MSG).unwrap_or_else(|| e.message().to_string());
 
         Self::Server { code, msg }
     }

src/client/src/lib.rs

@@ -18,6 +18,7 @@ mod database;
 pub mod error;
 pub mod load_balance;
 mod metrics;
+pub mod region;
 mod stream_insert;
 
 pub use api;
@@ -25,6 +26,8 @@ use api::v1::greptime_response::Response;
 use api::v1::{AffectedRows, GreptimeResponse};
 pub use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_error::status_code::StatusCode;
+pub use common_query::Output;
+pub use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
 use snafu::OptionExt;
 
 pub use self::client::Client;

src/client/src/metrics.rs

@@ -21,7 +21,6 @@ pub const METRIC_GRPC_SQL: &str = "grpc.sql";
 pub const METRIC_GRPC_LOGICAL_PLAN: &str = "grpc.logical_plan";
 pub const METRIC_GRPC_ALTER: &str = "grpc.alter";
 pub const METRIC_GRPC_DROP_TABLE: &str = "grpc.drop_table";
-pub const METRIC_GRPC_FLUSH_TABLE: &str = "grpc.flush_table";
-pub const METRIC_GRPC_COMPACT_TABLE: &str = "grpc.compact_table";
 pub const METRIC_GRPC_TRUNCATE_TABLE: &str = "grpc.truncate_table";
 pub const METRIC_GRPC_DO_GET: &str = "grpc.do_get";
+pub(crate) const METRIC_REGION_REQUEST_GRPC: &str = "grpc.region_request";

src/client/src/region.rs

@@ -0,0 +1,264 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::region::{QueryRequest, RegionRequest, RegionResponse};
+use api::v1::ResponseHeader;
+use arrow_flight::Ticket;
+use async_stream::stream;
+use async_trait::async_trait;
+use common_error::ext::{BoxedError, ErrorExt};
+use common_error::status_code::StatusCode;
+use common_grpc::flight::{FlightDecoder, FlightMessage};
+use common_meta::datanode_manager::{AffectedRows, Datanode};
+use common_meta::error::{self as meta_error, Result as MetaResult};
+use common_recordbatch::error::ExternalSnafu;
+use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
+use common_telemetry::{error, timer};
+use prost::Message;
+use snafu::{location, Location, OptionExt, ResultExt};
+use tokio_stream::StreamExt;
+
+use crate::error::Error::RegionServer;
+use crate::error::{
+    self, ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
+    MissingFieldSnafu, Result, ServerSnafu,
+};
+use crate::{metrics, Client, Error};
+
+#[derive(Debug)]
+pub struct RegionRequester {
+    client: Client,
+}
+
+#[async_trait]
+impl Datanode for RegionRequester {
+    async fn handle(&self, request: RegionRequest) -> MetaResult<AffectedRows> {
+        self.handle_inner(request).await.map_err(|err| {
+            if matches!(err, RegionServer { .. }) {
+                meta_error::Error::RetryLater {
+                    source: BoxedError::new(err),
+                }
+            } else {
+                meta_error::Error::External {
+                    source: BoxedError::new(err),
+                    location: location!(),
+                }
+            }
+        })
+    }
+
+    async fn handle_query(&self, request: QueryRequest) -> MetaResult<SendableRecordBatchStream> {
+        let ticket = Ticket {
+            ticket: request.encode_to_vec().into(),
+        };
+        self.do_get_inner(ticket)
+            .await
+            .map_err(BoxedError::new)
+            .context(meta_error::ExternalSnafu)
+    }
+}
+
+impl RegionRequester {
+    pub fn new(client: Client) -> Self {
+        Self { client }
+    }
+
+    pub async fn do_get_inner(&self, ticket: Ticket) -> Result<SendableRecordBatchStream> {
+        let mut flight_client = self.client.make_flight_client()?;
+        let response = flight_client
+            .mut_inner()
+            .do_get(ticket)
+            .await
+            .map_err(|e| {
+                let tonic_code = e.code();
+                let e: error::Error = e.into();
+                let code = e.status_code();
+                let msg = e.to_string();
+                let error = Error::FlightGet {
+                    tonic_code,
+                    addr: flight_client.addr().to_string(),
+                    source: BoxedError::new(ServerSnafu { code, msg }.build()),
+                };
+                error!(
+                    e; "Failed to do Flight get, addr: {}, code: {}",
+                    flight_client.addr(),
+                    tonic_code
+                );
+                error
+            })?;
+
+        let flight_data_stream = response.into_inner();
+        let mut decoder = FlightDecoder::default();
+
+        let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
+            flight_data
+                .map_err(Error::from)
+                .and_then(|data| decoder.try_decode(data).context(ConvertFlightDataSnafu))
+        });
+
+        let Some(first_flight_message) = flight_message_stream.next().await else {
+            return IllegalFlightMessagesSnafu {
+                reason: "Expect the response not to be empty",
+            }
+            .fail();
+        };
+        let FlightMessage::Schema(schema) = first_flight_message? else {
+            return IllegalFlightMessagesSnafu {
+                reason: "Expect schema to be the first flight message",
+            }
+            .fail();
+        };
+
+        let stream = Box::pin(stream!({
+            while let Some(flight_message) = flight_message_stream.next().await {
+                let flight_message = flight_message
+                    .map_err(BoxedError::new)
+                    .context(ExternalSnafu)?;
+                let FlightMessage::Recordbatch(record_batch) = flight_message else {
+                    yield IllegalFlightMessagesSnafu {
+                            reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"
+                        }
+                        .fail()
+                        .map_err(BoxedError::new)
+                        .context(ExternalSnafu);
+                    break;
+                };
+                yield Ok(record_batch);
+            }
+        }));
+        let record_batch_stream = RecordBatchStreamAdaptor {
+            schema,
+            stream,
+            output_ordering: None,
+        };
+        Ok(Box::pin(record_batch_stream))
+    }
+
+    async fn handle_inner(&self, request: RegionRequest) -> Result<AffectedRows> {
+        let request_type = request
+            .body
+            .as_ref()
+            .with_context(|| MissingFieldSnafu { field: "body" })?
+            .as_ref()
+            .to_string();
+
+        let _timer = timer!(
+            metrics::METRIC_REGION_REQUEST_GRPC,
+            &[("request_type", request_type)]
+        );
+
+        let mut client = self.client.raw_region_client()?;
+
+        let RegionResponse {
+            header,
+            affected_rows,
+        } = client
+            .handle(request)
+            .await
+            .map_err(|e| {
+                let code = e.code();
+                let err: error::Error = e.into();
+                // Uses `Error::RegionServer` instead of `Error::Server`
+                error::Error::RegionServer {
+                    code,
+                    source: BoxedError::new(err),
+                }
+            })?
+            .into_inner();
+
+        check_response_header(header)?;
+
+        Ok(affected_rows)
+    }
+
+    pub async fn handle(&self, request: RegionRequest) -> Result<AffectedRows> {
+        self.handle_inner(request).await
+    }
+}
+
+pub fn check_response_header(header: Option<ResponseHeader>) -> Result<()> {
+    let status = header
+        .and_then(|header| header.status)
+        .context(IllegalDatabaseResponseSnafu {
+            err_msg: "either response header or status is missing",
+        })?;
+
+    if StatusCode::is_success(status.status_code) {
+        Ok(())
+    } else {
+        let code =
+            StatusCode::from_u32(status.status_code).context(IllegalDatabaseResponseSnafu {
+                err_msg: format!("unknown server status: {:?}", status),
+            })?;
+        ServerSnafu {
+            code,
+            msg: status.err_msg,
+        }
+        .fail()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use api::v1::Status as PbStatus;
+
+    use super::*;
+    use crate::Error::{IllegalDatabaseResponse, Server};
+
+    #[test]
+    fn test_check_response_header() {
+        let result = check_response_header(None);
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader { status: None }));
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: StatusCode::Success as u32,
+                err_msg: "".to_string(),
+            }),
+        }));
+        assert!(result.is_ok());
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: u32::MAX,
+                err_msg: "".to_string(),
+            }),
+        }));
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: StatusCode::Internal as u32,
+                err_msg: "blabla".to_string(),
+            }),
+        }));
+        let Server { code, msg } = result.unwrap_err() else {
+            unreachable!()
+        };
+        assert_eq!(code, StatusCode::Internal);
+        assert_eq!(msg, "blabla");
+    }
+}

src/client/src/stream_insert.rs

@@ -16,6 +16,7 @@ use api::v1::greptime_database_client::GreptimeDatabaseClient;
 use api::v1::greptime_request::Request;
 use api::v1::{
     AuthHeader, GreptimeRequest, GreptimeResponse, InsertRequest, InsertRequests, RequestHeader,
+    RowInsertRequest, RowInsertRequests,
 };
 use tokio::sync::mpsc;
 use tokio::task::JoinHandle;
@@ -84,6 +85,18 @@ impl StreamInserter {
         })
     }
 
+    pub async fn row_insert(&self, requests: Vec<RowInsertRequest>) -> Result<()> {
+        let inserts = RowInsertRequests { inserts: requests };
+        let request = self.to_rpc_request(Request::RowInserts(inserts));
+
+        self.sender.send(request).await.map_err(|e| {
+            error::ClientStreamingSnafu {
+                err_msg: e.to_string(),
+            }
+            .build()
+        })
+    }
+
     pub async fn finish(self) -> Result<u32> {
         drop(self.sender);
 

src/cmd/Cargo.toml

@@ -17,13 +17,18 @@ metrics-process = ["servers/metrics-process"]
 [dependencies]
 anymap = "1.0.0-beta.2"
 async-trait.workspace = true
+auth.workspace = true
 catalog = { workspace = true }
 chrono.workspace = true
 clap = { version = "3.1", features = ["derive"] }
 client = { workspace = true }
 common-base = { workspace = true }
+common-catalog = { workspace = true }
+common-config = { workspace = true }
 common-error = { workspace = true }
+common-macro = { workspace = true }
 common-meta = { workspace = true }
+common-procedure = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
 common-telemetry = { workspace = true, features = [
@@ -34,17 +39,24 @@ datanode = { workspace = true }
 datatypes = { workspace = true }
 either = "1.8"
 etcd-client.workspace = true
+file-engine = { workspace = true }
 frontend = { workspace = true }
 futures.workspace = true
+lazy_static.workspace = true
 meta-client = { workspace = true }
 meta-srv = { workspace = true }
 metrics.workspace = true
+mito2 = { workspace = true }
 nu-ansi-term = "0.46"
 partition = { workspace = true }
+plugins.workspace = true
+prost.workspace = true
 query = { workspace = true }
 rand.workspace = true
+regex.workspace = true
 rustyline = "10.1"
 serde.workspace = true
+serde_json.workspace = true
 servers = { workspace = true }
 session = { workspace = true }
 snafu.workspace = true

src/cmd/src/bin/greptime.rs

@@ -116,7 +116,7 @@ impl SubCommand {
                 Ok(Application::Metasrv(app))
             }
             (SubCommand::Standalone(cmd), Options::Standalone(opts)) => {
-                let app = cmd.build(opts.fe_opts, opts.dn_opts).await?;
+                let app = cmd.build(*opts).await?;
                 Ok(Application::Standalone(app))
             }
             (SubCommand::Cli(cmd), Options::Cli(_)) => {

src/cmd/src/cli.rs

@@ -16,6 +16,8 @@ mod bench;
 mod cmd;
 mod helper;
 mod repl;
+// TODO(weny): Removes it
+#[allow(deprecated)]
 mod upgrade;
 
 use async_trait::async_trait;
@@ -76,7 +78,7 @@ impl Command {
 
 #[derive(Parser)]
 enum SubCommand {
-    Attach(AttachCommand),
+    // Attach(AttachCommand),
     Upgrade(UpgradeCommand),
     Bench(BenchTableMetadataCommand),
 }
@@ -84,7 +86,7 @@ enum SubCommand {
 impl SubCommand {
     async fn build(self) -> Result<Instance> {
         match self {
-            SubCommand::Attach(cmd) => cmd.build().await,
+            // SubCommand::Attach(cmd) => cmd.build().await,
             SubCommand::Upgrade(cmd) => cmd.build().await,
             SubCommand::Bench(cmd) => cmd.build().await,
         }
@@ -102,51 +104,9 @@ pub(crate) struct AttachCommand {
 }
 
 impl AttachCommand {
+    #[allow(dead_code)]
     async fn build(self) -> Result<Instance> {
         let repl = Repl::try_new(&self).await?;
         Ok(Instance::Repl(repl))
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_load_options() {
-        let cmd = Command {
-            cmd: SubCommand::Attach(AttachCommand {
-                grpc_addr: String::from(""),
-                meta_addr: None,
-                disable_helper: false,
-            }),
-        };
-
-        let opts = cmd.load_options(TopLevelOptions::default()).unwrap();
-        let logging_opts = opts.logging_options();
-        assert_eq!("/tmp/greptimedb/logs", logging_opts.dir);
-        assert!(logging_opts.level.is_none());
-        assert!(!logging_opts.enable_jaeger_tracing);
-    }
-
-    #[test]
-    fn test_top_level_options() {
-        let cmd = Command {
-            cmd: SubCommand::Attach(AttachCommand {
-                grpc_addr: String::from(""),
-                meta_addr: None,
-                disable_helper: false,
-            }),
-        };
-
-        let opts = cmd
-            .load_options(TopLevelOptions {
-                log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
-                log_level: Some("debug".to_string()),
-            })
-            .unwrap();
-        let logging_opts = opts.logging_options();
-        assert_eq!("/tmp/greptimedb/test/logs", logging_opts.dir);
-        assert_eq!("debug", logging_opts.level.as_ref().unwrap());
-    }
-}

src/cmd/src/cli/bench.rs

@@ -12,53 +12,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod datanode_table;
-mod table_info;
-mod table_name;
-mod table_region;
-
+use std::collections::BTreeMap;
 use std::future::Future;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Duration;
 
 use async_trait::async_trait;
 use clap::Parser;
-use common_meta::key::table_region::RegionDistribution;
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
+use common_meta::peer::Peer;
+use common_meta::rpc::router::{Region, RegionRoute};
 use common_meta::table_name::TableName;
 use common_telemetry::info;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use meta_srv::service::store::etcd::EtcdStore;
 use meta_srv::service::store::kv::KvBackendAdapter;
-use rand::prelude::SliceRandom;
+use rand::Rng;
 use table::metadata::{RawTableInfo, RawTableMeta, TableId, TableIdent, TableType};
 
-use crate::cli::bench::datanode_table::DatanodeTableBencher;
-use crate::cli::bench::table_info::TableInfoBencher;
-use crate::cli::bench::table_name::TableNameBencher;
-use crate::cli::bench::table_region::TableRegionBencher;
+use self::metadata::TableMetadataBencher;
 use crate::cli::{Instance, Tool};
 use crate::error::Result;
 
-async fn bench<F, Fut>(desc: &str, f: F, count: u32)
-where
-    F: Fn(u32) -> Fut,
-    Fut: Future<Output = ()>,
-{
-    let mut total = Duration::default();
-
-    for i in 1..=count {
-        let start = Instant::now();
-
-        f(i).await;
-
-        total += start.elapsed();
-    }
-
-    let cost = total.as_millis() as f64 / count as f64;
-    info!("{desc}, average operation cost: {cost:.2} ms");
-}
+mod metadata;
 
 async fn bench_self_recorded<F, Fut>(desc: &str, f: F, count: u32)
 where
@@ -107,31 +84,11 @@ struct BenchTableMetadata {
 #[async_trait]
 impl Tool for BenchTableMetadata {
     async fn do_work(&self) -> Result<()> {
-        info!("Start benching table name manager ...");
-        TableNameBencher::new(self.table_metadata_manager.table_name_manager(), self.count)
-            .start()
-            .await;
-
-        info!("Start benching table info manager ...");
-        TableInfoBencher::new(self.table_metadata_manager.table_info_manager(), self.count)
-            .start()
-            .await;
-
-        info!("Start benching table region manager ...");
-        TableRegionBencher::new(
-            self.table_metadata_manager.table_region_manager(),
-            self.count,
-        )
-        .start()
-        .await;
-
-        info!("Start benching datanode table manager ...");
-        DatanodeTableBencher::new(
-            self.table_metadata_manager.datanode_table_manager(),
-            self.count,
-        )
-        .start()
-        .await;
+        let bencher = TableMetadataBencher::new(self.table_metadata_manager.clone(), self.count);
+        bencher.bench_create().await;
+        bencher.bench_get().await;
+        bencher.bench_rename().await;
+        bencher.bench_delete().await;
         Ok(())
     }
 }
@@ -163,7 +120,6 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
         created_on: chrono::DateTime::default(),
         primary_key_indices: vec![],
         next_column_id: columns as u32 + 1,
-        engine_options: Default::default(),
         value_indices: vec![],
         options: Default::default(),
         region_numbers: (1..=100).collect(),
@@ -184,16 +140,25 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
     }
 }
 
-fn create_region_distribution() -> RegionDistribution {
-    let mut regions = (1..=100).collect::<Vec<u32>>();
-    regions.shuffle(&mut rand::thread_rng());
+fn create_region_routes() -> Vec<RegionRoute> {
+    let mut regions = Vec::with_capacity(100);
+    let mut rng = rand::thread_rng();
 
-    let mut region_distribution = RegionDistribution::new();
-    for datanode_id in 0..10 {
-        region_distribution.insert(
-            datanode_id as u64,
-            regions[datanode_id * 10..(datanode_id + 1) * 10].to_vec(),
-        );
+    for region_id in 0..64u64 {
+        regions.push(RegionRoute {
+            region: Region {
+                id: region_id.into(),
+                name: String::new(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer {
+                id: rng.gen_range(0..10),
+                addr: String::new(),
+            }),
+            follower_peers: vec![],
+        });
     }
-    region_distribution
+
+    regions
 }

src/cmd/src/cli/bench/datanode_table.rs

@@ -1,131 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableManager};
-
-use super::bench;
-
-pub struct DatanodeTableBencher<'a> {
-    datanode_table_manager: &'a DatanodeTableManager,
-    count: u32,
-}
-
-impl<'a> DatanodeTableBencher<'a> {
-    pub fn new(datanode_table_manager: &'a DatanodeTableManager, count: u32) -> Self {
-        Self {
-            datanode_table_manager,
-            count,
-        }
-    }
-
-    pub async fn start(&self) {
-        self.bench_create().await;
-        self.bench_get().await;
-        self.bench_move_region().await;
-        self.bench_tables().await;
-        self.bench_remove().await;
-    }
-
-    async fn bench_create(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: create {} datanode table keys",
-            self.count
-        );
-        bench(
-            &desc,
-            |i| async move {
-                self.datanode_table_manager
-                    .create(1, i, vec![1, 2, 3, 4])
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_get(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: get {} datanode table keys",
-            self.count
-        );
-        bench(
-            &desc,
-            |i| async move {
-                let key = DatanodeTableKey::new(1, i);
-                assert!(self
-                    .datanode_table_manager
-                    .get(&key)
-                    .await
-                    .unwrap()
-                    .is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_move_region(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: move {} datanode table regions",
-            self.count
-        );
-        bench(
-            &desc,
-            |i| async move {
-                self.datanode_table_manager
-                    .move_region(1, 2, i, 1)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_tables(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: list {} datanode table keys",
-            self.count
-        );
-        bench(
-            &desc,
-            |_| async move {
-                assert!(!self
-                    .datanode_table_manager
-                    .tables(1)
-                    .await
-                    .unwrap()
-                    .is_empty());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_remove(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: remove {} datanode table keys",
-            self.count
-        );
-        bench(
-            &desc,
-            |i| async move {
-                self.datanode_table_manager.remove(1, i).await.unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-}

src/cmd/src/cli/bench/metadata.rs

@@ -0,0 +1,136 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Instant;
+
+use common_meta::key::TableMetadataManagerRef;
+use common_meta::table_name::TableName;
+
+use super::{bench_self_recorded, create_region_routes, create_table_info};
+
+pub struct TableMetadataBencher {
+    table_metadata_manager: TableMetadataManagerRef,
+    count: u32,
+}
+
+impl TableMetadataBencher {
+    pub fn new(table_metadata_manager: TableMetadataManagerRef, count: u32) -> Self {
+        Self {
+            table_metadata_manager,
+            count,
+        }
+    }
+
+    pub async fn bench_create(&self) {
+        let desc = format!(
+            "TableMetadataBencher: creating {} table metadata",
+            self.count
+        );
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let table_name = format!("bench_table_name_{}", i);
+                let table_name = TableName::new("bench_catalog", "bench_schema", table_name);
+                let table_info = create_table_info(i, table_name);
+                let region_routes = create_region_routes();
+
+                let start = Instant::now();
+
+                self.table_metadata_manager
+                    .create_table_metadata(table_info, region_routes)
+                    .await
+                    .unwrap();
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    pub async fn bench_get(&self) {
+        let desc = format!(
+            "TableMetadataBencher: getting {} table info and region routes",
+            self.count
+        );
+
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let start = Instant::now();
+                self.table_metadata_manager
+                    .get_full_table_info(i)
+                    .await
+                    .unwrap();
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    pub async fn bench_delete(&self) {
+        let desc = format!(
+            "TableMetadataBencher: deleting {} table metadata",
+            self.count
+        );
+
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let (table_info, table_route) = self
+                    .table_metadata_manager
+                    .get_full_table_info(i)
+                    .await
+                    .unwrap();
+                let start = Instant::now();
+                let _ = self
+                    .table_metadata_manager
+                    .delete_table_metadata(&table_info.unwrap(), &table_route.unwrap())
+                    .await;
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    pub async fn bench_rename(&self) {
+        let desc = format!("TableMetadataBencher: renaming {} table", self.count);
+
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let (table_info, _) = self
+                    .table_metadata_manager
+                    .get_full_table_info(i)
+                    .await
+                    .unwrap();
+
+                let new_table_name = format!("renamed_{}", i);
+
+                let start = Instant::now();
+                let _ = self
+                    .table_metadata_manager
+                    .rename_table(table_info.unwrap(), new_table_name)
+                    .await;
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+}

src/cmd/src/cli/bench/table_info.rs

@@ -1,111 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::Instant;
-
-use common_meta::key::table_info::TableInfoManager;
-use common_meta::table_name::TableName;
-
-use super::{bench, bench_self_recorded, create_table_info};
-
-pub struct TableInfoBencher<'a> {
-    table_info_manager: &'a TableInfoManager,
-    count: u32,
-}
-
-impl<'a> TableInfoBencher<'a> {
-    pub fn new(table_info_manager: &'a TableInfoManager, count: u32) -> Self {
-        Self {
-            table_info_manager,
-            count,
-        }
-    }
-
-    pub async fn start(&self) {
-        self.bench_create().await;
-        self.bench_get().await;
-        self.bench_compare_and_put().await;
-        self.bench_remove().await;
-    }
-
-    async fn bench_create(&self) {
-        let desc = format!("TableInfoBencher: create {} table infos", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_{}", i);
-                let table_name = TableName::new("bench_catalog", "bench_schema", table_name);
-                let table_info = create_table_info(i, table_name);
-                self.table_info_manager
-                    .create(i, &table_info)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_get(&self) {
-        let desc = format!("TableInfoBencher: get {} table infos", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                assert!(self.table_info_manager.get(i).await.unwrap().is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_compare_and_put(&self) {
-        let desc = format!(
-            "TableInfoBencher: compare_and_put {} table infos",
-            self.count
-        );
-        bench_self_recorded(
-            &desc,
-            |i| async move {
-                let table_info_value = self.table_info_manager.get(i).await.unwrap().unwrap();
-
-                let mut new_table_info = table_info_value.table_info.clone();
-                new_table_info.ident.version += 1;
-
-                let start = Instant::now();
-
-                self.table_info_manager
-                    .compare_and_put(i, Some(table_info_value), new_table_info)
-                    .await
-                    .unwrap()
-                    .unwrap();
-
-                start.elapsed()
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_remove(&self) {
-        let desc = format!("TableInfoBencher: remove {} table infos", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                self.table_info_manager.remove(i).await.unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-}

src/cmd/src/cli/bench/table_name.rs

@@ -1,131 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_meta::key::table_name::{TableNameKey, TableNameManager};
-
-use super::bench;
-
-pub struct TableNameBencher<'a> {
-    table_name_manager: &'a TableNameManager,
-    count: u32,
-}
-
-impl<'a> TableNameBencher<'a> {
-    pub fn new(table_name_manager: &'a TableNameManager, count: u32) -> Self {
-        Self {
-            table_name_manager,
-            count,
-        }
-    }
-
-    pub async fn start(&self) {
-        self.bench_create().await;
-        self.bench_rename().await;
-        self.bench_get().await;
-        self.bench_tables().await;
-        self.bench_remove().await;
-    }
-
-    async fn bench_create(&self) {
-        let desc = format!("TableNameBencher: create {} table names", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_{}", i);
-                let table_name_key = create_table_name_key(&table_name);
-                self.table_name_manager
-                    .create(&table_name_key, i)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_rename(&self) {
-        let desc = format!("TableNameBencher: rename {} table names", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_{}", i);
-                let new_table_name = format!("bench_table_name_new_{}", i);
-                let table_name_key = create_table_name_key(&table_name);
-                self.table_name_manager
-                    .rename(table_name_key, i, &new_table_name)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_get(&self) {
-        let desc = format!("TableNameBencher: get {} table names", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_new_{}", i);
-                let table_name_key = create_table_name_key(&table_name);
-                assert!(self
-                    .table_name_manager
-                    .get(table_name_key)
-                    .await
-                    .unwrap()
-                    .is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_tables(&self) {
-        let desc = format!("TableNameBencher: list all {} table names", self.count);
-        bench(
-            &desc,
-            |_| async move {
-                assert!(!self
-                    .table_name_manager
-                    .tables("bench_catalog", "bench_schema")
-                    .await
-                    .unwrap()
-                    .is_empty());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_remove(&self) {
-        let desc = format!("TableNameBencher: remove {} table names", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_new_{}", i);
-                let table_name_key = create_table_name_key(&table_name);
-                self.table_name_manager
-                    .remove(table_name_key)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-}
-
-fn create_table_name_key(table_name: &str) -> TableNameKey {
-    TableNameKey::new("bench_catalog", "bench_schema", table_name)
-}

src/cmd/src/cli/bench/table_region.rs

@@ -1,112 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::Instant;
-
-use common_meta::key::table_region::TableRegionManager;
-
-use super::{bench, bench_self_recorded, create_region_distribution};
-
-pub struct TableRegionBencher<'a> {
-    table_region_manager: &'a TableRegionManager,
-    count: u32,
-}
-
-impl<'a> TableRegionBencher<'a> {
-    pub fn new(table_region_manager: &'a TableRegionManager, count: u32) -> Self {
-        Self {
-            table_region_manager,
-            count,
-        }
-    }
-
-    pub async fn start(&self) {
-        self.bench_create().await;
-        self.bench_get().await;
-        self.bench_compare_and_put().await;
-        self.bench_remove().await;
-    }
-
-    async fn bench_create(&self) {
-        let desc = format!("TableRegionBencher: create {} table regions", self.count);
-        bench_self_recorded(
-            &desc,
-            |i| async move {
-                let region_distribution = create_region_distribution();
-
-                let start = Instant::now();
-
-                self.table_region_manager
-                    .create(i, &region_distribution)
-                    .await
-                    .unwrap();
-
-                start.elapsed()
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_get(&self) {
-        let desc = format!("TableRegionBencher: get {} table regions", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                assert!(self.table_region_manager.get(i).await.unwrap().is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_compare_and_put(&self) {
-        let desc = format!(
-            "TableRegionBencher: compare_and_put {} table regions",
-            self.count
-        );
-        bench_self_recorded(
-            &desc,
-            |i| async move {
-                let table_region_value = self.table_region_manager.get(i).await.unwrap().unwrap();
-
-                let new_region_distribution = create_region_distribution();
-
-                let start = Instant::now();
-
-                self.table_region_manager
-                    .compare_and_put(i, Some(table_region_value), new_region_distribution)
-                    .await
-                    .unwrap()
-                    .unwrap();
-
-                start.elapsed()
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_remove(&self) {
-        let desc = format!("TableRegionBencher: remove {} table regions", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                assert!(self.table_region_manager.remove(i).await.unwrap().is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-}

src/cmd/src/cli/repl.rs

@@ -16,20 +16,16 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
 
-use catalog::remote::CachedMetaKvBackend;
+use catalog::kvbackend::{CachedMetaKvBackend, KvBackendCatalogManager};
 use client::client_manager::DatanodeClients;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_base::Plugins;
 use common_error::ext::ErrorExt;
-use common_meta::key::TableMetadataManager;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
 use common_telemetry::logging;
 use either::Either;
-use frontend::catalog::FrontendCatalogManager;
 use meta_client::client::MetaClientBuilder;
-use partition::manager::PartitionRuleManager;
-use partition::route::TableRoutes;
 use query::datafusion::DatafusionQueryEngine;
 use query::logical_optimizer::LogicalOptimizer;
 use query::parser::QueryLanguageParser;
@@ -39,7 +35,7 @@ use query::QueryEngine;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 use session::context::QueryContext;
-use snafu::{ErrorCompat, ResultExt};
+use snafu::ResultExt;
 use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 
 use crate::cli::cmd::ReplCommand;
@@ -152,7 +148,7 @@ impl Repl {
             .await
             .map_err(|e| {
                 let status_code = e.status_code();
-                let root_cause = e.iter_chain().last().unwrap();
+                let root_cause = e.output_msg();
                 println!("Error: {}({status_code}), {root_cause}", status_code as u32)
             })
             .is_ok()
@@ -180,7 +176,7 @@ impl Repl {
                 .encode(&plan)
                 .context(SubstraitEncodeLogicalPlanSnafu)?;
 
-            self.database.logical_plan(plan.to_vec(), None).await
+            self.database.logical_plan(plan.to_vec(), 0).await
         } else {
             self.database.sql(&sql).await
         }
@@ -254,24 +250,19 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
 
     let cached_meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
 
-    let table_routes = Arc::new(TableRoutes::new(meta_client));
-    let partition_manager = Arc::new(PartitionRuleManager::new(table_routes));
-
     let datanode_clients = Arc::new(DatanodeClients::default());
 
-    let catalog_list = Arc::new(FrontendCatalogManager::new(
+    let catalog_list = KvBackendCatalogManager::new(
         cached_meta_backend.clone(),
         cached_meta_backend.clone(),
-        partition_manager,
         datanode_clients,
-        Arc::new(TableMetadataManager::new(cached_meta_backend)),
-    ));
-    let plugins: Arc<Plugins> = Default::default();
+    );
+    let plugins: Plugins = Default::default();
     let state = Arc::new(QueryEngineState::new(
         catalog_list,
+        None,
+        None,
         false,
-        None,
-        None,
         plugins.clone(),
     ));