test: more reduce tests

.github/actions/build-dev-builder-image/action.yml

@@ -0,0 +1,93 @@
+name: Build and push dev-builder image
+description: Build and push dev-builder image to DockerHub and ACR
+inputs:
+  dockerhub-image-registry:
+    description: The dockerhub image registry to store the images
+    required: false
+    default: docker.io
+  dockerhub-image-registry-username:
+    description: The dockerhub username to login to the image registry
+    required: true
+  dockerhub-image-registry-token:
+    description: The dockerhub token to login to the image registry
+    required: true
+  dockerhub-image-namespace:
+    description: The dockerhub namespace of the image registry to store the images
+    required: false
+    default: greptime
+  acr-image-registry:
+    description: The ACR image registry to store the images
+    required: true
+  acr-image-registry-username:
+    description: The ACR username to login to the image registry
+    required: true
+  acr-image-registry-password:
+    description: The ACR password to login to the image registry
+    required: true
+  acr-image-namespace:
+    description: The ACR namespace of the image registry to store the images
+    required: false
+    default: greptime
+  version:
+    description: Version of the dev-builder
+    required: false
+    default: latest
+runs:
+  using: composite
+  steps:
+    - name: Login to Dockerhub
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.dockerhub-image-registry }}
+        username: ${{ inputs.dockerhub-image-registry-username }}
+        password: ${{ inputs.dockerhub-image-registry-token }}
+
+    - name: Build and push ubuntu dev builder image to dockerhub
+      shell: bash
+      run:
+        make dev-builder \
+        BASE_IMAGE=ubuntu \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push centos dev builder image to dockerhub
+      shell: bash
+      run:
+        make dev-builder \
+        BASE_IMAGE=centos \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Login to ACR
+      uses: docker/login-action@v2
+      continue-on-error: true
+      with:
+        registry: ${{ inputs.acr-image-registry }}
+        username: ${{ inputs.acr-image-registry-username }}
+        password: ${{ inputs.acr-image-registry-password }}
+
+    - name: Build and push ubuntu dev builder image to ACR
+      shell: bash
+      continue-on-error: true
+      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
+        make dev-builder \
+        BASE_IMAGE=ubuntu \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push centos dev builder image to ACR
+      shell: bash
+      continue-on-error: true
+      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
+        make dev-builder \
+        BASE_IMAGE=centos \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}

.github/actions/build-dev-builder-images/action.yml

@@ -1,76 +0,0 @@
-name: Build and push dev-builder images
-description: Build and push dev-builder images to DockerHub and ACR
-inputs:
-  dockerhub-image-registry:
-    description: The dockerhub image registry to store the images
-    required: false
-    default: docker.io
-  dockerhub-image-registry-username:
-    description: The dockerhub username to login to the image registry
-    required: true
-  dockerhub-image-registry-token:
-    description: The dockerhub token to login to the image registry
-    required: true
-  dockerhub-image-namespace:
-    description: The dockerhub namespace of the image registry to store the images
-    required: false
-    default: greptime
-  version:
-    description: Version of the dev-builder
-    required: false
-    default: latest
-  build-dev-builder-ubuntu:
-    description: Build dev-builder-ubuntu image
-    required: false
-    default: 'true'
-  build-dev-builder-centos:
-    description: Build dev-builder-centos image
-    required: false
-    default: 'true'
-  build-dev-builder-android:
-    description: Build dev-builder-android image
-    required: false
-    default: 'true'
-runs:
-  using: composite
-  steps:
-    - name: Login to Dockerhub
-      uses: docker/login-action@v2
-      with:
-        registry: ${{ inputs.dockerhub-image-registry }}
-        username: ${{ inputs.dockerhub-image-registry-username }}
-        password: ${{ inputs.dockerhub-image-registry-token }}
-
-    - name: Build and push dev-builder-ubuntu image
-      shell: bash
-      if: ${{ inputs.build-dev-builder-ubuntu == 'true' }}
-      run: |
-        make dev-builder \
-          BASE_IMAGE=ubuntu \
-          BUILDX_MULTI_PLATFORM_BUILD=true \
-          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
-          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-          IMAGE_TAG=${{ inputs.version }}
-
-    - name: Build and push dev-builder-centos image
-      shell: bash
-      if: ${{ inputs.build-dev-builder-centos == 'true' }}
-      run: |
-        make dev-builder \
-          BASE_IMAGE=centos \
-          BUILDX_MULTI_PLATFORM_BUILD=true \
-          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
-          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-          IMAGE_TAG=${{ inputs.version }}
-
-    - name: Build and push dev-builder-android image # Only build image for amd64 platform.
-      shell: bash
-      if: ${{ inputs.build-dev-builder-android == 'true' }}
-      run: |
-        make dev-builder \
-          BASE_IMAGE=android \
-          IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
-          IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-          IMAGE_TAG=${{ inputs.version }} && \
-        
-        docker push ${{ inputs.dockerhub-image-registry }}/${{ inputs.dockerhub-image-namespace }}/dev-builder-android:${{ inputs.version }}

.github/actions/build-greptime-binary/action.yml

@@ -16,20 +16,35 @@ inputs:
   version:
     description: Version of the artifact
     required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
   working-dir:
     description: Working directory to build the artifacts
     required: false
     default: .
-  build-android-artifacts:
-    description: Build android artifacts
-    required: false
-    default: 'false'
 runs:
   using: composite
   steps:
     - name: Build greptime binary
       shell: bash
-      if: ${{ inputs.build-android-artifacts == 'false' }}
       run: |
         cd ${{ inputs.working-dir }} && \
         make build-by-dev-builder \
@@ -39,25 +54,14 @@ runs:
 
     - name: Upload artifacts
       uses: ./.github/actions/upload-artifacts
-      if: ${{ inputs.build-android-artifacts == 'false' }}
       with:
         artifacts-dir: ${{ inputs.artifacts-dir }}
         target-file: ./target/${{ inputs.cargo-profile }}/greptime
         version: ${{ inputs.version }}
-        working-dir: ${{ inputs.working-dir }}
-
-    # TODO(zyy17): We can remove build-android-artifacts flag in the future.
-    - name: Build greptime binary
-      shell: bash
-      if: ${{ inputs.build-android-artifacts == 'true' }}
-      run: |
-        cd ${{ inputs.working-dir }} && make strip-android-bin
-
-    - name: Upload android artifacts
-      uses: ./.github/actions/upload-artifacts
-      if: ${{ inputs.build-android-artifacts == 'true' }}
-      with:
-        artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: ./target/aarch64-linux-android/release/greptime
-        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
         working-dir: ${{ inputs.working-dir }}

.github/actions/build-linux-artifacts/action.yml

@@ -13,10 +13,30 @@ inputs:
   disable-run-tests:
     description: Disable running integration tests
     required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
   dev-mode:
     description: Enable dev mode, only build standard greptime
     required: false
     default: 'false'
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
   working-dir:
     description: Working directory to build the artifacts
     required: false
@@ -48,6 +68,12 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }}
         version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
         working-dir: ${{ inputs.working-dir }}
 
     - name: Build greptime without pyo3
@@ -59,6 +85,12 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
         version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
         working-dir: ${{ inputs.working-dir }}
 
     - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build.
@@ -75,14 +107,10 @@ runs:
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
         version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
         working-dir: ${{ inputs.working-dir }}
-
-    - name: Build greptime on android base image
-      uses: ./.github/actions/build-greptime-binary
-      if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Only build android base image on amd64.
-      with:
-        base-image: android
-        artifacts-dir: greptime-android-arm64-${{ inputs.version }}
-        version: ${{ inputs.version }}
-        working-dir: ${{ inputs.working-dir }}
-        build-android-artifacts: true

.github/actions/build-macos-artifacts/action.yml

@@ -19,9 +19,21 @@ inputs:
   disable-run-tests:
     description: Disable running integration tests
     required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
   artifacts-dir:
     description: Directory to store artifacts
     required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
 runs:
   using: composite
   steps:
@@ -87,3 +99,7 @@ runs:
         artifacts-dir: ${{ inputs.artifacts-dir }}
         target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
         version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}

.github/actions/build-windows-artifacts/action.yml

@@ -1,80 +0,0 @@
-name: Build Windows artifacts
-description: Build Windows artifacts
-inputs:
-  arch:
-    description: Architecture to build
-    required: true
-  rust-toolchain:
-    description: Rust toolchain to use
-    required: true
-  cargo-profile:
-    description: Cargo profile to build
-    required: true
-  features:
-    description: Cargo features to build
-    required: true
-  version:
-    description: Version of the artifact
-    required: true
-  disable-run-tests:
-    description: Disable running integration tests
-    required: true
-  artifacts-dir:
-    description: Directory to store artifacts
-    required: true
-runs:
-  using: composite
-  steps:
-    - uses: arduino/setup-protoc@v1
-
-    - name: Install rust toolchain
-      uses: dtolnay/rust-toolchain@master
-      with:
-        toolchain: ${{ inputs.rust-toolchain }}
-        targets: ${{ inputs.arch }}
-        components: llvm-tools-preview
-
-    - name: Rust Cache
-      uses: Swatinem/rust-cache@v2
-
-    - name: Install Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-
-    - name: Install PyArrow Package
-      shell: pwsh
-      run: pip install pyarrow
-
-    - name: Install WSL distribution
-      uses: Vampire/setup-wsl@v2
-      with:
-        distribution: Ubuntu-22.04
-
-    - name: Install latest nextest release # For integration tests.
-      if: ${{ inputs.disable-run-tests == 'false' }}
-      uses: taiki-e/install-action@nextest
-
-    - name: Run integration tests
-      if: ${{ inputs.disable-run-tests == 'false' }}
-      shell: pwsh
-      run: make test sqlness-test
-
-    - name: Upload sqlness logs
-      if: ${{ failure() }} # Only upload logs when the integration tests failed.
-      uses: actions/upload-artifact@v3
-      with:
-        name: sqlness-logs
-        path: ${{ runner.temp }}/greptime-*.log
-        retention-days: 3
-
-    - name: Build greptime binary
-      shell: pwsh
-      run: cargo build --profile ${{ inputs.cargo-profile }} --features ${{ inputs.features }} --target ${{ inputs.arch }}
-
-    - name: Upload artifacts
-      uses: ./.github/actions/upload-artifacts
-      with:
-        artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
-        version: ${{ inputs.version }}

.github/actions/release-artifacts/action.yml

@@ -1,5 +1,5 @@
-name: Publish GitHub release
-description: Publish GitHub release
+name: Release artifacts
+description: Release artifacts
 inputs:
   version:
     description: Version to release

.github/actions/release-cn-artifacts/action.yaml

@@ -1,138 +0,0 @@
-name: Release CN artifacts
-description: Release artifacts to CN region
-inputs:
-  src-image-registry:
-    description: The source image registry to store the images
-    required: true
-    default: docker.io
-  src-image-namespace:
-    description: The namespace of the source image registry to store the images
-    required: true
-    default: greptime
-  src-image-name:
-    description: The name of the source image
-    required: false
-    default: greptimedb
-  dst-image-registry:
-    description: The destination image registry to store the images
-    required: true
-  dst-image-namespace:
-    description: The namespace of the destination image registry to store the images
-    required: true
-    default: greptime
-  dst-image-registry-username:
-    description: The username to login to the image registry
-    required: true
-  dst-image-registry-password:
-    description: The password to login to the image registry
-    required: true
-  version:
-    description: Version of the artifact
-    required: true
-  dev-mode:
-    description: Enable dev mode, only push standard greptime
-    required: false
-    default: 'false'
-  push-latest-tag:
-    description: Whether to push the latest tag of the image
-    required: false
-    default: 'true'
-  aws-cn-s3-bucket:
-    description: S3 bucket to store released artifacts in CN region
-    required: true
-  aws-cn-access-key-id:
-    description: AWS access key id in CN region
-    required: true
-  aws-cn-secret-access-key:
-    description: AWS secret access key in CN region
-    required: true
-  aws-cn-region:
-    description: AWS region in CN
-    required: true
-  upload-to-s3:
-    description: Upload to S3
-    required: false
-    default: 'true'
-  artifacts-dir:
-    description: Directory to store artifacts
-    required: false
-    default: 'artifacts'
-  update-version-info:
-    description: Update the version info in S3
-    required: false
-    default: 'true'
-  upload-max-retry-times:
-    description: Max retry times for uploading artifacts to S3
-    required: false
-    default: "20"
-  upload-retry-timeout:
-    description: Timeout for uploading artifacts to S3
-    required: false
-    default: "30" # minutes
-runs:
-  using: composite
-  steps:
-    - name: Download artifacts
-      uses: actions/download-artifact@v3
-      with:
-        path: ${{ inputs.artifacts-dir }}
-
-    - name: Release artifacts to cn region
-      uses: nick-invision/retry@v2
-      if: ${{ inputs.upload-to-s3 == 'true' }}
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.aws-cn-access-key-id }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-cn-secret-access-key }}
-        AWS_DEFAULT_REGION: ${{ inputs.aws-cn-region }}
-        UPDATE_VERSION_INFO: ${{ inputs.update-version-info }}
-      with:
-        max_attempts: ${{ inputs.upload-max-retry-times }}
-        timeout_minutes: ${{ inputs.upload-retry-timeout }}
-        command: |
-          ./.github/scripts/upload-artifacts-to-s3.sh \
-            ${{ inputs.artifacts-dir }} \
-            ${{ inputs.version }} \
-            ${{ inputs.aws-cn-s3-bucket }}
-
-    - name: Push greptimedb image from Dockerhub to ACR
-      shell: bash
-      env:
-        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
-        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
-      run: |
-        ./.github/scripts/copy-image.sh \
-         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}:${{ inputs.version }} \
-         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
-
-    - name: Push latest greptimedb image from Dockerhub to ACR
-      shell: bash
-      if: ${{ inputs.push-latest-tag == 'true' }}
-      env:
-        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
-        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
-      run: |
-        ./.github/scripts/copy-image.sh \
-         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}:latest \
-         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
-
-    - name: Push greptimedb-centos image from DockerHub to ACR
-      shell: bash
-      if: ${{ inputs.dev-mode == 'false' }}
-      env:
-        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
-        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
-      run: |
-        ./.github/scripts/copy-image.sh \
-         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}-centos:latest \
-         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}
-
-    - name: Push greptimedb-centos image from DockerHub to ACR
-      shell: bash
-      if: ${{ inputs.dev-mode == 'false' && inputs.push-latest-tag == 'true' }}
-      env:
-        DST_REGISTRY_USERNAME: ${{ inputs.dst-image-registry-username }}
-        DST_REGISTRY_PASSWORD: ${{ inputs.dst-image-registry-password }}
-      run: |
-        ./.github/scripts/copy-image.sh \
-         ${{ inputs.src-image-registry }}/${{ inputs.src-image-namespace }}/${{ inputs.src-image-name }}-centos:latest \
-         ${{ inputs.dst-image-registry }}/${{ inputs.dst-image-namespace }}

.github/actions/upload-artifacts/action.yml

@@ -10,6 +10,34 @@ inputs:
   version:
     description: Version of the artifact
     required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
+  upload-max-retry-times:
+    description: Max retry times for uploading artifacts to S3
+    required: false
+    default: "20"
+  upload-retry-timeout:
+    description: Timeout for uploading artifacts to S3
+    required: false
+    default: "10" # minutes
   working-dir:
     description: Working directory to upload the artifacts
     required: false
@@ -33,21 +61,9 @@ runs:
       working-directory: ${{ inputs.working-dir }}
       shell: bash
       run: |
-        tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }}
-
-    - name: Calculate checksum
-      if: runner.os != 'Windows'
-      working-directory: ${{ inputs.working-dir }}
-      shell: bash
-      run: |
+        tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }} && \
         echo $(shasum -a 256 ${{ inputs.artifacts-dir }}.tar.gz | cut -f1 -d' ') > ${{ inputs.artifacts-dir }}.sha256sum
 
-    - name: Calculate checksum on Windows
-      if: runner.os == 'Windows'
-      working-directory: ${{ inputs.working-dir }}
-      shell: pwsh
-      run: Get-FileHash ${{ inputs.artifacts-dir }}.tar.gz -Algorithm SHA256 | select -ExpandProperty Hash > ${{ inputs.artifacts-dir }}.sha256sum
-
     # Note: The artifacts will be double zip compressed(related issue: https://github.com/actions/upload-artifact/issues/39).
     # However, when we use 'actions/download-artifact@v3' to download the artifacts, it will be automatically unzipped.
     - name: Upload artifacts
@@ -61,3 +77,49 @@ runs:
       with:
         name: ${{ inputs.artifacts-dir }}.sha256sum
         path: ${{ inputs.working-dir }}/${{ inputs.artifacts-dir }}.sha256sum
+
+    - name: Upload artifacts to S3
+      if: ${{ inputs.upload-to-s3 == 'true' }}
+      uses: nick-invision/retry@v2
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
+      with:
+        max_attempts: ${{ inputs.upload-max-retry-times }}
+        timeout_minutes: ${{ inputs.upload-retry-timeout }}
+        # The bucket layout will be:
+        # releases/greptimedb
+        # ├── v0.1.0
+        # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
+        # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
+        # └── v0.2.0
+        #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
+        #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
+        command: |
+          cd ${{ inputs.working-dir }} && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.tar.gz \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.tar.gz && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.sha256sum \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.sha256sum
+
+    - name: Upload latest artifacts to S3
+      if: ${{ inputs.upload-to-s3 == 'true' && inputs.upload-latest-artifacts == 'true' }} # We'll also upload the latest artifacts to S3 in the scheduled and formal release.
+      uses: nick-invision/retry@v2
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
+      with:
+        max_attempts: ${{ inputs.upload-max-retry-times }}
+        timeout_minutes: ${{ inputs.upload-retry-timeout }}
+        command: |
+          cd ${{ inputs.working-dir }} && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.tar.gz \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/latest/${{ inputs.artifacts-dir }}.tar.gz && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.sha256sum \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/latest/${{ inputs.artifacts-dir }}.sha256sum

.github/scripts/copy-image.sh

@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -o pipefail
-
-SRC_IMAGE=$1
-DST_REGISTRY=$2
-SKOPEO_STABLE_IMAGE="quay.io/skopeo/stable:latest"
-
-# Check if necessary variables are set.
-function check_vars() {
-  for var in DST_REGISTRY_USERNAME DST_REGISTRY_PASSWORD DST_REGISTRY SRC_IMAGE; do
-    if [ -z "${!var}" ]; then
-      echo "$var is not set or empty."
-      echo "Usage: DST_REGISTRY_USERNAME=<your-dst-registry-username> DST_REGISTRY_PASSWORD=<your-dst-registry-password> $0 <dst-registry> <src-image>"
-      exit 1
-    fi
-  done
-}
-
-# Copies images from DockerHub to the destination registry.
-function copy_images_from_dockerhub() {
-  # Check if docker is installed.
-  if ! command -v docker &> /dev/null; then
-    echo "docker is not installed. Please install docker to continue."
-    exit 1
-  fi
-
-  # Extract the name and tag of the source image.
-  IMAGE_NAME=$(echo "$SRC_IMAGE" | sed "s/.*\///")
-
-  echo "Copying $SRC_IMAGE to $DST_REGISTRY/$IMAGE_NAME"
-
-  docker run "$SKOPEO_STABLE_IMAGE" copy -a docker://"$SRC_IMAGE" \
-    --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
-    docker://"$DST_REGISTRY/$IMAGE_NAME"
-}
-
-function main() {
-  check_vars
-  copy_images_from_dockerhub
-}
-
-# Usage example:
-# DST_REGISTRY_USERNAME=123 DST_REGISTRY_PASSWORD=456 \
-#   ./copy-image.sh greptime/greptimedb:v0.4.0 greptime-registry.cn-hangzhou.cr.aliyuncs.com
-main

.github/scripts/upload-artifacts-to-s3.sh

@@ -1,102 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -o pipefail
-
-ARTIFACTS_DIR=$1
-VERSION=$2
-AWS_S3_BUCKET=$3
-RELEASE_DIRS="releases/greptimedb"
-GREPTIMEDB_REPO="GreptimeTeam/greptimedb"
-
-# Check if necessary variables are set.
-function check_vars() {
-  for var in AWS_S3_BUCKET VERSION ARTIFACTS_DIR; do
-    if [ -z "${!var}" ]; then
-      echo "$var is not set or empty."
-      echo "Usage: $0 <artifacts-dir> <version> <aws-s3-bucket>"
-      exit 1
-    fi
-  done
-}
-
-# Uploads artifacts to AWS S3 bucket.
-function upload_artifacts() {
-  # The bucket layout will be:
-  # releases/greptimedb
-  # ├── latest-version.txt
-  # ├── latest-nightly-version.txt
-  # ├── v0.1.0
-  # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
-  # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
-  # └── v0.2.0
-  #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
-  #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
-  find "$ARTIFACTS_DIR" -type f \( -name "*.tar.gz" -o -name "*.sha256sum" \) | while IFS= read -r file; do
-    aws s3 cp \
-      "$file" "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/$VERSION/$(basename "$file")"
-  done
-}
-
-# Updates the latest version information in AWS S3 if UPDATE_VERSION_INFO is true.
-function update_version_info() {
-  if [ "$UPDATE_VERSION_INFO" == "true" ]; then
-    # If it's the officail release(like v1.0.0, v1.0.1, v1.0.2, etc.), update latest-version.txt.
-    if [[ "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-      echo "Updating latest-version.txt"
-      echo "$VERSION" > latest-version.txt
-      aws s3 cp \
-        latest-version.txt "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/latest-version.txt"
-    fi
-
-    # If it's the nightly release, update latest-nightly-version.txt.
-    if [[ "$VERSION" == *"nightly"* ]]; then
-      echo "Updating latest-nightly-version.txt"
-      echo "$VERSION" > latest-nightly-version.txt
-      aws s3 cp \
-        latest-nightly-version.txt "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/latest-nightly-version.txt"
-    fi
-  fi
-}
-
-# Downloads artifacts from Github if DOWNLOAD_ARTIFACTS_FROM_GITHUB is true.
-function download_artifacts_from_github() {
-  if [ "$DOWNLOAD_ARTIFACTS_FROM_GITHUB" == "true" ]; then
-    # Check if jq is installed.
-    if ! command -v jq &> /dev/null; then
-      echo "jq is not installed. Please install jq to continue."
-      exit 1
-    fi
-
-    # Get the latest release API response.
-    RELEASES_API_RESPONSE=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/$GREPTIMEDB_REPO/releases/latest")
-
-    # Extract download URLs for the artifacts.
-    # Exclude source code archives which are typically named as 'greptimedb-<version>.zip' or 'greptimedb-<version>.tar.gz'.
-    ASSET_URLS=$(echo "$RELEASES_API_RESPONSE" | jq -r '.assets[] | select(.name | test("greptimedb-.*\\.(zip|tar\\.gz)$") | not) | .browser_download_url')
-
-    # Download each asset.
-    while IFS= read -r url; do
-      if [ -n "$url" ]; then
-        curl -LJO "$url"
-        echo "Downloaded: $url"
-      fi
-    done <<< "$ASSET_URLS"
-  fi
-}
-
-function main() {
-  check_vars
-  download_artifacts_from_github
-  upload_artifacts
-  update_version_info
-}
-
-# Usage example:
-#   AWS_ACCESS_KEY_ID=<your_access_key_id> \
-#   AWS_SECRET_ACCESS_KEY=<your_secret_access_key> \
-#   AWS_DEFAULT_REGION=<your_region> \
-#   UPDATE_VERSION_INFO=true \
-#   DOWNLOAD_ARTIFACTS_FROM_GITHUB=false \
-#     ./upload-artifacts-to-s3.sh <artifacts-dir> <version> <aws-s3-bucket>
-main

.github/workflows/apidoc.yml

@@ -17,7 +17,7 @@ env:
 
 jobs:
   apidoc:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
     - uses: arduino/setup-protoc@v1

.github/workflows/dev-build.yml

@@ -16,11 +16,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-20.04
-          - ubuntu-20.04-8-cores
-          - ubuntu-20.04-16-cores
-          - ubuntu-20.04-32-cores
-          - ubuntu-20.04-64-cores
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -78,7 +78,7 @@ jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -164,7 +164,12 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
           dev-mode: true # Only build the standard greptime binary.
+          upload-to-s3: false # No need to upload to S3.
           working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
 
   build-linux-arm64-artifacts:
@@ -193,7 +198,12 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
           dev-mode: true # Only build the standard greptime binary.
+          upload-to-s3: false # No need to upload to S3.
           working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
 
   release-images-to-dockerhub:
@@ -204,7 +214,7 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
       build-result: ${{ steps.set-build-result.outputs.build-result }}
     steps:
@@ -229,44 +239,41 @@ jobs:
         run: |
           echo "build-result=success" >> $GITHUB_OUTPUT
 
-  release-cn-artifacts:
-    name: Release artifacts to CN region
+  release-images-to-acr:
+    name: Build and push images to ACR
     if: ${{ inputs.release_images || github.event_name == 'schedule' }}
     needs: [
       allocate-runners,
-      release-images-to-dockerhub,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
+    # When we push to ACR, it's easy to fail due to some unknown network issues.
+    # However, we don't want to fail the whole workflow because of this.
+    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
     continue-on-error: true
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Release artifacts to CN region
-        uses: ./.github/actions/release-cn-artifacts
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
         with:
-          src-image-registry: docker.io
-          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          src-image-name: ${{ env.IMAGE_NAME }}
-          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
-          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-name: ${{ env.IMAGE_NAME }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          dev-mode: true                     # Only build the standard images(exclude centos images).
-          push-latest-tag: false             # Don't push the latest tag to registry.
-          update-version-info: false         # Don't update the version info in S3.
+          push-latest-tag: false # Don't push the latest tag to registry.
+          dev-mode: true # Only build the standard images.
 
   stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -291,7 +298,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -318,7 +325,7 @@ jobs:
     needs: [
       release-images-to-dockerhub
     ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:

.github/workflows/develop.yml

@@ -34,7 +34,7 @@ env:
 jobs:
   typos:
     name: Spell Check with Typos
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: crate-ci/typos@v1.13.10
@@ -42,7 +42,7 @@ jobs:
   check:
     name: Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -60,7 +60,7 @@ jobs:
   toml:
     name: Toml Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -80,7 +80,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04-8-cores ]
+        os: [ ubuntu-latest-8-cores, windows-latest-8-cores ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -105,7 +105,7 @@ jobs:
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -124,7 +124,7 @@ jobs:
   clippy:
     name: Clippy
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -142,7 +142,7 @@ jobs:
 
   coverage:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04-8-cores
+    runs-on: ubuntu-latest-8-cores
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -188,3 +188,43 @@ jobs:
           flags: rust
           fail_ci_if_error: false
           verbose: true
+
+  test-on-windows:
+    if: github.event.pull_request.draft == false
+    runs-on: windows-latest-8-cores
+    timeout-minutes: 60
+    steps:
+      - run: git config --global core.autocrlf false
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
+      - name: Install WSL distribution
+        uses: Vampire/setup-wsl@v2
+        with:
+          distribution: Ubuntu-22.04
+      - name: Running tests
+        run: cargo nextest run -F pyo3_backend,dashboard
+        env:
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          GT_S3_REGION: ${{ secrets.S3_REGION }}
+          UNITTEST_LOG_DIR: "__unittest_logs"

.github/workflows/doc-issue.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   doc_issue:
     if: github.event.label.name == 'doc update required'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: create an issue in doc repo
         uses: dacbd/create-issue-action@main
@@ -25,7 +25,7 @@ jobs:
             ${{ github.event.issue.html_url || github.event.pull_request.html_url }}
   cloud_issue:
     if: github.event.label.name == 'cloud followup required'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: create an issue in cloud repo
         uses: dacbd/create-issue-action@main

.github/workflows/docs.yml

@@ -30,7 +30,7 @@ name: CI
 jobs:
   typos:
     name: Spell Check with Typos
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: crate-ci/typos@v1.13.10
@@ -38,33 +38,33 @@ jobs:
   check:
     name: Check
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - run: 'echo "No action required"'
 
   fmt:
     name: Rustfmt
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - run: 'echo "No action required"'
 
   clippy:
     name: Clippy
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - run: 'echo "No action required"'
 
   coverage:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - run: 'echo "No action required"'
 
   sqlness:
     name: Sqlness Test
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - run: 'echo "No action required"'

.github/workflows/license.yaml

@@ -8,7 +8,7 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
 jobs:
   license-header-check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: license-header-check
     steps:
     - uses: actions/checkout@v2

.github/workflows/nightly-build.yml

@@ -14,11 +14,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.2xlarge-amd64
         options:
-          - ubuntu-20.04
-          - ubuntu-20.04-8-cores
-          - ubuntu-20.04-16-cores
-          - ubuntu-20.04-32-cores
-          - ubuntu-20.04-64-cores
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -70,7 +70,7 @@ jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
@@ -147,6 +147,11 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-latest-artifacts: false
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -166,6 +171,11 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-latest-artifacts: false
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub
@@ -175,7 +185,7 @@ jobs:
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
       nightly-build-result: ${{ steps.set-nightly-build-result.outputs.nightly-build-result }}
     steps:
@@ -198,14 +208,15 @@ jobs:
         run: |
           echo "nightly-build-result=success" >> $GITHUB_OUTPUT
 
-  release-cn-artifacts:
-    name: Release artifacts to CN region
+  release-images-to-acr:
+    name: Build and push images to ACR
     if: ${{ inputs.release_images || github.event_name == 'schedule' }}
     needs: [
       allocate-runners,
-      release-images-to-dockerhub,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
     ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -215,30 +226,21 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Release artifacts to CN region
-        uses: ./.github/actions/release-cn-artifacts
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
         with:
-          src-image-registry: docker.io
-          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          src-image-name: greptimedb
-          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
-          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          dev-mode: false
-          update-version-info: false  # Don't update version info in S3.
-          push-latest-tag: false      # Don't push the latest tag to registry.
+          push-latest-tag: false # Don't push the latest tag to registry.
 
   stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -263,7 +265,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,
@@ -290,7 +292,7 @@ jobs:
     needs: [
       release-images-to-dockerhub
     ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
     steps:

.github/workflows/nightly-ci.yml

@@ -1,98 +0,0 @@
-# Nightly CI: runs tests every night for our second tier plaforms (Windows)
-
-on:
-  schedule:
-    - cron: '0 23 * * 1-5'
-  workflow_dispatch:
-
-name: Nightly CI
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  RUST_TOOLCHAIN: nightly-2023-08-07
-
-jobs:
-  sqlness:
-    name: Sqlness Test
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ windows-latest-8-cores ]
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v4.1.0
-      - uses: arduino/setup-protoc@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-      - name: Run sqlness
-        run: cargo sqlness
-      - name: Notify slack if failed
-        if: failure()
-        uses: slackapi/slack-github-action@v1.23.0
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
-        with:
-          payload: |
-            {"text": "Nightly CI failed for sqlness tests"}
-      - name: Upload sqlness logs
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: sqlness-logs
-          path: ${{ runner.temp }}/greptime-*.log
-          retention-days: 3
-
-  test-on-windows:
-    runs-on: windows-latest-8-cores
-    timeout-minutes: 60
-    steps:
-      - run: git config --global core.autocrlf false
-      - uses: actions/checkout@v4.1.0
-      - uses: arduino/setup-protoc@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          components: llvm-tools-preview
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-      - name: Install Cargo Nextest
-        uses: taiki-e/install-action@nextest
-      - name: Install Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Install PyArrow Package
-        run: pip install pyarrow
-      - name: Install WSL distribution
-        uses: Vampire/setup-wsl@v2
-        with:
-          distribution: Ubuntu-22.04
-      - name: Running tests
-        run: cargo nextest run -F pyo3_backend,dashboard
-        env:
-          RUST_BACKTRACE: 1
-          CARGO_INCREMENTAL: 0
-          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-          GT_S3_REGION: ${{ secrets.S3_REGION }}
-          UNITTEST_LOG_DIR: "__unittest_logs"
-      - name: Notify slack if failed
-        if: failure()
-        uses: slackapi/slack-github-action@v1.23.0
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
-        with:
-          payload: |
-            {"text": "Nightly CI failed for cargo test"}

.github/workflows/pr-title-checker.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
       - uses: thehanimo/pr-title-checker@v1.3.4
@@ -19,7 +19,7 @@ jobs:
           pass_on_octokit_error: false
           configuration_path: ".github/pr-title-checker-config.json"
   breaking:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
       - uses: thehanimo/pr-title-checker@v1.3.4

.github/workflows/release-dev-builder-images.yaml

@@ -1,85 +0,0 @@
-name: Release dev-builder images
-
-on:
-  workflow_dispatch: # Allows you to run this workflow manually.
-    inputs:
-      version:
-        description: Version of the dev-builder
-        required: false
-        default: latest
-      release_dev_builder_ubuntu_image:
-        type: boolean
-        description: Release dev-builder-ubuntu image
-        required: false
-        default: false
-      release_dev_builder_centos_image:
-        type: boolean
-        description: Release dev-builder-centos image
-        required: false
-        default: false
-      release_dev_builder_android_image:
-        type: boolean
-        description: Release dev-builder-android image
-        required: false
-        default: false
-
-jobs:
-  release-dev-builder-images:
-    name: Release dev builder images
-    if: ${{ inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }} # Only manually trigger this job.
-    runs-on: ubuntu-20.04-16-cores
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Build and push dev builder images
-        uses: ./.github/actions/build-dev-builder-images
-        with:
-          version: ${{ inputs.version }}
-          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
-          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
-          build-dev-builder-ubuntu: ${{ inputs.release_dev_builder_ubuntu_image }}
-          build-dev-builder-centos: ${{ inputs.release_dev_builder_centos_image }}
-          build-dev-builder-android: ${{ inputs.release_dev_builder_android_image }}
-
-  release-dev-builder-images-cn: # Note: Be careful issue: https://github.com/containers/skopeo/issues/1874 and we decide to use the latest stable skopeo container.
-    name: Release dev builder images to CN region
-    runs-on: ubuntu-20.04
-    needs: [
-      release-dev-builder-images
-    ]
-    steps:
-      - name: Push dev-builder-ubuntu image
-        shell: bash
-        if: ${{ inputs.release_dev_builder_ubuntu_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
-        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
-            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }}
-
-      - name: Push dev-builder-centos image
-        shell: bash
-        if: ${{ inputs.release_dev_builder_centos_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
-        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
-            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }}
-
-      - name: Push dev-builder-android image
-        shell: bash
-        if: ${{ inputs.release_dev_builder_android_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
-        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
-            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }}

.github/workflows/release.yml

@@ -18,11 +18,11 @@ on:
         description: The runner uses to build linux-amd64 artifacts
         default: ec2-c6i.4xlarge-amd64
         options:
-          - ubuntu-20.04
-          - ubuntu-20.04-8-cores
-          - ubuntu-20.04-16-cores
-          - ubuntu-20.04-32-cores
-          - ubuntu-20.04-64-cores
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
           - ec2-c6i.xlarge-amd64 # 4C8G
           - ec2-c6i.2xlarge-amd64 # 8C16G
           - ec2-c6i.4xlarge-amd64 # 16C32G
@@ -63,12 +63,7 @@ on:
         description: Build macos artifacts
         required: false
         default: false
-      build_windows_artifacts:
-        type: boolean
-        description: Build Windows artifacts
-        required: false
-        default: false
-      publish_github_release:
+      release_artifacts:
         type: boolean
         description: Create GitHub release and upload artifacts
         required: false
@@ -78,6 +73,11 @@ on:
         description: Build and push images to DockerHub and ACR
         required: false
         default: false
+      release_dev_builder_image:
+        type: boolean
+        description: Release dev-builder image
+        required: false
+        default: false
 
 # Use env variables to control all the release process.
 env:
@@ -91,18 +91,17 @@ env:
   # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
   NIGHTLY_RELEASE_PREFIX: nightly
   # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
-  NEXT_RELEASE_VERSION: v0.5.0
+  NEXT_RELEASE_VERSION: v0.4.0
 
 jobs:
   allocate-runners:
     name: Allocate runners
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
       linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
       linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
       macos-runner: ${{ inputs.macos_runner || vars.DEFAULT_MACOS_RUNNER }}
-      windows-runner: windows-latest-8-cores
 
       # The following EC2 resource id will be used for resource releasing.
       linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
@@ -178,6 +177,10 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -197,6 +200,10 @@ jobs:
           cargo-profile: ${{ env.CARGO_PROFILE }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
   build-macos-artifacts:
     name: Build macOS artifacts
@@ -238,43 +245,11 @@ jobs:
           features: ${{ matrix.features }}
           version: ${{ needs.allocate-runners.outputs.version }}
           disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
           artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
-
-  build-windows-artifacts:
-    name: Build Windows artifacts
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ${{ needs.allocate-runners.outputs.windows-runner }}
-            arch: x86_64-pc-windows-msvc
-            features: servers/dashboard
-            artifacts-dir-prefix: greptime-windows-amd64
-          - os: ${{ needs.allocate-runners.outputs.windows-runner }}
-            arch: x86_64-pc-windows-msvc
-            features: pyo3_backend,servers/dashboard
-            artifacts-dir-prefix: greptime-windows-amd64-pyo3
-    runs-on: ${{ matrix.os }}
-    needs: [
-      allocate-runners,
-    ]
-    if: ${{ inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
-    steps:
-      - run: git config --global core.autocrlf false
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - uses: ./.github/actions/build-windows-artifacts
-        with:
-          arch: ${{ matrix.arch }}
-          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
-          cargo-profile: ${{ env.CARGO_PROFILE }}
-          features: ${{ matrix.features }}
-          version: ${{ needs.allocate-runners.outputs.version }}
-          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
-          artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub
@@ -299,18 +274,15 @@ jobs:
           image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
           version: ${{ needs.allocate-runners.outputs.version }}
 
-  release-cn-artifacts:
-    name: Release artifacts to CN region
+  release-images-to-acr:
+    name: Build and push images to ACR
     if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
-    needs: [ # The job have to wait for all the artifacts are built.
+    needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
-      build-macos-artifacts,
-      build-windows-artifacts,
-      release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-2004-16-cores
     # When we push to ACR, it's easy to fail due to some unknown network issues.
     # However, we don't want to fail the whole workflow because of this.
     # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
@@ -320,47 +292,55 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Release artifacts to CN region
-        uses: ./.github/actions/release-cn-artifacts
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
         with:
-          src-image-registry: docker.io
-          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
-          src-image-name: greptimedb
-          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
-          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
-          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
-          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
           version: ${{ needs.allocate-runners.outputs.version }}
-          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
-          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
-          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
-          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
-          dev-mode: false
-          update-version-info: true
-          push-latest-tag: true
 
-  publish-github-release:
+  release-artifacts:
     name: Create GitHub release and upload artifacts
-    if: ${{ inputs.publish_github_release || github.event_name == 'push' || github.event_name == 'schedule' }}
-    needs: [ # The job have to wait for all the artifacts are built.
+    if: ${{ inputs.release_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
       build-linux-arm64-artifacts,
       build-macos-artifacts,
-      build-windows-artifacts,
       release-images-to-dockerhub,
     ]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Publish GitHub release
-        uses: ./.github/actions/publish-github-release
+      - name: Release artifacts
+        uses: ./.github/actions/release-artifacts
         with:
           version: ${{ needs.allocate-runners.outputs.version }}
 
+  release-dev-builder-image:
+    name: Release dev builder image
+    if: ${{ inputs.release_dev_builder_image }} # Only manually trigger this job.
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push dev builder image
+        uses: ./.github/actions/build-dev-builder-image
+        with:
+          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          acr-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          acr-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          acr-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+
   ### Stop runners ###
   # It's very necessary to split the job of releasing runners into 'stop-linux-amd64-runner' and 'stop-linux-arm64-runner'.
   # Because we can terminate the specified EC2 instance immediately after the job is finished without uncessary waiting.
@@ -368,7 +348,7 @@ jobs:
     name: Stop linux-amd64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -393,7 +373,7 @@ jobs:
     name: Stop linux-arm64 runner
     # Only run this job when the runner is allocated.
     if: ${{ always() }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [
       allocate-runners,
       build-linux-arm64-artifacts,

.github/workflows/size-label.yml

@@ -1,26 +0,0 @@
-name: size-labeler
-
-on: [pull_request]
-
-jobs:
-  labeler:
-    runs-on: ubuntu-latest
-    name: Label the PR size
-    steps:
-      - uses: codelytv/pr-size-labeler@v1
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          s_label: 'Size: S'
-          s_max_size: '100'
-          m_label: 'Size: M'
-          m_max_size: '500'
-          l_label: 'Size: L'
-          l_max_size: '1000'
-          xl_label: 'Size: XL'
-          fail_if_xl: 'false'
-          message_if_xl: >
-            This PR exceeds the recommended size of 1000 lines.
-            Please make sure you are NOT addressing multiple issues with one PR.
-            Note this PR might be rejected due to its size.
-          github_api_url: 'api.github.com'
-          files_to_ignore: 'Cargo.lock'

Cargo.toml

@@ -8,11 +8,10 @@ members = [
     "src/cmd",
     "src/common/base",
     "src/common/catalog",
-    "src/common/config",
     "src/common/datasource",
     "src/common/error",
     "src/common/function",
-    "src/common/macro",
+    "src/common/function-macro",
     "src/common/greptimedb-telemetry",
     "src/common/grpc",
     "src/common/grpc-expr",
@@ -30,16 +29,15 @@ members = [
     "src/common/version",
     "src/datanode",
     "src/datatypes",
-    "src/file-engine",
+    "src/file-table-engine",
     "src/frontend",
     "src/log-store",
     "src/meta-client",
     "src/meta-srv",
+    "src/mito",
     "src/mito2",
     "src/object-store",
-    "src/operator",
     "src/partition",
-    "src/plugins",
     "src/promql",
     "src/query",
     "src/script",
@@ -48,62 +46,53 @@ members = [
     "src/sql",
     "src/storage",
     "src/store-api",
+    "src/flow",
     "src/table",
+    "src/table-procedure",
     "tests-integration",
     "tests/runner",
 ]
 resolver = "2"
 
 [workspace.package]
-version = "0.4.2"
+version = "0.4.0-nightly"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
-aquamarine = "0.3"
 arrow = { version = "43.0" }
+etcd-client = "0.11"
 arrow-array = "43.0"
 arrow-flight = "43.0"
 arrow-schema = { version = "43.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "b6f3b28b6fe91924cc8dd3d83726b766f2a706ec" }
+datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
 derive_builder = "0.12"
-etcd-client = "0.11"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "1f1dd532a111e3834cc3019c5605e2993ffb9dc3" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "4a277f27caa035a801d5b9c020a0449777736614" }
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
-metrics = "0.20"
-moka = "0.12"
 once_cell = "1.18"
-opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics", "traces"] }
+opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
 parquet = "43.0"
 paste = "1.0"
 prost = "0.11"
-raft-engine = { git = "https://github.com/tikv/raft-engine.git", rev = "22dfb426cd994602b57725ef080287d3e53db479" }
 rand = "0.8"
 regex = "1.8"
-reqwest = { version = "0.11", default-features = false, features = [
-    "json",
-    "rustls-tls-native-roots",
-    "stream",
-] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-smallvec = "1"
 snafu = { version = "0.7", features = ["backtraces"] }
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "6cf9d23d5b8fbecd65efc1d9afb7e80ad7a424da", features = [
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4", features = [
     "visitor",
 ] }
 strum = { version = "0.25", features = ["derive"] }
@@ -113,6 +102,8 @@ tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.7"
 tonic = { version = "0.9", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
+metrics = "0.20"
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
 ## workspaces members
 api = { path = "src/api" }
 auth = { path = "src/auth" }
@@ -121,29 +112,29 @@ client = { path = "src/client" }
 cmd = { path = "src/cmd" }
 common-base = { path = "src/common/base" }
 common-catalog = { path = "src/common/catalog" }
-common-config = { path = "src/common/config" }
 common-datasource = { path = "src/common/datasource" }
 common-error = { path = "src/common/error" }
 common-function = { path = "src/common/function" }
+common-function-macro = { path = "src/common/function-macro" }
 common-greptimedb-telemetry = { path = "src/common/greptimedb-telemetry" }
 common-grpc = { path = "src/common/grpc" }
 common-grpc-expr = { path = "src/common/grpc-expr" }
-common-macro = { path = "src/common/macro" }
 common-mem-prof = { path = "src/common/mem-prof" }
 common-meta = { path = "src/common/meta" }
-common-pprof = { path = "src/common/pprof" }
 common-procedure = { path = "src/common/procedure" }
 common-procedure-test = { path = "src/common/procedure-test" }
+common-pprof = { path = "src/common/pprof" }
 common-query = { path = "src/common/query" }
 common-recordbatch = { path = "src/common/recordbatch" }
 common-runtime = { path = "src/common/runtime" }
+substrait = { path = "src/common/substrait" }
 common-telemetry = { path = "src/common/telemetry" }
 common-test-util = { path = "src/common/test-util" }
 common-time = { path = "src/common/time" }
 common-version = { path = "src/common/version" }
 datanode = { path = "src/datanode" }
 datatypes = { path = "src/datatypes" }
-file-engine = { path = "src/file-engine" }
+file-table-engine = { path = "src/file-table-engine" }
 frontend = { path = "src/frontend" }
 log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
@@ -151,9 +142,7 @@ meta-srv = { path = "src/meta-srv" }
 mito = { path = "src/mito" }
 mito2 = { path = "src/mito2" }
 object-store = { path = "src/object-store" }
-operator = { path = "src/operator" }
 partition = { path = "src/partition" }
-plugins = { path = "src/plugins" }
 promql = { path = "src/promql" }
 query = { path = "src/query" }
 script = { path = "src/script" }
@@ -162,8 +151,8 @@ session = { path = "src/session" }
 sql = { path = "src/sql" }
 storage = { path = "src/storage" }
 store-api = { path = "src/store-api" }
-substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }
+table-procedure = { path = "src/table-procedure" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"

Makefile

@@ -55,15 +55,11 @@ else
 	BUILDX_MULTI_PLATFORM_BUILD_OPTS := -o type=docker
 endif
 
-ifneq ($(strip $(CARGO_BUILD_EXTRA_OPTS)),)
-	CARGO_BUILD_OPTS += ${CARGO_BUILD_EXTRA_OPTS}
-endif
-
 ##@ Build
 
 .PHONY: build
 build: ## Build debug version greptime.
-	cargo ${CARGO_EXTENSION} build ${CARGO_BUILD_OPTS}
+	cargo build ${CARGO_BUILD_OPTS}
 
 .POHNY: build-by-dev-builder
 build-by-dev-builder: ## Build greptime by dev-builder.
@@ -71,34 +67,11 @@ build-by-dev-builder: ## Build greptime by dev-builder.
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
 	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
 	make build \
-	CARGO_EXTENSION="${CARGO_EXTENSION}" \
 	CARGO_PROFILE=${CARGO_PROFILE} \
 	FEATURES=${FEATURES} \
 	TARGET_DIR=${TARGET_DIR} \
 	TARGET=${TARGET} \
-	RELEASE=${RELEASE} \
-	CARGO_BUILD_EXTRA_OPTS="${CARGO_BUILD_EXTRA_OPTS}"
-
-.PHONY: build-android-bin
-build-android-bin: ## Build greptime binary for android.
-	docker run --network=host \
-	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \
-	make build \
-	CARGO_EXTENSION="ndk --platform 23 -t aarch64-linux-android" \
-	CARGO_PROFILE=release \
-	FEATURES="${FEATURES}" \
-	TARGET_DIR="${TARGET_DIR}" \
-	TARGET="${TARGET}" \
-	RELEASE="${RELEASE}" \
-	CARGO_BUILD_EXTRA_OPTS="--bin greptime --no-default-features"
-
-.PHONY: strip-android-bin
-strip-android-bin: build-android-bin ## Strip greptime binary for android.
-	docker run --network=host \
-	-v ${PWD}:/greptimedb \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:latest \
-	bash -c '$${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip /greptimedb/target/aarch64-linux-android/release/greptime'
+	RELEASE=${RELEASE}
 
 .PHONY: clean
 clean: ## Clean the project.

README.md

@@ -27,14 +27,6 @@
     <a href="https://greptime.com/slack"><img src="https://img.shields.io/badge/slack-GreptimeDB-0abd59?logo=slack" alt="slack" /></a>
 </p>
 
-## Upcoming Event
-Come and meet us in **KubeCon + CloudNativeCon North America 2023!**
-<p align="center">
-  <picture>
-    <img alt="KubeCon + CloudNativeCon North Logo" src="./docs/banner/KCCNC_NA_2023_1000x200_Email Banner.png" width="800px">
-  </picture>
-</p>
-
 ## What is GreptimeDB
 
 GreptimeDB is an open-source time-series database with a special focus on
@@ -104,11 +96,11 @@ Or if you built from docker:
 docker run -p 4002:4002 -v "$(pwd):/tmp/greptimedb" greptime/greptimedb standalone start
 ```
 
-Please see the online document site for more installation options and [operations info](https://docs.greptime.com/user-guide/operations/overview).
+Please see [the online document site](https://docs.greptime.com/getting-started/overview#install-greptimedb) for more installation options and [operations info](https://docs.greptime.com/user-guide/operations/overview).
 
 ### Get started
 
-Read the [complete getting started guide](https://docs.greptime.com/getting-started/try-out-greptimedb) on our [official document site](https://docs.greptime.com/).
+Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview#connect) on our [official document site](https://docs.greptime.com/).
 
 To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients/overview).
 

benchmarks/Cargo.toml

@@ -6,10 +6,8 @@ license.workspace = true
 
 [dependencies]
 arrow.workspace = true
-chrono.workspace = true
 clap = { version = "4.0", features = ["derive"] }
 client = { workspace = true }
-futures-util.workspace = true
 indicatif = "0.17.1"
 itertools.workspace = true
 parquet.workspace = true

benchmarks/src/bin/nyc-taxi.rs

@@ -27,16 +27,16 @@ use arrow::record_batch::RecordBatch;
 use clap::Parser;
 use client::api::v1::column::Values;
 use client::api::v1::{
-    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests, SemanticType,
+    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests,
 };
-use client::{Client, Database, Output, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use futures_util::TryStreamExt;
+use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
 use tokio::task::JoinSet;
 
 const CATALOG_NAME: &str = "greptime";
 const SCHEMA_NAME: &str = "public";
+const TABLE_NAME: &str = "nyc_taxi";
 
 #[derive(Parser)]
 #[command(name = "NYC benchmark runner")]
@@ -74,12 +74,7 @@ fn get_file_list<P: AsRef<Path>>(path: P) -> Vec<PathBuf> {
         .collect()
 }
 
-fn new_table_name() -> String {
-    format!("nyc_taxi_{}", chrono::Utc::now().timestamp())
-}
-
 async fn write_data(
-    table_name: &str,
     batch_size: usize,
     db: &Database,
     path: PathBuf,
@@ -109,7 +104,8 @@ async fn write_data(
         }
         let (columns, row_count) = convert_record_batch(record_batch);
         let request = InsertRequest {
-            table_name: table_name.to_string(),
+            table_name: TABLE_NAME.to_string(),
+            region_number: 0,
             columns,
             row_count,
         };
@@ -118,7 +114,7 @@ async fn write_data(
         };
 
         let now = Instant::now();
-        db.insert(requests).await.unwrap();
+        let _ = db.insert(requests).await.unwrap();
         let elapsed = now.elapsed();
         total_rpc_elapsed_ms += elapsed.as_millis();
         progress_bar.inc(row_count as _);
@@ -136,11 +132,6 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
 
     for (array, field) in record_batch.columns().iter().zip(fields.iter()) {
         let (values, datatype) = build_values(array);
-        let semantic_type = match field.name().as_str() {
-            "VendorID" => SemanticType::Tag,
-            "tpep_pickup_datetime" => SemanticType::Timestamp,
-            _ => SemanticType::Field,
-        };
 
         let column = Column {
             column_name: field.name().clone(),
@@ -151,7 +142,8 @@ fn convert_record_batch(record_batch: RecordBatch) -> (Vec<Column>, u32) {
                 .map(|bitmap| bitmap.buffer().as_slice().to_vec())
                 .unwrap_or_default(),
             datatype: datatype.into(),
-            semantic_type: semantic_type as i32,
+            // datatype and semantic_type are set to default
+            ..Default::default()
         };
         columns.push(column);
     }
@@ -197,7 +189,7 @@ fn build_values(column: &ArrayRef) -> (Values, ColumnDataType) {
             let values = array.values();
             (
                 Values {
-                    timestamp_microsecond_values: values.to_vec(),
+                    ts_microsecond_values: values.to_vec(),
                     ..Default::default()
                 },
                 ColumnDataType::TimestampMicrosecond,
@@ -252,193 +244,156 @@ fn is_record_batch_full(batch: &RecordBatch) -> bool {
     batch.columns().iter().all(|col| col.null_count() == 0)
 }
 
-fn create_table_expr(table_name: &str) -> CreateTableExpr {
+fn create_table_expr() -> CreateTableExpr {
     CreateTableExpr {
         catalog_name: CATALOG_NAME.to_string(),
         schema_name: SCHEMA_NAME.to_string(),
-        table_name: table_name.to_string(),
+        table_name: TABLE_NAME.to_string(),
         desc: "".to_string(),
         column_defs: vec![
             ColumnDef {
                 name: "VendorID".to_string(),
-                data_type: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Tag as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "tpep_pickup_datetime".to_string(),
-                data_type: ColumnDataType::TimestampMicrosecond as i32,
-                is_nullable: false,
+                datatype: ColumnDataType::TimestampMicrosecond as i32,
+                is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Timestamp as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "tpep_dropoff_datetime".to_string(),
-                data_type: ColumnDataType::TimestampMicrosecond as i32,
+                datatype: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "passenger_count".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "trip_distance".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "RatecodeID".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "store_and_fwd_flag".to_string(),
-                data_type: ColumnDataType::String as i32,
+                datatype: ColumnDataType::String as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "PULocationID".to_string(),
-                data_type: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "DOLocationID".to_string(),
-                data_type: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "payment_type".to_string(),
-                data_type: ColumnDataType::Int64 as i32,
+                datatype: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "fare_amount".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "extra".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "mta_tax".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "tip_amount".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "tolls_amount".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "improvement_surcharge".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "total_amount".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "congestion_surcharge".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "airport_fee".to_string(),
-                data_type: ColumnDataType::Float64 as i32,
+                datatype: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
         ],
         time_index: "tpep_pickup_datetime".to_string(),
         primary_keys: vec!["VendorID".to_string()],
-        create_if_not_exists: true,
+        create_if_not_exists: false,
         table_options: Default::default(),
+        region_numbers: vec![0],
         table_id: None,
         engine: "mito".to_string(),
     }
 }
 
-fn query_set(table_name: &str) -> HashMap<String, String> {
+fn query_set() -> HashMap<String, String> {
     HashMap::from([
         (
             "count_all".to_string(), 
-            format!("SELECT COUNT(*) FROM {table_name};"),
+            format!("SELECT COUNT(*) FROM {TABLE_NAME};"),
         ),
         (
             "fare_amt_by_passenger".to_string(),
-            format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {table_name} GROUP BY passenger_count"),
+            format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {TABLE_NAME} GROUP BY passenger_count"),
         )
     ])
 }
 
-async fn do_write(args: &Args, db: &Database, table_name: &str) {
+async fn do_write(args: &Args, db: &Database) {
     let mut file_list = get_file_list(args.path.clone().expect("Specify data path in argument"));
     let mut write_jobs = JoinSet::new();
 
-    let create_table_result = db.create(create_table_expr(table_name)).await;
+    let create_table_result = db.create(create_table_expr()).await;
     println!("Create table result: {create_table_result:?}");
 
     let progress_bar_style = ProgressStyle::with_template(
@@ -456,10 +411,8 @@ async fn do_write(args: &Args, db: &Database, table_name: &str) {
             let db = db.clone();
             let mpb = multi_progress_bar.clone();
             let pb_style = progress_bar_style.clone();
-            let table_name = table_name.to_string();
-            let _ = write_jobs.spawn(async move {
-                write_data(&table_name, batch_size, &db, path, mpb, pb_style).await
-            });
+            let _ = write_jobs
+                .spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
         }
     }
     while write_jobs.join_next().await.is_some() {
@@ -468,32 +421,24 @@ async fn do_write(args: &Args, db: &Database, table_name: &str) {
             let db = db.clone();
             let mpb = multi_progress_bar.clone();
             let pb_style = progress_bar_style.clone();
-            let table_name = table_name.to_string();
-            let _ = write_jobs.spawn(async move {
-                write_data(&table_name, batch_size, &db, path, mpb, pb_style).await
-            });
+            let _ = write_jobs
+                .spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
         }
     }
 }
 
-async fn do_query(num_iter: usize, db: &Database, table_name: &str) {
-    for (query_name, query) in query_set(table_name) {
+async fn do_query(num_iter: usize, db: &Database) {
+    for (query_name, query) in query_set() {
         println!("Running query: {query}");
         for i in 0..num_iter {
             let now = Instant::now();
-            let res = db.sql(&query).await.unwrap();
-            match res {
-                Output::AffectedRows(_) | Output::RecordBatches(_) => (),
-                Output::Stream(stream) => {
-                    stream.try_collect::<Vec<_>>().await.unwrap();
-                }
-            }
+            let _res = db.sql(&query).await.unwrap();
             let elapsed = now.elapsed();
             println!(
                 "query {}, iteration {}: {}ms",
                 query_name,
                 i,
-                elapsed.as_millis(),
+                elapsed.as_millis()
             );
         }
     }
@@ -510,14 +455,13 @@ fn main() {
         .block_on(async {
             let client = Client::with_urls(vec![&args.endpoint]);
             let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
-            let table_name = new_table_name();
 
             if !args.skip_write {
-                do_write(&args, &db, &table_name).await;
+                do_write(&args, &db).await;
             }
 
             if !args.skip_read {
-                do_query(args.iter_num, &db, &table_name).await;
+                do_query(args.iter_num, &db).await;
             }
         })
 }

config/datanode.example.toml

@@ -1,5 +1,7 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
+# Whether to use in-memory catalog, see `standalone.example.toml`.
+enable_memory_catalog = false
 # The datanode identifier, should be unique.
 node_id = 42
 # gRPC server address, "127.0.0.1:3001" by default.
@@ -8,24 +10,19 @@ rpc_addr = "127.0.0.1:3001"
 rpc_hostname = "127.0.0.1"
 # The number of gRPC server worker threads, 8 by default.
 rpc_runtime_size = 8
-# Start services after regions have obtained leases.
-# It will block the datanode start if it can't receive leases in the heartbeat from metasrv.
-require_lease_before_startup = false
 
 [heartbeat]
-# Interval for sending heartbeat messages to the Metasrv, 3 seconds by default.
-interval = "3s"
+# Interval for sending heartbeat messages to the Metasrv in milliseconds, 5000 by default.
+interval_millis = 5000
 
 # Metasrv client options.
-[meta_client]
+[meta_client_options]
 # Metasrv address list.
 metasrv_addrs = ["127.0.0.1:3002"]
-# Heartbeat timeout, 500 milliseconds by default.
-heartbeat_timeout = "500ms"
-# Operation timeout, 3 seconds by default.
-timeout = "3s"
-# Connect server timeout, 1 second by default.
-connect_timeout = "1s"
+# Operation timeout in milliseconds, 3000 by default.
+timeout_millis = 3000
+# Connect server timeout in milliseconds, 5000 by default.
+connect_timeout_millis = 5000
 # `TCP_NODELAY` option for accepted connections, true by default.
 tcp_nodelay = true
 
@@ -47,12 +44,6 @@ type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
 
-# Cache configuration for object storage such as 'S3' etc.
-# The local file cache directory
-# cache_path = "/path/local_cache"
-# The local file cache capacity in bytes.
-# cache_capacity = "256MB"
-
 # Compaction options, see `standalone.example.toml`.
 [storage.compaction]
 max_inflight_tasks = 4
@@ -80,27 +71,10 @@ auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
 
-# Mito engine options
-[[region_engine]]
-[region_engine.mito]
-# Number of region workers
-num_workers = 8
-# Request channel size of each worker
-worker_channel_size = 128
-# Max batch size for a worker to handle requests
-worker_request_batch_size = 64
-# Number of meta action updated to trigger a new checkpoint for the manifest
-manifest_checkpoint_distance = 10
-# Manifest compression type
-manifest_compress_type = "Uncompressed"
-# Max number of running background jobs
-max_background_jobs = 4
-# Interval to auto flush a region if it has not flushed yet.
-auto_flush_interval = "1h"
-# Global write buffer size for all regions.
-global_write_buffer_size = "1GB"
-# Global write buffer size threshold to reject write requests (default 2G).
-global_write_buffer_reject_size = "2GB"
+# Procedure storage options, see `standalone.example.toml`.
+[procedure]
+max_retry_times = 3
+retry_delay = "500ms"
 
 # Log options
 # [logging]

config/frontend.example.toml

@@ -2,67 +2,64 @@
 mode = "distributed"
 
 [heartbeat]
-# Interval for sending heartbeat task to the Metasrv, 5 seconds by default.
-interval = "5s"
-# Interval for retry sending heartbeat task, 5 seconds by default.
-retry_interval = "5s"
+# Interval for sending heartbeat task to the Metasrv in milliseconds, 5000 by default.
+interval_millis = 5000
+# Interval for retry sending heartbeat task in milliseconds, 5000 by default.
+retry_interval_millis = 5000
 
 # HTTP server options, see `standalone.example.toml`.
-[http]
+[http_options]
 addr = "127.0.0.1:4000"
 timeout = "30s"
 body_limit = "64MB"
 
 # gRPC server options, see `standalone.example.toml`.
-[grpc]
+[grpc_options]
 addr = "127.0.0.1:4001"
 runtime_size = 8
 
 # MySQL server options, see `standalone.example.toml`.
-[mysql]
-enable = true
+[mysql_options]
 addr = "127.0.0.1:4002"
 runtime_size = 2
 
 # MySQL server TLS options, see `standalone.example.toml`.
-[mysql.tls]
+[mysql_options.tls]
 mode = "disable"
 cert_path = ""
 key_path = ""
 
 # PostgresSQL server options, see `standalone.example.toml`.
-[postgres]
-enable = true
+[postgres_options]
 addr = "127.0.0.1:4003"
 runtime_size = 2
 
 # PostgresSQL server TLS options, see `standalone.example.toml`.
-[postgres.tls]
+[postgres_options.tls]
 mode = "disable"
 cert_path = ""
 key_path = ""
 
 # OpenTSDB protocol options, see `standalone.example.toml`.
-[opentsdb]
-enable = true
+[opentsdb_options]
 addr = "127.0.0.1:4242"
 runtime_size = 2
 
 # InfluxDB protocol options, see `standalone.example.toml`.
-[influxdb]
+[influxdb_options]
 enable = true
 
 # Prometheus remote storage options, see `standalone.example.toml`.
-[prom_store]
+[prom_store_options]
 enable = true
 
 # Metasrv client options, see `datanode.example.toml`.
-[meta_client]
+[meta_client_options]
 metasrv_addrs = ["127.0.0.1:3002"]
-timeout = "3s"
+timeout_millis = 3000
 # DDL timeouts options.
-ddl_timeout = "10s"
-connect_timeout = "1s"
+ddl_timeout_millis = 10000
+connect_timeout_millis = 5000
 tcp_nodelay = true
 
 # Log options, see `standalone.example.toml`

config/metasrv.example.toml

@@ -6,6 +6,8 @@ bind_addr = "127.0.0.1:3002"
 server_addr = "127.0.0.1:3002"
 # Etcd server address, "127.0.0.1:2379" by default.
 store_addr = "127.0.0.1:2379"
+# Datanode lease in seconds, 15 seconds by default.
+datanode_lease_secs = 15
 # Datanode selector type.
 # - "LeaseBased" (default value).
 # - "LoadBased"
@@ -32,6 +34,6 @@ retry_delay = "500ms"
 # [datanode]
 # # Datanode client options.
 # [datanode.client_options]
-# timeout = "10s"
-# connect_timeout = "10s"
+# timeout_millis = 10000
+# connect_timeout_millis = 10000
 # tcp_nodelay = true

config/standalone.example.toml

@@ -1,10 +1,12 @@
 # Node running mode, "standalone" or "distributed".
 mode = "standalone"
+# Whether to use in-memory catalog, `false` by default.
+enable_memory_catalog = false
 # Whether to enable greptimedb telemetry, true by default.
 enable_telemetry = true
 
 # HTTP server options.
-[http]
+[http_options]
 # Server address, "127.0.0.1:4000" by default.
 addr = "127.0.0.1:4000"
 # HTTP request timeout, 30s by default.
@@ -14,23 +16,21 @@ timeout = "30s"
 body_limit = "64MB"
 
 # gRPC server options.
-[grpc]
+[grpc_options]
 # Server address, "127.0.0.1:4001" by default.
 addr = "127.0.0.1:4001"
 # The number of server worker threads, 8 by default.
 runtime_size = 8
 
 # MySQL server options.
-[mysql]
-# Whether to enable
-enable = true
+[mysql_options]
 # Server address, "127.0.0.1:4002" by default.
 addr = "127.0.0.1:4002"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # MySQL server TLS options.
-[mysql.tls]
+[mysql_options.tls]
 # TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
 # - "disable" (default value)
 # - "prefer"
@@ -44,16 +44,14 @@ cert_path = ""
 key_path = ""
 
 # PostgresSQL server options.
-[postgres]
-# Whether to enable
-enable = true
+[postgres_options]
 # Server address, "127.0.0.1:4003" by default.
 addr = "127.0.0.1:4003"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # PostgresSQL server TLS options, see `[mysql_options.tls]` section.
-[postgres.tls]
+[postgres_options.tls]
 # TLS mode.
 mode = "disable"
 # certificate file path.
@@ -62,21 +60,19 @@ cert_path = ""
 key_path = ""
 
 # OpenTSDB protocol options.
-[opentsdb]
-# Whether to enable
-enable = true
+[opentsdb_options]
 # OpenTSDB telnet API server address, "127.0.0.1:4242" by default.
 addr = "127.0.0.1:4242"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # InfluxDB protocol options.
-[influxdb]
+[influxdb_options]
 # Whether to enable InfluxDB protocol in HTTP API, true by default.
 enable = true
 
 # Prometheus remote storage options
-[prom_store]
+[prom_store_options]
 # Whether to enable Prometheus remote write and read in HTTP API, true by default.
 enable = true
 
@@ -95,20 +91,6 @@ read_batch_size = 128
 # Whether to sync log file after every write.
 sync_write = false
 
-# Metadata storage options.
-[metadata_store]
-# Kv file size in bytes.
-file_size = "256MB"
-# Kv purge threshold.
-purge_threshold = "4GB"
-
-# Procedure storage options.
-[procedure]
-# Procedure max retry time.
-max_retry_times = 3
-# Initial retry delay of procedures, increases exponentially
-retry_delay = "500ms"
-
 # Storage options.
 [storage]
 # The working home directory.
@@ -117,10 +99,6 @@ data_home = "/tmp/greptimedb/"
 type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
-# Cache configuration for object storage such as 'S3' etc.
-# cache_path = "/path/local_cache"
-# The local file cache capacity in bytes.
-# cache_capacity = "256MB"
 
 # Compaction options.
 [storage.compaction]
@@ -152,6 +130,13 @@ auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
 
+# Procedure storage options.
+[procedure]
+# Procedure max retry time.
+max_retry_times = 3
+# Initial retry delay of procedures, increases exponentially
+retry_delay = "500ms"
+
 # Log options
 # [logging]
 # Specify logs directory.

docker/buildx/ubuntu/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04 as builder
+FROM ubuntu:22.04 as builder
 
 ARG CARGO_PROFILE
 ARG FEATURES
@@ -7,11 +7,6 @@ ARG OUTPUT_DIR
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
-# Add PPA for Python 3.10.
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
-    add-apt-repository ppa:deadsnakes/ppa -y
-
 # Install dependencies.
 RUN --mount=type=cache,target=/var/cache/apt \
     apt-get update && apt-get install -y \

docker/dev-builder/android/Dockerfile

@@ -1,41 +0,0 @@
-FROM --platform=linux/amd64 saschpe/android-ndk:34-jdk17.0.8_7-ndk25.2.9519653-cmake3.22.1
-
-ENV LANG en_US.utf8
-WORKDIR /greptimedb
-
-# Rename libunwind to libgcc
-RUN cp ${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/14.0.7/lib/linux/aarch64/libunwind.a ${NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/14.0.7/lib/linux/aarch64/libgcc.a
-
-# Install dependencies.
-RUN apt-get update && apt-get install -y \
-    libssl-dev \
-    protobuf-compiler \
-    curl \
-    git \
-    build-essential \
-    pkg-config \
-    python3 \
-    python3-dev \
-    python3-pip \
-    && pip3 install --upgrade pip \
-    && pip3 install pyarrow
-
-# Trust workdir
-RUN git config --global --add safe.directory /greptimedb
-
-# Install Rust.
-SHELL ["/bin/bash", "-c"]
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
-ENV PATH /root/.cargo/bin/:$PATH
-
-# Add android toolchains
-ARG RUST_TOOLCHAIN
-RUN rustup toolchain install ${RUST_TOOLCHAIN}
-RUN rustup target add aarch64-linux-android
-
-# Install cargo-ndk
-RUN cargo install cargo-ndk
-ENV ANDROID_NDK_HOME $NDK_ROOT
-
-# Builder entrypoint.
-CMD ["cargo", "ndk", "--platform", "23", "-t", "aarch64-linux-android", "build", "--bin", "greptime", "--profile", "release", "--no-default-features"]

docker/dev-builder/ubuntu/Dockerfile

@@ -1,13 +1,8 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
-# Add PPA for Python 3.10.
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
-    add-apt-repository ppa:deadsnakes/ppa -y
-
 # Install dependencies.
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     libssl-dev \
@@ -19,13 +14,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     build-essential \
     pkg-config \
     python3.10 \
-    python3.10-dev
-
-# Remove Python 3.8 and install pip.
-RUN apt-get -y purge python3.8 && \
-    apt-get -y autoremove && \
-    ln -s /usr/bin/python3.10 /usr/bin/python3 && \
-    curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+    python3.10-dev \
+    python3-pip
 
 RUN git config --global --add safe.directory /greptimedb
 

docs/benchmarks/tsbs/v0.4.0.md

@@ -1,61 +0,0 @@
-# TSBS benchmark - v0.4.0
-
-## Environment
-
-### Local
-|        |                                    |
-| ------ | ---------------------------------- |
-| CPU    | AMD Ryzen 7 7735HS (8 core 3.2GHz) |
-| Memory | 32GB                               |
-| Disk   | SOLIDIGM SSDPFKNU010TZ             |
-| OS     | Ubuntu 22.04.2 LTS                 |
-
-### Aliyun amd64
-
-|         |                |
-| ------- | -------------- |
-| Machine | ecs.g7.4xlarge |
-| CPU     | 16 core        |
-| Memory  | 64GB           |
-| Disk    | 100G           |
-| OS      | Ubuntu  22.04  |
-
-### Aliyun arm64
-
-|         |                   |
-| ------- | ----------------- |
-| Machine | ecs.g8y.4xlarge   |
-| CPU     | 16 core           |
-| Memory  | 64GB              |
-| Disk    | 100G              |
-| OS      | Ubuntu  22.04 ARM |
-
-
-## Write performance
-
-| Environment        | Ingest rate（rows/s） |
-| ------------------ | --------------------- |
-| Local              | 365280.60             |
-| Aliyun g7.4xlarge  | 341368.72             |
-| Aliyun g8y.4xlarge | 320907.29             |
-
-
-## Query performance
-
-| Query type            | Local (ms) | Aliyun g7.4xlarge (ms) | Aliyun g8y.4xlarge (ms) |
-| --------------------- | ---------- | ---------------------- | ----------------------- |
-| cpu-max-all-1         | 50.70      | 31.46                  | 47.61                   |
-| cpu-max-all-8         | 262.16     | 129.26                 | 152.43                  |
-| double-groupby-1      | 2512.71    | 1408.19                | 1586.10                 |
-| double-groupby-5      | 3896.15    | 2304.29                | 2585.29                 |
-| double-groupby-all    | 5404.67    | 3337.61                | 3773.91                 |
-| groupby-orderby-limit | 3786.98    | 2065.72                | 2312.57                 |
-| high-cpu-1            | 71.96      | 37.29                  | 54.01                   |
-| high-cpu-all          | 9468.75    | 7595.69                | 8467.46                 |
-| lastpoint             | 13379.43   | 11253.76               | 12949.40                |
-| single-groupby-1-1-1  | 20.72      | 12.16                  | 13.35                   |
-| single-groupby-1-1-12 | 28.53      | 15.67                  | 21.62                   |
-| single-groupby-1-8-1  | 72.23      | 37.90                  | 43.52                   |
-| single-groupby-5-1-1  | 26.75      | 15.59                  | 17.48                   |
-| single-groupby-5-1-12 | 45.41      | 22.90                  | 31.96                   |
-| single-groupby-5-8-1  | 107.96     | 59.76                  | 69.58                   |

src/api/Cargo.toml

@@ -7,7 +7,6 @@ license.workspace = true
 [dependencies]
 common-base = { workspace = true }
 common-error = { workspace = true }
-common-macro = { workspace = true }
 common-time = { workspace = true }
 datatypes = { workspace = true }
 greptime-proto.workspace = true

src/api/src/error.rs

@@ -16,16 +16,14 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
 use snafu::Location;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Unknown proto column datatype: {}", datatype))]
     UnknownColumnDataType { datatype: i32, location: Location },
@@ -36,14 +34,22 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to convert column default constraint, column: {}", column))]
+    #[snafu(display(
+        "Failed to convert column default constraint, column: {}, source: {}",
+        column,
+        source
+    ))]
     ConvertColumnDefaultConstraint {
         column: String,
         location: Location,
         source: datatypes::error::Error,
     },
 
-    #[snafu(display("Invalid column default constraint, column: {}", column))]
+    #[snafu(display(
+        "Invalid column default constraint, column: {}, source: {}",
+        column,
+        source
+    ))]
     InvalidColumnDefaultConstraint {
         column: String,
         location: Location,

src/api/src/helper.rs

@@ -18,32 +18,32 @@ use common_base::BitVec;
 use common_time::interval::IntervalUnit;
 use common_time::time::Time;
 use common_time::timestamp::TimeUnit;
-use common_time::{Date, DateTime, Duration, Interval, Timestamp};
+use common_time::{Date, DateTime, Interval, Timestamp};
 use datatypes::prelude::{ConcreteDataType, ValueRef};
 use datatypes::scalars::ScalarVector;
 use datatypes::types::{
-    DurationType, Int16Type, Int8Type, IntervalType, TimeType, TimestampType, UInt16Type, UInt8Type,
+    Int16Type, Int8Type, IntervalType, TimeType, TimestampType, UInt16Type, UInt8Type,
 };
 use datatypes::value::{OrderedF32, OrderedF64, Value};
 use datatypes::vectors::{
-    BinaryVector, BooleanVector, DateTimeVector, DateVector, DurationMicrosecondVector,
-    DurationMillisecondVector, DurationNanosecondVector, DurationSecondVector, Float32Vector,
-    Float64Vector, Int32Vector, Int64Vector, IntervalDayTimeVector, IntervalMonthDayNanoVector,
+    BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
+    Int32Vector, Int64Vector, IntervalDayTimeVector, IntervalMonthDayNanoVector,
     IntervalYearMonthVector, PrimitiveVector, StringVector, TimeMicrosecondVector,
     TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector,
     TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, UInt32Vector,
     UInt64Vector, VectorRef,
 };
+use greptime_proto::v1;
 use greptime_proto::v1::ddl_request::Expr;
 use greptime_proto::v1::greptime_request::Request;
 use greptime_proto::v1::query_request::Query;
 use greptime_proto::v1::value::ValueData;
-use greptime_proto::v1::{self, DdlRequest, IntervalMonthDayNano, QueryRequest, Row, SemanticType};
+use greptime_proto::v1::{DdlRequest, IntervalMonthDayNano, QueryRequest, SemanticType};
 use snafu::prelude::*;
 
 use crate::error::{self, Result};
 use crate::v1::column::Values;
-use crate::v1::{Column, ColumnDataType, Value as GrpcValue};
+use crate::v1::{Column, ColumnDataType};
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct ColumnDataTypeWrapper(ColumnDataType);
@@ -101,14 +101,6 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
             ColumnDataType::IntervalMonthDayNano => {
                 ConcreteDataType::interval_month_day_nano_datatype()
             }
-            ColumnDataType::DurationSecond => ConcreteDataType::duration_second_datatype(),
-            ColumnDataType::DurationMillisecond => {
-                ConcreteDataType::duration_millisecond_datatype()
-            }
-            ColumnDataType::DurationMicrosecond => {
-                ConcreteDataType::duration_microsecond_datatype()
-            }
-            ColumnDataType::DurationNanosecond => ConcreteDataType::duration_nanosecond_datatype(),
         }
     }
 }
@@ -150,12 +142,6 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
                 IntervalType::DayTime(_) => ColumnDataType::IntervalDayTime,
                 IntervalType::MonthDayNano(_) => ColumnDataType::IntervalMonthDayNano,
             },
-            ConcreteDataType::Duration(d) => match d {
-                DurationType::Second(_) => ColumnDataType::DurationSecond,
-                DurationType::Millisecond(_) => ColumnDataType::DurationMillisecond,
-                DurationType::Microsecond(_) => ColumnDataType::DurationMicrosecond,
-                DurationType::Nanosecond(_) => ColumnDataType::DurationNanosecond,
-            },
             ConcreteDataType::Null(_)
             | ConcreteDataType::List(_)
             | ConcreteDataType::Dictionary(_) => {
@@ -229,19 +215,19 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
             ..Default::default()
         },
         ColumnDataType::TimestampSecond => Values {
-            timestamp_second_values: Vec::with_capacity(capacity),
+            ts_second_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
         ColumnDataType::TimestampMillisecond => Values {
-            timestamp_millisecond_values: Vec::with_capacity(capacity),
+            ts_millisecond_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
         ColumnDataType::TimestampMicrosecond => Values {
-            timestamp_microsecond_values: Vec::with_capacity(capacity),
+            ts_microsecond_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
         ColumnDataType::TimestampNanosecond => Values {
-            timestamp_nanosecond_values: Vec::with_capacity(capacity),
+            ts_nanosecond_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
         ColumnDataType::TimeSecond => Values {
@@ -272,22 +258,6 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
             interval_month_day_nano_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
-        ColumnDataType::DurationSecond => Values {
-            duration_second_values: Vec::with_capacity(capacity),
-            ..Default::default()
-        },
-        ColumnDataType::DurationMillisecond => Values {
-            duration_millisecond_values: Vec::with_capacity(capacity),
-            ..Default::default()
-        },
-        ColumnDataType::DurationMicrosecond => Values {
-            duration_microsecond_values: Vec::with_capacity(capacity),
-            ..Default::default()
-        },
-        ColumnDataType::DurationNanosecond => Values {
-            duration_nanosecond_values: Vec::with_capacity(capacity),
-            ..Default::default()
-        },
     }
 }
 
@@ -317,10 +287,10 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
         Value::Date(val) => values.date_values.push(val.val()),
         Value::DateTime(val) => values.datetime_values.push(val.val()),
         Value::Timestamp(val) => match val.unit() {
-            TimeUnit::Second => values.timestamp_second_values.push(val.value()),
-            TimeUnit::Millisecond => values.timestamp_millisecond_values.push(val.value()),
-            TimeUnit::Microsecond => values.timestamp_microsecond_values.push(val.value()),
-            TimeUnit::Nanosecond => values.timestamp_nanosecond_values.push(val.value()),
+            TimeUnit::Second => values.ts_second_values.push(val.value()),
+            TimeUnit::Millisecond => values.ts_millisecond_values.push(val.value()),
+            TimeUnit::Microsecond => values.ts_microsecond_values.push(val.value()),
+            TimeUnit::Nanosecond => values.ts_nanosecond_values.push(val.value()),
         },
         Value::Time(val) => match val.unit() {
             TimeUnit::Second => values.time_second_values.push(val.value()),
@@ -335,12 +305,6 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
                 .interval_month_day_nano_values
                 .push(convert_i128_to_interval(val.to_i128())),
         },
-        Value::Duration(val) => match val.unit() {
-            TimeUnit::Second => values.duration_second_values.push(val.value()),
-            TimeUnit::Millisecond => values.duration_millisecond_values.push(val.value()),
-            TimeUnit::Microsecond => values.duration_microsecond_values.push(val.value()),
-            TimeUnit::Nanosecond => values.duration_nanosecond_values.push(val.value()),
-        },
         Value::List(_) => unreachable!(),
     });
     column.null_mask = null_mask.into_vec();
@@ -375,6 +339,8 @@ fn ddl_request_type(request: &DdlRequest) -> &'static str {
         Some(Expr::CreateTable(_)) => "ddl.create_table",
         Some(Expr::Alter(_)) => "ddl.alter",
         Some(Expr::DropTable(_)) => "ddl.drop_table",
+        Some(Expr::FlushTable(_)) => "ddl.flush_table",
+        Some(Expr::CompactTable(_)) => "ddl.compact_table",
         Some(Expr::TruncateTable(_)) => "ddl.truncate_table",
         None => "ddl.empty",
     }
@@ -412,16 +378,10 @@ pub fn pb_value_to_value_ref(value: &v1::Value) -> ValueRef {
         ValueData::StringValue(string) => ValueRef::String(string.as_str()),
         ValueData::DateValue(d) => ValueRef::Date(Date::from(*d)),
         ValueData::DatetimeValue(d) => ValueRef::DateTime(DateTime::new(*d)),
-        ValueData::TimestampSecondValue(t) => ValueRef::Timestamp(Timestamp::new_second(*t)),
-        ValueData::TimestampMillisecondValue(t) => {
-            ValueRef::Timestamp(Timestamp::new_millisecond(*t))
-        }
-        ValueData::TimestampMicrosecondValue(t) => {
-            ValueRef::Timestamp(Timestamp::new_microsecond(*t))
-        }
-        ValueData::TimestampNanosecondValue(t) => {
-            ValueRef::Timestamp(Timestamp::new_nanosecond(*t))
-        }
+        ValueData::TsSecondValue(t) => ValueRef::Timestamp(Timestamp::new_second(*t)),
+        ValueData::TsMillisecondValue(t) => ValueRef::Timestamp(Timestamp::new_millisecond(*t)),
+        ValueData::TsMicrosecondValue(t) => ValueRef::Timestamp(Timestamp::new_microsecond(*t)),
+        ValueData::TsNanosecondValue(t) => ValueRef::Timestamp(Timestamp::new_nanosecond(*t)),
         ValueData::TimeSecondValue(t) => ValueRef::Time(Time::new_second(*t)),
         ValueData::TimeMillisecondValue(t) => ValueRef::Time(Time::new_millisecond(*t)),
         ValueData::TimeMicrosecondValue(t) => ValueRef::Time(Time::new_microsecond(*t)),
@@ -432,10 +392,6 @@ pub fn pb_value_to_value_ref(value: &v1::Value) -> ValueRef {
             let interval = Interval::from_month_day_nano(v.months, v.days, v.nanoseconds);
             ValueRef::Interval(interval)
         }
-        ValueData::DurationSecondValue(v) => ValueRef::Duration(Duration::new_second(*v)),
-        ValueData::DurationMillisecondValue(v) => ValueRef::Duration(Duration::new_millisecond(*v)),
-        ValueData::DurationMicrosecondValue(v) => ValueRef::Duration(Duration::new_microsecond(*v)),
-        ValueData::DurationNanosecondValue(v) => ValueRef::Duration(Duration::new_nanosecond(*v)),
     }
 }
 
@@ -465,17 +421,17 @@ pub fn pb_values_to_vector_ref(data_type: &ConcreteDataType, values: Values) ->
         ConcreteDataType::Date(_) => Arc::new(DateVector::from_vec(values.date_values)),
         ConcreteDataType::DateTime(_) => Arc::new(DateTimeVector::from_vec(values.datetime_values)),
         ConcreteDataType::Timestamp(unit) => match unit {
-            TimestampType::Second(_) => Arc::new(TimestampSecondVector::from_vec(
-                values.timestamp_second_values,
-            )),
+            TimestampType::Second(_) => {
+                Arc::new(TimestampSecondVector::from_vec(values.ts_second_values))
+            }
             TimestampType::Millisecond(_) => Arc::new(TimestampMillisecondVector::from_vec(
-                values.timestamp_millisecond_values,
+                values.ts_millisecond_values,
             )),
             TimestampType::Microsecond(_) => Arc::new(TimestampMicrosecondVector::from_vec(
-                values.timestamp_microsecond_values,
+                values.ts_microsecond_values,
             )),
             TimestampType::Nanosecond(_) => Arc::new(TimestampNanosecondVector::from_vec(
-                values.timestamp_nanosecond_values,
+                values.ts_nanosecond_values,
             )),
         },
         ConcreteDataType::Time(unit) => match unit {
@@ -508,20 +464,6 @@ pub fn pb_values_to_vector_ref(data_type: &ConcreteDataType, values: Values) ->
                 ))
             }
         },
-        ConcreteDataType::Duration(unit) => match unit {
-            DurationType::Second(_) => Arc::new(DurationSecondVector::from_vec(
-                values.duration_second_values,
-            )),
-            DurationType::Millisecond(_) => Arc::new(DurationMillisecondVector::from_vec(
-                values.duration_millisecond_values,
-            )),
-            DurationType::Microsecond(_) => Arc::new(DurationMicrosecondVector::from_vec(
-                values.duration_microsecond_values,
-            )),
-            DurationType::Nanosecond(_) => Arc::new(DurationNanosecondVector::from_vec(
-                values.duration_nanosecond_values,
-            )),
-        },
         ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
             unreachable!()
         }
@@ -611,22 +553,22 @@ pub fn pb_values_to_values(data_type: &ConcreteDataType, values: Values) -> Vec<
             .map(|v| Value::Date(v.into()))
             .collect(),
         ConcreteDataType::Timestamp(TimestampType::Second(_)) => values
-            .timestamp_second_values
+            .ts_second_values
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_second(v)))
             .collect(),
         ConcreteDataType::Timestamp(TimestampType::Millisecond(_)) => values
-            .timestamp_millisecond_values
+            .ts_millisecond_values
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_millisecond(v)))
             .collect(),
         ConcreteDataType::Timestamp(TimestampType::Microsecond(_)) => values
-            .timestamp_microsecond_values
+            .ts_microsecond_values
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_microsecond(v)))
             .collect(),
         ConcreteDataType::Timestamp(TimestampType::Nanosecond(_)) => values
-            .timestamp_nanosecond_values
+            .ts_nanosecond_values
             .into_iter()
             .map(|v| Value::Timestamp(Timestamp::new_nanosecond(v)))
             .collect(),
@@ -672,26 +614,6 @@ pub fn pb_values_to_values(data_type: &ConcreteDataType, values: Values) -> Vec<
                 ))
             })
             .collect(),
-        ConcreteDataType::Duration(DurationType::Second(_)) => values
-            .duration_second_values
-            .into_iter()
-            .map(|v| Value::Duration(Duration::new_second(v)))
-            .collect(),
-        ConcreteDataType::Duration(DurationType::Millisecond(_)) => values
-            .duration_millisecond_values
-            .into_iter()
-            .map(|v| Value::Duration(Duration::new_millisecond(v)))
-            .collect(),
-        ConcreteDataType::Duration(DurationType::Microsecond(_)) => values
-            .duration_microsecond_values
-            .into_iter()
-            .map(|v| Value::Duration(Duration::new_microsecond(v)))
-            .collect(),
-        ConcreteDataType::Duration(DurationType::Nanosecond(_)) => values
-            .duration_nanosecond_values
-            .into_iter()
-            .map(|v| Value::Duration(Duration::new_nanosecond(v)))
-            .collect(),
         ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
             unreachable!()
         }
@@ -763,16 +685,16 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
         },
         Value::Timestamp(v) => match v.unit() {
             TimeUnit::Second => v1::Value {
-                value_data: Some(ValueData::TimestampSecondValue(v.value())),
+                value_data: Some(ValueData::TsSecondValue(v.value())),
             },
             TimeUnit::Millisecond => v1::Value {
-                value_data: Some(ValueData::TimestampMillisecondValue(v.value())),
+                value_data: Some(ValueData::TsMillisecondValue(v.value())),
             },
             TimeUnit::Microsecond => v1::Value {
-                value_data: Some(ValueData::TimestampMicrosecondValue(v.value())),
+                value_data: Some(ValueData::TsMicrosecondValue(v.value())),
             },
             TimeUnit::Nanosecond => v1::Value {
-                value_data: Some(ValueData::TimestampNanosecondValue(v.value())),
+                value_data: Some(ValueData::TsNanosecondValue(v.value())),
             },
         },
         Value::Time(v) => match v.unit() {
@@ -802,20 +724,6 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
                 )),
             },
         },
-        Value::Duration(v) => match v.unit() {
-            TimeUnit::Second => v1::Value {
-                value_data: Some(ValueData::DurationSecondValue(v.value())),
-            },
-            TimeUnit::Millisecond => v1::Value {
-                value_data: Some(ValueData::DurationMillisecondValue(v.value())),
-            },
-            TimeUnit::Microsecond => v1::Value {
-                value_data: Some(ValueData::DurationMicrosecondValue(v.value())),
-            },
-            TimeUnit::Nanosecond => v1::Value {
-                value_data: Some(ValueData::DurationNanosecondValue(v.value())),
-            },
-        },
         Value::List(_) => return None,
     };
 
@@ -842,10 +750,10 @@ pub fn proto_value_type(value: &v1::Value) -> Option<ColumnDataType> {
         ValueData::StringValue(_) => ColumnDataType::String,
         ValueData::DateValue(_) => ColumnDataType::Date,
         ValueData::DatetimeValue(_) => ColumnDataType::Datetime,
-        ValueData::TimestampSecondValue(_) => ColumnDataType::TimestampSecond,
-        ValueData::TimestampMillisecondValue(_) => ColumnDataType::TimestampMillisecond,
-        ValueData::TimestampMicrosecondValue(_) => ColumnDataType::TimestampMicrosecond,
-        ValueData::TimestampNanosecondValue(_) => ColumnDataType::TimestampNanosecond,
+        ValueData::TsSecondValue(_) => ColumnDataType::TimestampSecond,
+        ValueData::TsMillisecondValue(_) => ColumnDataType::TimestampMillisecond,
+        ValueData::TsMicrosecondValue(_) => ColumnDataType::TimestampMicrosecond,
+        ValueData::TsNanosecondValue(_) => ColumnDataType::TimestampNanosecond,
         ValueData::TimeSecondValue(_) => ColumnDataType::TimeSecond,
         ValueData::TimeMillisecondValue(_) => ColumnDataType::TimeMillisecond,
         ValueData::TimeMicrosecondValue(_) => ColumnDataType::TimeMicrosecond,
@@ -853,10 +761,6 @@ pub fn proto_value_type(value: &v1::Value) -> Option<ColumnDataType> {
         ValueData::IntervalYearMonthValues(_) => ColumnDataType::IntervalYearMonth,
         ValueData::IntervalDayTimeValues(_) => ColumnDataType::IntervalDayTime,
         ValueData::IntervalMonthDayNanoValues(_) => ColumnDataType::IntervalMonthDayNano,
-        ValueData::DurationSecondValue(_) => ColumnDataType::DurationSecond,
-        ValueData::DurationMillisecondValue(_) => ColumnDataType::DurationMillisecond,
-        ValueData::DurationMicrosecondValue(_) => ColumnDataType::DurationMicrosecond,
-        ValueData::DurationNanosecondValue(_) => ColumnDataType::DurationNanosecond,
     };
     Some(value_type)
 }
@@ -893,92 +797,15 @@ pub fn to_column_data_type(data_type: &ConcreteDataType) -> Option<ColumnDataTyp
         ConcreteDataType::Time(TimeType::Millisecond(_)) => ColumnDataType::TimeMillisecond,
         ConcreteDataType::Time(TimeType::Microsecond(_)) => ColumnDataType::TimeMicrosecond,
         ConcreteDataType::Time(TimeType::Nanosecond(_)) => ColumnDataType::TimeNanosecond,
-        ConcreteDataType::Duration(DurationType::Second(_)) => ColumnDataType::DurationSecond,
-        ConcreteDataType::Duration(DurationType::Millisecond(_)) => {
-            ColumnDataType::DurationMillisecond
-        }
-        ConcreteDataType::Duration(DurationType::Microsecond(_)) => {
-            ColumnDataType::DurationMicrosecond
-        }
-        ConcreteDataType::Duration(DurationType::Nanosecond(_)) => {
-            ColumnDataType::DurationNanosecond
-        }
-        ConcreteDataType::Interval(IntervalType::YearMonth(_)) => ColumnDataType::IntervalYearMonth,
-        ConcreteDataType::Interval(IntervalType::MonthDayNano(_)) => {
-            ColumnDataType::IntervalMonthDayNano
-        }
-        ConcreteDataType::Interval(IntervalType::DayTime(_)) => ColumnDataType::IntervalDayTime,
-        ConcreteDataType::Null(_) | ConcreteDataType::List(_) | ConcreteDataType::Dictionary(_) => {
-            return None
-        }
+        ConcreteDataType::Null(_)
+        | ConcreteDataType::Interval(_)
+        | ConcreteDataType::List(_)
+        | ConcreteDataType::Dictionary(_) => return None,
     };
 
     Some(column_data_type)
 }
 
-pub fn vectors_to_rows<'a>(
-    columns: impl Iterator<Item = &'a VectorRef>,
-    row_count: usize,
-) -> Vec<Row> {
-    let mut rows = vec![Row { values: vec![] }; row_count];
-    for column in columns {
-        for (row_index, row) in rows.iter_mut().enumerate() {
-            row.values.push(value_to_grpc_value(column.get(row_index)))
-        }
-    }
-
-    rows
-}
-
-pub fn value_to_grpc_value(value: Value) -> GrpcValue {
-    GrpcValue {
-        value_data: match value {
-            Value::Null => None,
-            Value::Boolean(v) => Some(ValueData::BoolValue(v)),
-            Value::UInt8(v) => Some(ValueData::U8Value(v as _)),
-            Value::UInt16(v) => Some(ValueData::U16Value(v as _)),
-            Value::UInt32(v) => Some(ValueData::U32Value(v)),
-            Value::UInt64(v) => Some(ValueData::U64Value(v)),
-            Value::Int8(v) => Some(ValueData::I8Value(v as _)),
-            Value::Int16(v) => Some(ValueData::I16Value(v as _)),
-            Value::Int32(v) => Some(ValueData::I32Value(v)),
-            Value::Int64(v) => Some(ValueData::I64Value(v)),
-            Value::Float32(v) => Some(ValueData::F32Value(*v)),
-            Value::Float64(v) => Some(ValueData::F64Value(*v)),
-            Value::String(v) => Some(ValueData::StringValue(v.as_utf8().to_string())),
-            Value::Binary(v) => Some(ValueData::BinaryValue(v.to_vec())),
-            Value::Date(v) => Some(ValueData::DateValue(v.val())),
-            Value::DateTime(v) => Some(ValueData::DatetimeValue(v.val())),
-            Value::Timestamp(v) => Some(match v.unit() {
-                TimeUnit::Second => ValueData::TimestampSecondValue(v.value()),
-                TimeUnit::Millisecond => ValueData::TimestampMillisecondValue(v.value()),
-                TimeUnit::Microsecond => ValueData::TimestampMicrosecondValue(v.value()),
-                TimeUnit::Nanosecond => ValueData::TimestampNanosecondValue(v.value()),
-            }),
-            Value::Time(v) => Some(match v.unit() {
-                TimeUnit::Second => ValueData::TimeSecondValue(v.value()),
-                TimeUnit::Millisecond => ValueData::TimeMillisecondValue(v.value()),
-                TimeUnit::Microsecond => ValueData::TimeMicrosecondValue(v.value()),
-                TimeUnit::Nanosecond => ValueData::TimeNanosecondValue(v.value()),
-            }),
-            Value::Interval(v) => Some(match v.unit() {
-                IntervalUnit::YearMonth => ValueData::IntervalYearMonthValues(v.to_i32()),
-                IntervalUnit::DayTime => ValueData::IntervalDayTimeValues(v.to_i64()),
-                IntervalUnit::MonthDayNano => {
-                    ValueData::IntervalMonthDayNanoValues(convert_i128_to_interval(v.to_i128()))
-                }
-            }),
-            Value::Duration(v) => Some(match v.unit() {
-                TimeUnit::Second => ValueData::DurationSecondValue(v.value()),
-                TimeUnit::Millisecond => ValueData::DurationMillisecondValue(v.value()),
-                TimeUnit::Microsecond => ValueData::DurationMicrosecondValue(v.value()),
-                TimeUnit::Nanosecond => ValueData::DurationNanosecondValue(v.value()),
-            }),
-            Value::List(_) => unreachable!(),
-        },
-    }
-}
-
 /// Returns true if the column type is equal to expected type.
 fn is_column_type_eq(column_type: ColumnDataType, expect_type: &ConcreteDataType) -> bool {
     if let Some(expect) = to_column_data_type(expect_type) {
@@ -993,16 +820,14 @@ mod tests {
     use std::sync::Arc;
 
     use datatypes::types::{
-        DurationMillisecondType, DurationSecondType, Int32Type, IntervalDayTimeType,
-        IntervalMonthDayNanoType, IntervalYearMonthType, TimeMillisecondType, TimeSecondType,
-        TimestampMillisecondType, TimestampSecondType, UInt32Type,
+        IntervalDayTimeType, IntervalMonthDayNanoType, IntervalYearMonthType, TimeMillisecondType,
+        TimeSecondType, TimestampMillisecondType, TimestampSecondType,
     };
     use datatypes::vectors::{
-        BooleanVector, DurationMicrosecondVector, DurationMillisecondVector,
-        DurationNanosecondVector, DurationSecondVector, IntervalDayTimeVector,
-        IntervalMonthDayNanoVector, IntervalYearMonthVector, TimeMicrosecondVector,
-        TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector,
-        TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, Vector,
+        BooleanVector, IntervalDayTimeVector, IntervalMonthDayNanoVector, IntervalYearMonthVector,
+        TimeMicrosecondVector, TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector,
+        TimestampMicrosecondVector, TimestampMillisecondVector, TimestampNanosecondVector,
+        TimestampSecondVector, Vector,
     };
     use paste::paste;
 
@@ -1063,7 +888,7 @@ mod tests {
         assert_eq!(2, values.capacity());
 
         let values = values_with_capacity(ColumnDataType::TimestampMillisecond, 2);
-        let values = values.timestamp_millisecond_values;
+        let values = values.ts_millisecond_values;
         assert_eq!(2, values.capacity());
 
         let values = values_with_capacity(ColumnDataType::TimeMillisecond, 2);
@@ -1077,10 +902,6 @@ mod tests {
         let values = values_with_capacity(ColumnDataType::IntervalMonthDayNano, 2);
         let values = values.interval_month_day_nano_values;
         assert_eq!(2, values.capacity());
-
-        let values = values_with_capacity(ColumnDataType::DurationMillisecond, 2);
-        let values = values.duration_millisecond_values;
-        assert_eq!(2, values.capacity());
     }
 
     #[test]
@@ -1165,10 +986,6 @@ mod tests {
             ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano),
             ColumnDataTypeWrapper(ColumnDataType::IntervalMonthDayNano).into()
         );
-        assert_eq!(
-            ConcreteDataType::duration_millisecond_datatype(),
-            ColumnDataTypeWrapper(ColumnDataType::DurationMillisecond).into()
-        )
     }
 
     #[test]
@@ -1257,12 +1074,6 @@ mod tests {
                 .try_into()
                 .unwrap()
         );
-        assert_eq!(
-            ColumnDataTypeWrapper(ColumnDataType::DurationMillisecond),
-            ConcreteDataType::duration_millisecond_datatype()
-                .try_into()
-                .unwrap()
-        );
 
         let result: Result<ColumnDataTypeWrapper> = ConcreteDataType::null_datatype().try_into();
         assert!(result.is_err());
@@ -1296,28 +1107,28 @@ mod tests {
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![1, 2, 3],
-            column.values.as_ref().unwrap().timestamp_nanosecond_values
+            column.values.as_ref().unwrap().ts_nanosecond_values
         );
 
         let vector = Arc::new(TimestampMillisecondVector::from_vec(vec![4, 5, 6]));
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![4, 5, 6],
-            column.values.as_ref().unwrap().timestamp_millisecond_values
+            column.values.as_ref().unwrap().ts_millisecond_values
         );
 
         let vector = Arc::new(TimestampMicrosecondVector::from_vec(vec![7, 8, 9]));
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![7, 8, 9],
-            column.values.as_ref().unwrap().timestamp_microsecond_values
+            column.values.as_ref().unwrap().ts_microsecond_values
         );
 
         let vector = Arc::new(TimestampSecondVector::from_vec(vec![10, 11, 12]));
         push_vals(&mut column, 3, vector);
         assert_eq!(
             vec![10, 11, 12],
-            column.values.as_ref().unwrap().timestamp_second_values
+            column.values.as_ref().unwrap().ts_second_values
         );
     }
 
@@ -1406,47 +1217,6 @@ mod tests {
         });
     }
 
-    #[test]
-    fn test_column_put_duration_values() {
-        let mut column = Column {
-            column_name: "test".to_string(),
-            semantic_type: 0,
-            values: Some(Values {
-                ..Default::default()
-            }),
-            null_mask: vec![],
-            datatype: 0,
-        };
-
-        let vector = Arc::new(DurationNanosecondVector::from_vec(vec![1, 2, 3]));
-        push_vals(&mut column, 3, vector);
-        assert_eq!(
-            vec![1, 2, 3],
-            column.values.as_ref().unwrap().duration_nanosecond_values
-        );
-
-        let vector = Arc::new(DurationMicrosecondVector::from_vec(vec![7, 8, 9]));
-        push_vals(&mut column, 3, vector);
-        assert_eq!(
-            vec![7, 8, 9],
-            column.values.as_ref().unwrap().duration_microsecond_values
-        );
-
-        let vector = Arc::new(DurationMillisecondVector::from_vec(vec![4, 5, 6]));
-        push_vals(&mut column, 3, vector);
-        assert_eq!(
-            vec![4, 5, 6],
-            column.values.as_ref().unwrap().duration_millisecond_values
-        );
-
-        let vector = Arc::new(DurationSecondVector::from_vec(vec![10, 11, 12]));
-        push_vals(&mut column, 3, vector);
-        assert_eq!(
-            vec![10, 11, 12],
-            column.values.as_ref().unwrap().duration_second_values
-        );
-    }
-
     #[test]
     fn test_column_put_vector() {
         use crate::v1::SemanticType;
@@ -1487,7 +1257,7 @@ mod tests {
         let actual = pb_values_to_values(
             &ConcreteDataType::Timestamp(TimestampType::Second(TimestampSecondType)),
             Values {
-                timestamp_second_values: vec![1_i64, 2_i64, 3_i64],
+                ts_second_values: vec![1_i64, 2_i64, 3_i64],
                 ..Default::default()
             },
         );
@@ -1502,7 +1272,7 @@ mod tests {
         let actual = pb_values_to_values(
             &ConcreteDataType::Timestamp(TimestampType::Millisecond(TimestampMillisecondType)),
             Values {
-                timestamp_millisecond_values: vec![1_i64, 2_i64, 3_i64],
+                ts_millisecond_values: vec![1_i64, 2_i64, 3_i64],
                 ..Default::default()
             },
         );
@@ -1547,39 +1317,6 @@ mod tests {
         assert_eq!(expect, actual);
     }
 
-    #[test]
-    fn test_convert_duration_values() {
-        // second
-        let actual = pb_values_to_values(
-            &ConcreteDataType::Duration(DurationType::Second(DurationSecondType)),
-            Values {
-                duration_second_values: vec![1_i64, 2_i64, 3_i64],
-                ..Default::default()
-            },
-        );
-        let expect = vec![
-            Value::Duration(Duration::new_second(1_i64)),
-            Value::Duration(Duration::new_second(2_i64)),
-            Value::Duration(Duration::new_second(3_i64)),
-        ];
-        assert_eq!(expect, actual);
-
-        // millisecond
-        let actual = pb_values_to_values(
-            &ConcreteDataType::Duration(DurationType::Millisecond(DurationMillisecondType)),
-            Values {
-                duration_millisecond_values: vec![1_i64, 2_i64, 3_i64],
-                ..Default::default()
-            },
-        );
-        let expect = vec![
-            Value::Duration(Duration::new_millisecond(1_i64)),
-            Value::Duration(Duration::new_millisecond(2_i64)),
-            Value::Duration(Duration::new_millisecond(3_i64)),
-        ];
-        assert_eq!(expect, actual);
-    }
-
     #[test]
     fn test_convert_interval_values() {
         // year_month
@@ -1787,76 +1524,4 @@ mod tests {
             Value::DateTime(3.into())
         ]
     );
-
-    #[test]
-    fn test_vectors_to_rows_for_different_types() {
-        let boolean_vec = BooleanVector::from_vec(vec![true, false, true]);
-        let int8_vec = PrimitiveVector::<Int8Type>::from_iter_values(vec![1, 2, 3]);
-        let int32_vec = PrimitiveVector::<Int32Type>::from_iter_values(vec![100, 200, 300]);
-        let uint8_vec = PrimitiveVector::<UInt8Type>::from_iter_values(vec![10, 20, 30]);
-        let uint32_vec = PrimitiveVector::<UInt32Type>::from_iter_values(vec![1000, 2000, 3000]);
-        let float32_vec = Float32Vector::from_vec(vec![1.1, 2.2, 3.3]);
-        let date_vec = DateVector::from_vec(vec![10, 20, 30]);
-        let string_vec = StringVector::from_vec(vec!["a", "b", "c"]);
-
-        let vector_refs: Vec<VectorRef> = vec![
-            Arc::new(boolean_vec),
-            Arc::new(int8_vec),
-            Arc::new(int32_vec),
-            Arc::new(uint8_vec),
-            Arc::new(uint32_vec),
-            Arc::new(float32_vec),
-            Arc::new(date_vec),
-            Arc::new(string_vec),
-        ];
-
-        let result = vectors_to_rows(vector_refs.iter(), 3);
-
-        assert_eq!(result.len(), 3);
-
-        assert_eq!(result[0].values.len(), 8);
-        let values = result[0]
-            .values
-            .iter()
-            .map(|v| v.value_data.clone().unwrap())
-            .collect::<Vec<_>>();
-        assert_eq!(values[0], ValueData::BoolValue(true));
-        assert_eq!(values[1], ValueData::I8Value(1));
-        assert_eq!(values[2], ValueData::I32Value(100));
-        assert_eq!(values[3], ValueData::U8Value(10));
-        assert_eq!(values[4], ValueData::U32Value(1000));
-        assert_eq!(values[5], ValueData::F32Value(1.1));
-        assert_eq!(values[6], ValueData::DateValue(10));
-        assert_eq!(values[7], ValueData::StringValue("a".to_string()));
-
-        assert_eq!(result[1].values.len(), 8);
-        let values = result[1]
-            .values
-            .iter()
-            .map(|v| v.value_data.clone().unwrap())
-            .collect::<Vec<_>>();
-        assert_eq!(values[0], ValueData::BoolValue(false));
-        assert_eq!(values[1], ValueData::I8Value(2));
-        assert_eq!(values[2], ValueData::I32Value(200));
-        assert_eq!(values[3], ValueData::U8Value(20));
-        assert_eq!(values[4], ValueData::U32Value(2000));
-        assert_eq!(values[5], ValueData::F32Value(2.2));
-        assert_eq!(values[6], ValueData::DateValue(20));
-        assert_eq!(values[7], ValueData::StringValue("b".to_string()));
-
-        assert_eq!(result[2].values.len(), 8);
-        let values = result[2]
-            .values
-            .iter()
-            .map(|v| v.value_data.clone().unwrap())
-            .collect::<Vec<_>>();
-        assert_eq!(values[0], ValueData::BoolValue(true));
-        assert_eq!(values[1], ValueData::I8Value(3));
-        assert_eq!(values[2], ValueData::I32Value(300));
-        assert_eq!(values[3], ValueData::U8Value(30));
-        assert_eq!(values[4], ValueData::U32Value(3000));
-        assert_eq!(values[5], ValueData::F32Value(3.3));
-        assert_eq!(values[6], ValueData::DateValue(30));
-        assert_eq!(values[7], ValueData::StringValue("c".to_string()));
-    }
 }

src/api/src/v1/column_def.rs

@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-
-use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, COMMENT_KEY};
+use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
@@ -22,7 +20,7 @@ use crate::helper::ColumnDataTypeWrapper;
 use crate::v1::ColumnDef;
 
 pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
-    let data_type = ColumnDataTypeWrapper::try_new(column_def.data_type)?;
+    let data_type = ColumnDataTypeWrapper::try_new(column_def.datatype)?;
 
     let constraint = if column_def.default_constraint.is_empty() {
         None
@@ -36,17 +34,9 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
         )
     };
 
-    let mut metadata = HashMap::new();
-    if !column_def.comment.is_empty() {
-        metadata.insert(COMMENT_KEY.to_string(), column_def.comment.clone());
-    }
-
-    Ok(
-        ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
-            .with_default_constraint(constraint)
-            .context(error::InvalidColumnDefaultConstraintSnafu {
-                column: &column_def.name,
-            })?
-            .with_metadata(metadata),
-    )
+    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
+        .with_default_constraint(constraint)
+        .context(error::InvalidColumnDefaultConstraintSnafu {
+            column: &column_def.name,
+        })
 }

src/auth/Cargo.toml

@@ -4,6 +4,8 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
 [features]
 default = []
 testing = []
@@ -12,7 +14,6 @@ testing = []
 api.workspace = true
 async-trait.workspace = true
 common-error.workspace = true
-common-macro.workspace = true
 digest = "0.10"
 hex = { version = "0.4" }
 secrecy = { version = "0.8", features = ["serde", "alloc"] }

src/auth/src/common.rs

@@ -26,7 +26,7 @@ use crate::{UserInfoRef, UserProviderRef};
 
 pub(crate) const DEFAULT_USERNAME: &str = "greptime";
 
-/// construct a [`UserInfo`](crate::user_info::UserInfo) impl with name
+/// construct a [`UserInfo`] impl with name
 /// use default username `greptime` if None is provided
 pub fn userinfo_by_name(username: Option<String>) -> UserInfoRef {
     DefaultUserInfo::with_name(username.unwrap_or_else(|| DEFAULT_USERNAME.to_string()))

src/auth/src/error.rs

@@ -14,12 +14,10 @@
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Invalid config value: {}, {}", value, msg))]
     InvalidConfig { value: String, msg: String },
@@ -30,14 +28,13 @@ pub enum Error {
     #[snafu(display("Internal state error: {}", msg))]
     InternalState { msg: String },
 
-    #[snafu(display("IO error"))]
+    #[snafu(display("IO error, source: {}", source))]
     Io {
-        #[snafu(source)]
-        error: std::io::Error,
+        source: std::io::Error,
         location: Location,
     },
 
-    #[snafu(display("Auth failed"))]
+    #[snafu(display("Auth failed, source: {}", source))]
     AuthBackend {
         location: Location,
         source: BoxedError,

src/auth/src/user_provider.rs

@@ -22,15 +22,15 @@ use crate::UserInfoRef;
 pub trait UserProvider: Send + Sync {
     fn name(&self) -> &str;
 
-    /// Checks whether a user is valid and allowed to access the database.
+    /// [`authenticate`] checks whether a user is valid and allowed to access the database.
     async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfoRef>;
 
-    /// Checks whether a connection request
+    /// [`authorize`] checks whether a connection request
     /// from a certain user to a certain catalog/schema is legal.
-    /// This method should be called after [authenticate()](UserProvider::authenticate()).
+    /// This method should be called after [`authenticate`].
     async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfoRef) -> Result<()>;
 
-    /// Combination of [authenticate()](UserProvider::authenticate()) and [authorize()](UserProvider::authorize()).
+    /// [`auth`] is a combination of [`authenticate`] and [`authorize`].
     /// In most cases it's preferred for both convenience and performance.
     async fn auth(
         &self,

src/catalog/Cargo.toml

@@ -16,7 +16,6 @@ async-trait = "0.1"
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
-common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
@@ -31,9 +30,8 @@ futures-util.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 metrics.workspace = true
-moka = { workspace = true, features = ["future"] }
+moka = { version = "0.11", features = ["future"] }
 parking_lot = "0.12"
-partition.workspace = true
 regex.workspace = true
 serde.workspace = true
 serde_json = "1.0"
@@ -48,6 +46,7 @@ catalog = { workspace = true, features = ["testing"] }
 chrono.workspace = true
 common-test-util = { workspace = true }
 log-store = { workspace = true }
+mito = { workspace = true, features = ["test"] }
 object-store = { workspace = true }
 storage = { workspace = true }
 tokio.workspace = true

src/catalog/src/error.rs

@@ -17,48 +17,54 @@ use std::fmt::Debug;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
-use table::metadata::TableId;
 use tokio::task::JoinError;
 
-#[derive(Snafu)]
+use crate::DeregisterTableRequest;
+
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to list catalogs"))]
+    #[snafu(display("Failed to list catalogs, source: {}", source))]
     ListCatalogs {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to list {}'s schemas", catalog))]
+    #[snafu(display("Failed to list {}'s schemas, source: {}", catalog, source))]
     ListSchemas {
         location: Location,
         catalog: String,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to re-compile script due to internal error"))]
+    #[snafu(display(
+        "Failed to re-compile script due to internal error, source: {}",
+        source
+    ))]
     CompileScriptInternal {
         location: Location,
         source: BoxedError,
     },
-    #[snafu(display("Failed to open system catalog table"))]
+    #[snafu(display("Failed to open system catalog table, source: {}", source))]
     OpenSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to create system catalog table"))]
+    #[snafu(display("Failed to create system catalog table, source: {}", source))]
     CreateSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to create table, table info: {}", table_info))]
+    #[snafu(display(
+        "Failed to create table, table info: {}, source: {}",
+        table_info,
+        source
+    ))]
     CreateTable {
         table_info: String,
         location: Location,
@@ -92,14 +98,13 @@ pub enum Error {
     #[snafu(display("Catalog value is not present"))]
     EmptyValue { location: Location },
 
-    #[snafu(display("Failed to deserialize value"))]
+    #[snafu(display("Failed to deserialize value, source: {}", source))]
     ValueDeserialize {
-        #[snafu(source)]
-        error: serde_json::error::Error,
+        source: serde_json::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Table engine not found: {}", engine_name))]
+    #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
     TableEngineNotFound {
         engine_name: String,
         location: Location,
@@ -137,18 +142,15 @@ pub enum Error {
     #[snafu(display("Operation {} not supported", op))]
     NotSupported { op: String, location: Location },
 
-    #[snafu(display("Failed to open table {table_id}"))]
+    #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
     OpenTable {
-        table_id: TableId,
+        table_info: String,
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to open table in parallel"))]
-    ParallelOpenTable {
-        #[snafu(source)]
-        error: JoinError,
-    },
+    #[snafu(display("Failed to open table in parallel, source: {}", source))]
+    ParallelOpenTable { source: JoinError },
 
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
     TableNotFound {
@@ -162,52 +164,72 @@ pub enum Error {
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to create recordbatch"))]
+    #[snafu(display("Failed to create recordbatch, source: {}", source))]
     CreateRecordBatch {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to insert table creation record to system catalog"))]
+    #[snafu(display(
+        "Failed to insert table creation record to system catalog, source: {}",
+        source
+    ))]
     InsertCatalogRecord {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to scan system catalog table"))]
+    #[snafu(display(
+        "Failed to deregister table, request: {:?}, source: {}",
+        request,
+        source
+    ))]
+    DeregisterTable {
+        request: DeregisterTableRequest,
+        location: Location,
+        source: table::error::Error,
+    },
+
+    #[snafu(display("Illegal catalog manager state: {}", msg))]
+    IllegalManagerState { location: Location, msg: String },
+
+    #[snafu(display("Failed to scan system catalog table, source: {}", source))]
     SystemCatalogTableScan {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display(""))]
+    #[snafu(display("{source}"))]
     Internal {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to upgrade weak catalog manager reference"))]
+    #[snafu(display(
+        "Failed to upgrade weak catalog manager reference. location: {}",
+        location
+    ))]
     UpgradeWeakCatalogManagerRef { location: Location },
 
-    #[snafu(display("Failed to execute system catalog table scan"))]
+    #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
     SystemCatalogTableScanExec {
         location: Location,
         source: common_query::error::Error,
     },
 
-    #[snafu(display("Cannot parse catalog value"))]
+    #[snafu(display("Cannot parse catalog value, source: {}", source))]
     InvalidCatalogValue {
         location: Location,
         source: common_catalog::error::Error,
     },
 
-    #[snafu(display("Failed to perform metasrv operation"))]
+    #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
     MetaSrv {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Invalid table info in catalog"))]
+    #[snafu(display("Invalid table info in catalog, source: {}", source))]
     InvalidTableInfoInCatalog {
         location: Location,
         source: datatypes::error::Error,
@@ -216,14 +238,17 @@ pub enum Error {
     #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
     QueryAccessDenied { catalog: String, schema: String },
 
-    #[snafu(display(""))]
+    #[snafu(display("Invalid system table definition: {err_msg}"))]
+    InvalidSystemTableDef { err_msg: String, location: Location },
+
+    #[snafu(display("{}: {}", msg, source))]
     Datafusion {
-        #[snafu(source)]
-        error: DataFusionError,
+        msg: String,
+        source: DataFusionError,
         location: Location,
     },
 
-    #[snafu(display("Table schema mismatch"))]
+    #[snafu(display("Table schema mismatch, source: {}", source))]
     TableSchemaMismatch {
         location: Location,
         source: table::error::Error,
@@ -232,7 +257,7 @@ pub enum Error {
     #[snafu(display("A generic error has occurred, msg: {}", msg))]
     Generic { msg: String, location: Location },
 
-    #[snafu(display("Table metadata manager error"))]
+    #[snafu(display("Table metadata manager error: {}", source))]
     TableMetadataManager {
         source: common_meta::error::Error,
         location: Location,
@@ -247,8 +272,10 @@ impl ErrorExt for Error {
             Error::InvalidKey { .. }
             | Error::SchemaNotFound { .. }
             | Error::TableNotFound { .. }
+            | Error::IllegalManagerState { .. }
             | Error::CatalogNotFound { .. }
             | Error::InvalidEntryType { .. }
+            | Error::InvalidSystemTableDef { .. }
             | Error::ParallelOpenTable { .. } => StatusCode::Unexpected,
 
             Error::SystemCatalog { .. }
@@ -279,6 +306,7 @@ impl ErrorExt for Error {
             | Error::InsertCatalogRecord { source, .. }
             | Error::OpenTable { source, .. }
             | Error::CreateTable { source, .. }
+            | Error::DeregisterTable { source, .. }
             | Error::TableSchemaMismatch { source, .. } => source.status_code(),
 
             Error::MetaSrv { source, .. } => source.status_code(),

src/catalog/src/information_schema/columns.rs

@@ -158,7 +158,7 @@ impl InformationSchemaColumnsBuilder {
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
-                .schema_exists(&catalog_name, &schema_name)
+                .schema_exist(&catalog_name, &schema_name)
                 .await?
             {
                 continue;

src/catalog/src/information_schema/tables.rs

@@ -154,7 +154,7 @@ impl InformationSchemaTablesBuilder {
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
-                .schema_exists(&catalog_name, &schema_name)
+                .schema_exist(&catalog_name, &schema_name)
                 .await?
             {
                 continue;

src/catalog/src/kvbackend/manager.rs

@@ -1,292 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::collections::BTreeSet;
-use std::sync::{Arc, Weak};
-
-use common_catalog::consts::{DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, NUMBERS_TABLE_ID};
-use common_error::ext::BoxedError;
-use common_meta::cache_invalidator::{CacheInvalidator, CacheInvalidatorRef, Context};
-use common_meta::datanode_manager::DatanodeManagerRef;
-use common_meta::error::Result as MetaResult;
-use common_meta::key::catalog_name::CatalogNameKey;
-use common_meta::key::schema_name::SchemaNameKey;
-use common_meta::key::table_name::TableNameKey;
-use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
-use common_meta::kv_backend::KvBackendRef;
-use common_meta::table_name::TableName;
-use futures_util::TryStreamExt;
-use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
-use snafu::prelude::*;
-use table::dist_table::DistTable;
-use table::metadata::TableId;
-use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
-use table::TableRef;
-
-use crate::error::{
-    self as catalog_err, ListCatalogsSnafu, ListSchemasSnafu, Result as CatalogResult,
-    TableMetadataManagerSnafu,
-};
-use crate::information_schema::{InformationSchemaProvider, COLUMNS, TABLES};
-use crate::CatalogManager;
-
-/// Access all existing catalog, schema and tables.
-///
-/// The result comes from two source, all the user tables are presented in
-/// a kv-backend which persists the metadata of a table. And system tables
-/// comes from `SystemCatalog`, which is static and read-only.
-#[derive(Clone)]
-pub struct KvBackendCatalogManager {
-    // TODO(LFC): Maybe use a real implementation for Standalone mode.
-    // Now we use `NoopKvCacheInvalidator` for Standalone mode. In Standalone mode, the KV backend
-    // is implemented by RaftEngine. Maybe we need a cache for it?
-    cache_invalidator: CacheInvalidatorRef,
-    partition_manager: PartitionRuleManagerRef,
-    table_metadata_manager: TableMetadataManagerRef,
-    datanode_manager: DatanodeManagerRef,
-    /// A sub-CatalogManager that handles system tables
-    system_catalog: SystemCatalog,
-}
-
-#[async_trait::async_trait]
-impl CacheInvalidator for KvBackendCatalogManager {
-    async fn invalidate_table_name(&self, ctx: &Context, table_name: TableName) -> MetaResult<()> {
-        self.cache_invalidator
-            .invalidate_table_name(ctx, table_name)
-            .await
-    }
-
-    async fn invalidate_table_id(&self, ctx: &Context, table_id: TableId) -> MetaResult<()> {
-        self.cache_invalidator
-            .invalidate_table_id(ctx, table_id)
-            .await
-    }
-}
-
-impl KvBackendCatalogManager {
-    pub fn new(
-        backend: KvBackendRef,
-        cache_invalidator: CacheInvalidatorRef,
-        datanode_manager: DatanodeManagerRef,
-    ) -> Arc<Self> {
-        Arc::new_cyclic(|me| Self {
-            partition_manager: Arc::new(PartitionRuleManager::new(backend.clone())),
-            table_metadata_manager: Arc::new(TableMetadataManager::new(backend)),
-            cache_invalidator,
-            datanode_manager,
-            system_catalog: SystemCatalog {
-                catalog_manager: me.clone(),
-            },
-        })
-    }
-
-    pub fn partition_manager(&self) -> PartitionRuleManagerRef {
-        self.partition_manager.clone()
-    }
-
-    pub fn table_metadata_manager_ref(&self) -> &TableMetadataManagerRef {
-        &self.table_metadata_manager
-    }
-
-    pub fn datanode_manager(&self) -> DatanodeManagerRef {
-        self.datanode_manager.clone()
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogManager for KvBackendCatalogManager {
-    async fn catalog_names(&self) -> CatalogResult<Vec<String>> {
-        let stream = self
-            .table_metadata_manager
-            .catalog_manager()
-            .catalog_names()
-            .await;
-
-        let keys = stream
-            .try_collect::<Vec<_>>()
-            .await
-            .map_err(BoxedError::new)
-            .context(ListCatalogsSnafu)?;
-
-        Ok(keys)
-    }
-
-    async fn schema_names(&self, catalog: &str) -> CatalogResult<Vec<String>> {
-        let stream = self
-            .table_metadata_manager
-            .schema_manager()
-            .schema_names(catalog)
-            .await;
-        let mut keys = stream
-            .try_collect::<BTreeSet<_>>()
-            .await
-            .map_err(BoxedError::new)
-            .context(ListSchemasSnafu { catalog })?
-            .into_iter()
-            .collect::<Vec<_>>();
-
-        keys.extend_from_slice(&self.system_catalog.schema_names());
-
-        Ok(keys)
-    }
-
-    async fn table_names(&self, catalog: &str, schema: &str) -> CatalogResult<Vec<String>> {
-        let mut tables = self
-            .table_metadata_manager
-            .table_name_manager()
-            .tables(catalog, schema)
-            .await
-            .context(TableMetadataManagerSnafu)?
-            .into_iter()
-            .map(|(k, _)| k)
-            .collect::<Vec<String>>();
-        tables.extend_from_slice(&self.system_catalog.table_names(schema));
-
-        Ok(tables)
-    }
-
-    async fn catalog_exists(&self, catalog: &str) -> CatalogResult<bool> {
-        self.table_metadata_manager
-            .catalog_manager()
-            .exists(CatalogNameKey::new(catalog))
-            .await
-            .context(TableMetadataManagerSnafu)
-    }
-
-    async fn schema_exists(&self, catalog: &str, schema: &str) -> CatalogResult<bool> {
-        if self.system_catalog.schema_exist(schema) {
-            return Ok(true);
-        }
-
-        self.table_metadata_manager
-            .schema_manager()
-            .exists(SchemaNameKey::new(catalog, schema))
-            .await
-            .context(TableMetadataManagerSnafu)
-    }
-
-    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> CatalogResult<bool> {
-        if self.system_catalog.table_exist(schema, table) {
-            return Ok(true);
-        }
-
-        let key = TableNameKey::new(catalog, schema, table);
-        self.table_metadata_manager
-            .table_name_manager()
-            .get(key)
-            .await
-            .context(TableMetadataManagerSnafu)
-            .map(|x| x.is_some())
-    }
-
-    async fn table(
-        &self,
-        catalog: &str,
-        schema: &str,
-        table_name: &str,
-    ) -> CatalogResult<Option<TableRef>> {
-        if let Some(table) = self.system_catalog.table(catalog, schema, table_name) {
-            return Ok(Some(table));
-        }
-
-        let key = TableNameKey::new(catalog, schema, table_name);
-        let Some(table_name_value) = self
-            .table_metadata_manager
-            .table_name_manager()
-            .get(key)
-            .await
-            .context(TableMetadataManagerSnafu)?
-        else {
-            return Ok(None);
-        };
-        let table_id = table_name_value.table_id();
-
-        let Some(table_info_value) = self
-            .table_metadata_manager
-            .table_info_manager()
-            .get(table_id)
-            .await
-            .context(TableMetadataManagerSnafu)?
-            .map(|v| v.into_inner())
-        else {
-            return Ok(None);
-        };
-        let table_info = Arc::new(
-            table_info_value
-                .table_info
-                .try_into()
-                .context(catalog_err::InvalidTableInfoInCatalogSnafu)?,
-        );
-        Ok(Some(DistTable::table(table_info)))
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-// TODO: This struct can hold a static map of all system tables when
-// the upper layer (e.g., procedure) can inform the catalog manager
-// a new catalog is created.
-/// Existing system tables:
-/// - public.numbers
-/// - information_schema.tables
-/// - information_schema.columns
-#[derive(Clone)]
-struct SystemCatalog {
-    catalog_manager: Weak<KvBackendCatalogManager>,
-}
-
-impl SystemCatalog {
-    fn schema_names(&self) -> Vec<String> {
-        vec![INFORMATION_SCHEMA_NAME.to_string()]
-    }
-
-    fn table_names(&self, schema: &str) -> Vec<String> {
-        if schema == INFORMATION_SCHEMA_NAME {
-            vec![TABLES.to_string(), COLUMNS.to_string()]
-        } else if schema == DEFAULT_SCHEMA_NAME {
-            vec![NUMBERS_TABLE_NAME.to_string()]
-        } else {
-            vec![]
-        }
-    }
-
-    fn schema_exist(&self, schema: &str) -> bool {
-        schema == INFORMATION_SCHEMA_NAME
-    }
-
-    fn table_exist(&self, schema: &str, table: &str) -> bool {
-        if schema == INFORMATION_SCHEMA_NAME {
-            table == TABLES || table == COLUMNS
-        } else if schema == DEFAULT_SCHEMA_NAME {
-            table == NUMBERS_TABLE_NAME
-        } else {
-            false
-        }
-    }
-
-    fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Option<TableRef> {
-        if schema == INFORMATION_SCHEMA_NAME {
-            let information_schema_provider =
-                InformationSchemaProvider::new(catalog.to_string(), self.catalog_manager.clone());
-            information_schema_provider.table(table_name)
-        } else if schema == DEFAULT_SCHEMA_NAME && table_name == NUMBERS_TABLE_NAME {
-            Some(NumbersTable::table(NUMBERS_TABLE_ID))
-        } else {
-            None
-        }
-    }
-}

src/catalog/src/lib.rs

@@ -17,38 +17,79 @@
 #![feature(try_blocks)]
 
 use std::any::Any;
+use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use futures::future::BoxFuture;
-use table::metadata::TableId;
+use api::v1::meta::{RegionStat, TableIdent, TableName};
+use common_telemetry::{info, warn};
+use snafu::ResultExt;
+use table::engine::{EngineContext, TableEngineRef};
+use table::metadata::{TableId, TableType};
 use table::requests::CreateTableRequest;
 use table::TableRef;
 
-use crate::error::Result;
+use crate::error::{CreateTableSnafu, Result};
 
 pub mod error;
 pub mod information_schema;
-pub mod kvbackend;
-pub mod memory;
+pub mod local;
 mod metrics;
+pub mod remote;
+pub mod system;
 pub mod table_source;
+pub mod tables;
 
 #[async_trait::async_trait]
 pub trait CatalogManager: Send + Sync {
     fn as_any(&self) -> &dyn Any;
 
+    /// Starts a catalog manager.
+    async fn start(&self) -> Result<()>;
+
+    /// Registers a catalog to catalog manager, returns whether the catalog exist before.
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool>;
+
+    /// Register a schema with catalog name and schema name. Retuens whether the
+    /// schema registered.
+    ///
+    /// # Errors
+    ///
+    /// This method will/should fail if catalog not exist
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool>;
+
+    /// Deregisters a database within given catalog/schema to catalog manager
+    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool>;
+
+    /// Registers a table within given catalog/schema to catalog manager,
+    /// returns whether the table registered.
+    ///
+    /// # Errors
+    ///
+    /// This method will/should fail if catalog or schema not exist
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
+
+    /// Deregisters a table within given catalog/schema to catalog manager
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()>;
+
+    /// Rename a table to [RenameTableRequest::new_table_name], returns whether the table is renamed.
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool>;
+
+    /// Register a system table, should be called before starting the manager.
+    async fn register_system_table(&self, request: RegisterSystemTableRequest)
+        -> error::Result<()>;
+
     async fn catalog_names(&self) -> Result<Vec<String>>;
 
     async fn schema_names(&self, catalog: &str) -> Result<Vec<String>>;
 
     async fn table_names(&self, catalog: &str, schema: &str) -> Result<Vec<String>>;
 
-    async fn catalog_exists(&self, catalog: &str) -> Result<bool>;
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool>;
 
-    async fn schema_exists(&self, catalog: &str, schema: &str) -> Result<bool>;
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool>;
 
-    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
 
     /// Returns the table by catalog, schema and table name.
     async fn table(
@@ -62,8 +103,7 @@ pub trait CatalogManager: Send + Sync {
 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
 
 /// Hook called after system table opening.
-pub type OpenSystemTableHook =
-    Box<dyn Fn(TableRef) -> BoxFuture<'static, Result<()>> + Send + Sync>;
+pub type OpenSystemTableHook = Arc<dyn Fn(TableRef) -> Result<()> + Send + Sync>;
 
 /// Register system table request:
 /// - When system table is already created and registered, the hook will be called
@@ -122,3 +162,120 @@ pub struct RegisterSchemaRequest {
     pub catalog: String,
     pub schema: String,
 }
+
+pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
+    manager: &'a M,
+    engine: TableEngineRef,
+    sys_table_requests: &'a mut Vec<RegisterSystemTableRequest>,
+) -> Result<()> {
+    for req in sys_table_requests.drain(..) {
+        let catalog_name = &req.create_table_request.catalog_name;
+        let schema_name = &req.create_table_request.schema_name;
+        let table_name = &req.create_table_request.table_name;
+        let table_id = req.create_table_request.id;
+
+        let table = manager.table(catalog_name, schema_name, table_name).await?;
+        let table = if let Some(table) = table {
+            table
+        } else {
+            let table = engine
+                .create_table(&EngineContext::default(), req.create_table_request.clone())
+                .await
+                .with_context(|_| CreateTableSnafu {
+                    table_info: common_catalog::format_full_table_name(
+                        catalog_name,
+                        schema_name,
+                        table_name,
+                    ),
+                })?;
+            let _ = manager
+                .register_table(RegisterTableRequest {
+                    catalog: catalog_name.clone(),
+                    schema: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    table_id,
+                    table: table.clone(),
+                })
+                .await?;
+            info!("Created and registered system table: {table_name}");
+            table
+        };
+        if let Some(hook) = req.open_hook {
+            (hook)(table)?;
+        }
+    }
+    Ok(())
+}
+
+/// The stat of regions in the datanode node.
+/// The number of regions can be got from len of vec.
+///
+/// Ignores any errors occurred during iterating regions. The intention of this method is to
+/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
+/// "try our best" job.
+pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
+    let mut region_number: u64 = 0;
+    let mut region_stats = Vec::new();
+
+    let Ok(catalog_names) = catalog_manager.catalog_names().await else {
+        return (region_number, region_stats);
+    };
+    for catalog_name in catalog_names {
+        let Ok(schema_names) = catalog_manager.schema_names(&catalog_name).await else {
+            continue;
+        };
+        for schema_name in schema_names {
+            let Ok(table_names) = catalog_manager
+                .table_names(&catalog_name, &schema_name)
+                .await
+            else {
+                continue;
+            };
+            for table_name in table_names {
+                let Ok(Some(table)) = catalog_manager
+                    .table(&catalog_name, &schema_name, &table_name)
+                    .await
+                else {
+                    continue;
+                };
+
+                if table.table_type() != TableType::Base {
+                    continue;
+                }
+
+                let table_info = table.table_info();
+                let region_numbers = &table_info.meta.region_numbers;
+                region_number += region_numbers.len() as u64;
+
+                let engine = &table_info.meta.engine;
+                let table_id = table_info.ident.table_id;
+
+                match table.region_stats() {
+                    Ok(stats) => {
+                        let stats = stats.into_iter().map(|stat| RegionStat {
+                            region_id: stat.region_id,
+                            table_ident: Some(TableIdent {
+                                table_id,
+                                table_name: Some(TableName {
+                                    catalog_name: catalog_name.clone(),
+                                    schema_name: schema_name.clone(),
+                                    table_name: table_name.clone(),
+                                }),
+                                engine: engine.clone(),
+                            }),
+                            approximate_bytes: stat.disk_usage_bytes as i64,
+                            attrs: HashMap::from([("engine_name".to_owned(), engine.clone())]),
+                            ..Default::default()
+                        });
+
+                        region_stats.extend(stats);
+                    }
+                    Err(e) => {
+                        warn!("Failed to get region status, err: {:?}", e);
+                    }
+                };
+            }
+        }
+    }
+    (region_number, region_stats)
+}

src/catalog/src/local.rs

@@ -0,0 +1,19 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod manager;
+pub mod memory;
+
+pub use manager::LocalCatalogManager;
+pub use memory::{new_memory_catalog_manager, MemoryCatalogManager};

src/catalog/src/local/manager.rs

@@ -0,0 +1,633 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
+    MITO_ENGINE, NUMBERS_TABLE_ID, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID,
+    SYSTEM_CATALOG_TABLE_NAME,
+};
+use common_catalog::format_full_table_name;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use common_telemetry::{error, info};
+use datatypes::prelude::ScalarVector;
+use datatypes::vectors::{BinaryVector, UInt8Vector};
+use futures_util::lock::Mutex;
+use metrics::increment_gauge;
+use snafu::{ensure, OptionExt, ResultExt};
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::EngineContext;
+use table::metadata::TableId;
+use table::requests::OpenTableRequest;
+use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+use table::table::TableIdProvider;
+use table::TableRef;
+
+use crate::error::{
+    self, CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu,
+    Result, SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu,
+    SystemCatalogTypeMismatchSnafu, TableEngineNotFoundSnafu, TableExistsSnafu, TableNotExistSnafu,
+    TableNotFoundSnafu, UnimplementedSnafu,
+};
+use crate::local::memory::MemoryCatalogManager;
+use crate::system::{
+    decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
+    VALUE_INDEX,
+};
+use crate::tables::SystemCatalog;
+use crate::{
+    handle_system_table_request, CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest,
+    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
+};
+
+/// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
+pub struct LocalCatalogManager {
+    system: Arc<SystemCatalog>,
+    catalogs: Arc<MemoryCatalogManager>,
+    engine_manager: TableEngineManagerRef,
+    next_table_id: AtomicU32,
+    init_lock: Mutex<bool>,
+    register_lock: Mutex<()>,
+    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+}
+
+impl LocalCatalogManager {
+    /// Create a new [CatalogManager] with given user catalogs and mito engine
+    pub async fn try_new(engine_manager: TableEngineManagerRef) -> Result<Self> {
+        let engine = engine_manager
+            .engine(MITO_ENGINE)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: MITO_ENGINE,
+            })?;
+        let table = SystemCatalogTable::new(engine.clone()).await?;
+        let memory_catalog_manager = crate::local::memory::new_memory_catalog_manager()?;
+        let system_catalog = Arc::new(SystemCatalog::new(table));
+        Ok(Self {
+            system: system_catalog,
+            catalogs: memory_catalog_manager,
+            engine_manager,
+            next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
+            init_lock: Mutex::new(false),
+            register_lock: Mutex::new(()),
+            system_table_requests: Mutex::new(Vec::default()),
+        })
+    }
+
+    /// Scan all entries from system catalog table
+    pub async fn init(&self) -> Result<()> {
+        self.init_system_catalog().await?;
+        let system_records = self.system.information_schema.system.records().await?;
+        let entries = self.collect_system_catalog_entries(system_records).await?;
+        let max_table_id = self.handle_system_catalog_entries(entries).await?;
+
+        info!(
+            "All system catalog entries processed, max table id: {}",
+            max_table_id
+        );
+        self.next_table_id
+            .store((max_table_id + 1).max(MIN_USER_TABLE_ID), Ordering::Relaxed);
+        *self.init_lock.lock().await = true;
+
+        // Processing system table hooks
+        let mut sys_table_requests = self.system_table_requests.lock().await;
+        let engine = self
+            .engine_manager
+            .engine(MITO_ENGINE)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: MITO_ENGINE,
+            })?;
+
+        handle_system_table_request(self, engine, &mut sys_table_requests).await?;
+        Ok(())
+    }
+
+    async fn init_system_catalog(&self) -> Result<()> {
+        // register default catalog and default schema
+        self.catalogs
+            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())?;
+        self.catalogs.register_schema_sync(RegisterSchemaRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+        })?;
+
+        // register SystemCatalogTable
+        self.catalogs
+            .register_catalog_sync(SYSTEM_CATALOG_NAME.to_string())?;
+        self.catalogs.register_schema_sync(RegisterSchemaRequest {
+            catalog: SYSTEM_CATALOG_NAME.to_string(),
+            schema: INFORMATION_SCHEMA_NAME.to_string(),
+        })?;
+        let register_table_req = RegisterTableRequest {
+            catalog: SYSTEM_CATALOG_NAME.to_string(),
+            schema: INFORMATION_SCHEMA_NAME.to_string(),
+            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+            table_id: SYSTEM_CATALOG_TABLE_ID,
+            table: self.system.information_schema.system.as_table_ref(),
+        };
+        self.catalogs.register_table(register_table_req).await?;
+
+        // Add numbers table for test
+        let register_number_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: NUMBERS_TABLE_NAME.to_string(),
+            table_id: NUMBERS_TABLE_ID,
+            table: NumbersTable::table(NUMBERS_TABLE_ID),
+        };
+
+        self.catalogs
+            .register_table(register_number_table_req)
+            .await?;
+
+        Ok(())
+    }
+
+    /// Collect stream of system catalog entries to `Vec<Entry>`
+    async fn collect_system_catalog_entries(
+        &self,
+        stream: SendableRecordBatchStream,
+    ) -> Result<Vec<Entry>> {
+        let record_batch = common_recordbatch::util::collect(stream)
+            .await
+            .context(ReadSystemCatalogSnafu)?;
+        let rbs = record_batch
+            .into_iter()
+            .map(Self::record_batch_to_entry)
+            .collect::<Result<Vec<_>>>()?;
+        Ok(rbs.into_iter().flat_map(Vec::into_iter).collect::<_>())
+    }
+
+    /// Convert `RecordBatch` to a vector of `Entry`.
+    fn record_batch_to_entry(rb: RecordBatch) -> Result<Vec<Entry>> {
+        ensure!(
+            rb.num_columns() >= 6,
+            SystemCatalogSnafu {
+                msg: format!("Length mismatch: {}", rb.num_columns())
+            }
+        );
+
+        let entry_type = rb
+            .column(ENTRY_TYPE_INDEX)
+            .as_any()
+            .downcast_ref::<UInt8Vector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(ENTRY_TYPE_INDEX).data_type(),
+            })?;
+
+        let key = rb
+            .column(KEY_INDEX)
+            .as_any()
+            .downcast_ref::<BinaryVector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(KEY_INDEX).data_type(),
+            })?;
+
+        let value = rb
+            .column(VALUE_INDEX)
+            .as_any()
+            .downcast_ref::<BinaryVector>()
+            .with_context(|| SystemCatalogTypeMismatchSnafu {
+                data_type: rb.column(VALUE_INDEX).data_type(),
+            })?;
+
+        let mut res = Vec::with_capacity(rb.num_rows());
+        for ((t, k), v) in entry_type
+            .iter_data()
+            .zip(key.iter_data())
+            .zip(value.iter_data())
+        {
+            let entry = decode_system_catalog(t, k, v)?;
+            res.push(entry);
+        }
+        Ok(res)
+    }
+
+    /// Processes records from system catalog table and returns the max table id persisted
+    /// in system catalog table.
+    async fn handle_system_catalog_entries(&self, entries: Vec<Entry>) -> Result<TableId> {
+        let entries = Self::sort_entries(entries);
+        let mut max_table_id = 0;
+        for entry in entries {
+            match entry {
+                Entry::Catalog(c) => {
+                    self.catalogs
+                        .register_catalog_sync(c.catalog_name.clone())?;
+                    info!("Register catalog: {}", c.catalog_name);
+                }
+                Entry::Schema(s) => {
+                    let req = RegisterSchemaRequest {
+                        catalog: s.catalog_name.clone(),
+                        schema: s.schema_name.clone(),
+                    };
+                    let _ = self.catalogs.register_schema_sync(req)?;
+                    info!("Registered schema: {:?}", s);
+                }
+                Entry::Table(t) => {
+                    max_table_id = max_table_id.max(t.table_id);
+                    if t.is_deleted {
+                        continue;
+                    }
+                    self.open_and_register_table(&t).await?;
+                    info!("Registered table: {:?}", t);
+                }
+            }
+        }
+        Ok(max_table_id)
+    }
+
+    /// Sort catalog entries to ensure catalog entries comes first, then schema entries,
+    /// and table entries is the last.
+    fn sort_entries(mut entries: Vec<Entry>) -> Vec<Entry> {
+        entries.sort();
+        entries
+    }
+
+    async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
+        self.check_catalog_schema_exist(&t.catalog_name, &t.schema_name)
+            .await?;
+
+        let context = EngineContext {};
+        let open_request = OpenTableRequest {
+            catalog_name: t.catalog_name.clone(),
+            schema_name: t.schema_name.clone(),
+            table_name: t.table_name.clone(),
+            table_id: t.table_id,
+            region_numbers: vec![0],
+        };
+        let engine = self
+            .engine_manager
+            .engine(&t.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &t.engine,
+            })?;
+
+        let table_ref = engine
+            .open_table(&context, open_request)
+            .await
+            .with_context(|_| OpenTableSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id: {}",
+                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
+                ),
+            })?
+            .with_context(|| TableNotFoundSnafu {
+                table_info: format!(
+                    "{}.{}.{}, id: {}",
+                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
+                ),
+            })?;
+
+        let register_request = RegisterTableRequest {
+            catalog: t.catalog_name.clone(),
+            schema: t.schema_name.clone(),
+            table_name: t.table_name.clone(),
+            table_id: t.table_id,
+            table: table_ref,
+        };
+        let _ = self.catalogs.register_table(register_request).await?;
+
+        Ok(())
+    }
+
+    async fn check_state(&self) -> Result<()> {
+        let started = self.init_lock.lock().await;
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+        Ok(())
+    }
+
+    async fn check_catalog_schema_exist(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+    ) -> Result<()> {
+        if !self.catalogs.catalog_exist(catalog_name).await? {
+            return CatalogNotFoundSnafu { catalog_name }.fail()?;
+        }
+        if !self
+            .catalogs
+            .schema_exist(catalog_name, schema_name)
+            .await?
+        {
+            return SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            }
+            .fail()?;
+        }
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl TableIdProvider for LocalCatalogManager {
+    async fn next_table_id(&self) -> table::Result<TableId> {
+        Ok(self.next_table_id.fetch_add(1, Ordering::Relaxed))
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for LocalCatalogManager {
+    /// Start [LocalCatalogManager] to load all information from system catalog table.
+    /// Make sure table engine is initialized before starting [MemoryCatalogManager].
+    async fn start(&self) -> Result<()> {
+        self.init().await
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        self.check_state().await?;
+
+        let catalog_name = request.catalog.clone();
+        let schema_name = request.schema.clone();
+
+        self.check_catalog_schema_exist(&catalog_name, &schema_name)
+            .await?;
+
+        {
+            let _lock = self.register_lock.lock().await;
+            if let Some(existing) = self
+                .catalogs
+                .table(&request.catalog, &request.schema, &request.table_name)
+                .await?
+            {
+                if existing.table_info().ident.table_id != request.table_id {
+                    error!(
+                        "Unexpected table register request: {:?}, existing: {:?}",
+                        request,
+                        existing.table_info()
+                    );
+                    return TableExistsSnafu {
+                        table: format_full_table_name(
+                            &catalog_name,
+                            &schema_name,
+                            &request.table_name,
+                        ),
+                    }
+                    .fail();
+                }
+                // Try to register table with same table id, just ignore.
+                Ok(false)
+            } else {
+                // table does not exist
+                let engine = request.table.table_info().meta.engine.to_string();
+                let table_name = request.table_name.clone();
+                let table_id = request.table_id;
+                let _ = self.catalogs.register_table(request).await?;
+                let _ = self
+                    .system
+                    .register_table(
+                        catalog_name.clone(),
+                        schema_name.clone(),
+                        table_name,
+                        table_id,
+                        engine,
+                    )
+                    .await?;
+                increment_gauge!(
+                    crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+                    1.0,
+                    &[crate::metrics::db_label(&catalog_name, &schema_name)],
+                );
+                Ok(true)
+            }
+        }
+    }
+
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
+        self.check_state().await?;
+
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        self.check_catalog_schema_exist(catalog_name, schema_name)
+            .await?;
+        ensure!(
+            self.catalogs
+                .table(catalog_name, schema_name, &request.new_table_name)
+                .await?
+                .is_none(),
+            TableExistsSnafu {
+                table: &request.new_table_name
+            }
+        );
+
+        let _lock = self.register_lock.lock().await;
+        let old_table = self
+            .catalogs
+            .table(catalog_name, schema_name, &request.table_name)
+            .await?
+            .context(TableNotExistSnafu {
+                table: &request.table_name,
+            })?;
+
+        let engine = old_table.table_info().meta.engine.to_string();
+        // rename table in system catalog
+        let _ = self
+            .system
+            .register_table(
+                catalog_name.clone(),
+                schema_name.clone(),
+                request.new_table_name.clone(),
+                request.table_id,
+                engine,
+            )
+            .await?;
+
+        self.catalogs.rename_table(request).await
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
+        self.check_state().await?;
+
+        {
+            let _ = self.register_lock.lock().await;
+
+            let DeregisterTableRequest {
+                catalog,
+                schema,
+                table_name,
+            } = &request;
+            let table_id = self
+                .catalogs
+                .table(catalog, schema, table_name)
+                .await?
+                .with_context(|| error::TableNotExistSnafu {
+                    table: format_full_table_name(catalog, schema, table_name),
+                })?
+                .table_info()
+                .ident
+                .table_id;
+
+            self.system.deregister_table(&request, table_id).await?;
+            self.catalogs.deregister_table(request).await
+        }
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        self.check_state().await?;
+
+        let catalog_name = &request.catalog;
+        let schema_name = &request.schema;
+
+        if !self.catalogs.catalog_exist(catalog_name).await? {
+            return CatalogNotFoundSnafu { catalog_name }.fail()?;
+        }
+
+        {
+            let _lock = self.register_lock.lock().await;
+            ensure!(
+                !self
+                    .catalogs
+                    .schema_exist(catalog_name, schema_name)
+                    .await?,
+                SchemaExistsSnafu {
+                    schema: schema_name,
+                }
+            );
+            let _ = self
+                .system
+                .register_schema(request.catalog.clone(), schema_name.clone())
+                .await?;
+            self.catalogs.register_schema_sync(request)
+        }
+    }
+
+    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
+        UnimplementedSnafu {
+            operation: "deregister schema",
+        }
+        .fail()
+    }
+
+    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        let catalog_name = request.create_table_request.catalog_name.clone();
+        let schema_name = request.create_table_request.schema_name.clone();
+
+        let mut sys_table_requests = self.system_table_requests.lock().await;
+        sys_table_requests.push(request);
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&catalog_name, &schema_name)],
+        );
+        Ok(())
+    }
+
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
+        self.catalogs.schema_exist(catalog, schema).await
+    }
+
+    async fn table(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        self.catalogs
+            .table(catalog_name, schema_name, table_name)
+            .await
+    }
+
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
+        if catalog.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
+            Ok(true)
+        } else {
+            self.catalogs.catalog_exist(catalog).await
+        }
+    }
+
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+        self.catalogs.table_exist(catalog, schema, table).await
+    }
+
+    async fn catalog_names(&self) -> Result<Vec<String>> {
+        self.catalogs.catalog_names().await
+    }
+
+    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
+        self.catalogs.schema_names(catalog_name).await
+    }
+
+    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
+        self.catalogs.table_names(catalog_name, schema_name).await
+    }
+
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
+        self.catalogs.clone().register_catalog(name).await
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+
+    use mito::engine::MITO_ENGINE;
+
+    use super::*;
+    use crate::system::{CatalogEntry, SchemaEntry};
+
+    #[test]
+    fn test_sort_entry() {
+        let vec = vec![
+            Entry::Table(TableEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+                table_name: "T1".to_string(),
+                table_id: 1,
+                engine: MITO_ENGINE.to_string(),
+                is_deleted: false,
+            }),
+            Entry::Catalog(CatalogEntry {
+                catalog_name: "C2".to_string(),
+            }),
+            Entry::Schema(SchemaEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+            }),
+            Entry::Schema(SchemaEntry {
+                catalog_name: "C2".to_string(),
+                schema_name: "S2".to_string(),
+            }),
+            Entry::Catalog(CatalogEntry {
+                catalog_name: "".to_string(),
+            }),
+            Entry::Table(TableEntry {
+                catalog_name: "C1".to_string(),
+                schema_name: "S1".to_string(),
+                table_name: "T2".to_string(),
+                table_id: 2,
+                engine: MITO_ENGINE.to_string(),
+                is_deleted: false,
+            }),
+        ];
+        let res = LocalCatalogManager::sort_entries(vec);
+        assert_matches!(res[0], Entry::Catalog(..));
+        assert_matches!(res[1], Entry::Catalog(..));
+        assert_matches!(res[2], Entry::Schema(..));
+        assert_matches!(res[3], Entry::Schema(..));
+        assert_matches!(res[4], Entry::Table(..));
+        assert_matches!(res[5], Entry::Table(..));
+    }
+}

src/catalog/src/local/memory.rs

@@ -15,29 +15,129 @@
 use std::any::Any;
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
+use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::{Arc, RwLock, Weak};
 
-use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
+};
 use metrics::{decrement_gauge, increment_gauge};
 use snafu::OptionExt;
+use table::metadata::TableId;
+use table::table::TableIdProvider;
 use table::TableRef;
 
-use crate::error::{CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu};
+use crate::error::{
+    CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, TableNotFoundSnafu,
+};
 use crate::information_schema::InformationSchemaProvider;
-use crate::{CatalogManager, DeregisterTableRequest, RegisterSchemaRequest, RegisterTableRequest};
+use crate::{
+    CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest, RegisterSchemaRequest,
+    RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
+};
 
 type SchemaEntries = HashMap<String, HashMap<String, TableRef>>;
 
 /// Simple in-memory list of catalogs
-#[derive(Clone)]
 pub struct MemoryCatalogManager {
     /// Collection of catalogs containing schemas and ultimately Tables
-    catalogs: Arc<RwLock<HashMap<String, SchemaEntries>>>,
+    pub catalogs: RwLock<HashMap<String, SchemaEntries>>,
+    pub table_id: AtomicU32,
+}
+
+#[async_trait::async_trait]
+impl TableIdProvider for MemoryCatalogManager {
+    async fn next_table_id(&self) -> table::error::Result<TableId> {
+        Ok(self.table_id.fetch_add(1, Ordering::Relaxed))
+    }
 }
 
 #[async_trait::async_trait]
 impl CatalogManager for MemoryCatalogManager {
-    async fn schema_exists(&self, catalog: &str, schema: &str) -> Result<bool> {
+    async fn start(&self) -> Result<()> {
+        self.table_id.store(MIN_USER_TABLE_ID, Ordering::Relaxed);
+        Ok(())
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        self.register_table_sync(request)
+    }
+
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .get_mut(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+
+        // check old and new table names
+        if !schema.contains_key(&request.table_name) {
+            return TableNotFoundSnafu {
+                table_info: request.table_name.to_string(),
+            }
+            .fail()?;
+        }
+        if schema.contains_key(&request.new_table_name) {
+            return TableExistsSnafu {
+                table: &request.new_table_name,
+            }
+            .fail();
+        }
+
+        let table = schema.remove(&request.table_name).unwrap();
+        let _ = schema.insert(request.new_table_name, table);
+
+        Ok(true)
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
+        self.deregister_table_sync(request)
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        self.register_schema_sync(request)
+    }
+
+    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schemas = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?;
+        let table_count = schemas
+            .remove(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?
+            .len();
+        decrement_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            table_count as f64,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
+
+        decrement_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
+        Ok(true)
+    }
+
+    async fn register_system_table(&self, _request: RegisterSystemTableRequest) -> Result<()> {
+        // TODO(ruihang): support register system table request
+        Ok(())
+    }
+
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
         self.schema_exist_sync(catalog, schema)
     }
 
@@ -59,11 +159,11 @@ impl CatalogManager for MemoryCatalogManager {
         Ok(result)
     }
 
-    async fn catalog_exists(&self, catalog: &str) -> Result<bool> {
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
         self.catalog_exist_sync(catalog)
     }
 
-    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
         let catalogs = self.catalogs.read().unwrap();
         Ok(catalogs
             .get(catalog)
@@ -108,27 +208,28 @@ impl CatalogManager for MemoryCatalogManager {
             .collect())
     }
 
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
+        self.register_catalog_sync(name)
+    }
+
     fn as_any(&self) -> &dyn Any {
         self
     }
 }
 
 impl MemoryCatalogManager {
-    pub fn new() -> Arc<Self> {
-        Arc::new(Self {
-            catalogs: Default::default(),
-        })
-    }
-
     /// Creates a manager with some default setups
     /// (e.g. default catalog/schema and information schema)
     pub fn with_default_setup() -> Arc<Self> {
         let manager = Arc::new(Self {
+            table_id: AtomicU32::new(MIN_USER_TABLE_ID),
             catalogs: Default::default(),
         });
 
         // Safety: default catalog/schema is registered in order so no CatalogNotFound error will occur
-        manager.register_catalog_sync(DEFAULT_CATALOG_NAME).unwrap();
+        manager
+            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())
+            .unwrap();
         manager
             .register_schema_sync(RegisterSchemaRequest {
                 catalog: DEFAULT_CATALOG_NAME.to_string(),
@@ -156,15 +257,12 @@ impl MemoryCatalogManager {
     }
 
     /// Registers a catalog if it does not exist and returns false if the schema exists.
-    pub fn register_catalog_sync(&self, name: &str) -> Result<bool> {
-        let name = name.to_string();
-
+    pub fn register_catalog_sync(self: &Arc<Self>, name: String) -> Result<bool> {
         let mut catalogs = self.catalogs.write().unwrap();
 
         match catalogs.entry(name.clone()) {
             Entry::Vacant(e) => {
-                let arc_self = Arc::new(self.clone());
-                let catalog = arc_self.create_catalog_entry(name);
+                let catalog = self.create_catalog_entry(name);
                 e.insert(catalog);
                 increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
                 Ok(true)
@@ -263,7 +361,7 @@ impl MemoryCatalogManager {
         let schema = &table.table_info().schema_name;
 
         if !manager.catalog_exist_sync(catalog).unwrap() {
-            manager.register_catalog_sync(catalog).unwrap();
+            manager.register_catalog_sync(catalog.to_string()).unwrap();
         }
 
         if !manager.schema_exist_sync(catalog, schema).unwrap() {
@@ -295,6 +393,8 @@ pub fn new_memory_catalog_manager() -> Result<Arc<MemoryCatalogManager>> {
 #[cfg(test)]
 mod tests {
     use common_catalog::consts::*;
+    use common_error::ext::ErrorExt;
+    use common_error::status_code::StatusCode;
     use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
 
     use super::*;
@@ -311,7 +411,7 @@ mod tests {
             table: NumbersTable::table(NUMBERS_TABLE_ID),
         };
 
-        catalog_list.register_table_sync(register_request).unwrap();
+        let _ = catalog_list.register_table(register_request).await.unwrap();
         let table = catalog_list
             .table(
                 DEFAULT_CATALOG_NAME,
@@ -328,11 +428,130 @@ mod tests {
             .is_none());
     }
 
+    #[tokio::test]
+    async fn test_mem_manager_rename_table() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+        let table_name = "test_table";
+        assert!(!catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
+        // register test table
+        let table_id = 2333;
+        let register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table: NumbersTable::table(table_id),
+        };
+        assert!(catalog.register_table(register_request).await.unwrap());
+        assert!(catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
+
+        // rename test table
+        let new_table_name = "test_table_renamed";
+        let rename_request = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        let _ = catalog.rename_table(rename_request).await.unwrap();
+
+        // test old table name not exist
+        assert!(!catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
+
+        // test new table name exists
+        assert!(catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap());
+        let registered_table = catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
+
+        let dup_register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: new_table_name.to_string(),
+            table_id: table_id + 1,
+            table: NumbersTable::table(table_id + 1),
+        };
+        let result = catalog.register_table(dup_register_request).await;
+        let err = result.err().unwrap();
+        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
+    }
+
+    #[tokio::test]
+    async fn test_catalog_rename_table() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+        let table_name = "num";
+        let table_id = 2333;
+        let table = NumbersTable::table(table_id);
+
+        // register table
+        let register_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table,
+        };
+        assert!(catalog.register_table(register_table_req).await.unwrap());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_some());
+
+        // rename table
+        let new_table_name = "numbers_new";
+        let rename_table_req = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        assert!(catalog.rename_table(rename_table_req).await.unwrap());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_none());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .is_some());
+
+        let registered_table = catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
+    }
+
     #[test]
     pub fn test_register_catalog_sync() {
         let list = MemoryCatalogManager::with_default_setup();
-        assert!(list.register_catalog_sync("test_catalog").unwrap());
-        assert!(!list.register_catalog_sync("test_catalog").unwrap());
+        assert!(list
+            .register_catalog_sync("test_catalog".to_string())
+            .unwrap());
+        assert!(!list
+            .register_catalog_sync("test_catalog".to_string())
+            .unwrap());
     }
 
     #[tokio::test]
@@ -347,7 +566,7 @@ mod tests {
             table_id: 2333,
             table: NumbersTable::table(2333),
         };
-        catalog.register_table_sync(register_table_req).unwrap();
+        let _ = catalog.register_table(register_table_req).await.unwrap();
         assert!(catalog
             .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
             .await
@@ -359,11 +578,53 @@ mod tests {
             schema: DEFAULT_SCHEMA_NAME.to_string(),
             table_name: table_name.to_string(),
         };
-        catalog.deregister_table_sync(deregister_table_req).unwrap();
+        catalog
+            .deregister_table(deregister_table_req)
+            .await
+            .unwrap();
         assert!(catalog
             .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
             .await
             .unwrap()
             .is_none());
     }
+
+    #[tokio::test]
+    async fn test_catalog_deregister_schema() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+
+        // Registers a catalog, a schema, and a table.
+        let catalog_name = "foo_catalog".to_string();
+        let schema_name = "foo_schema".to_string();
+        let table_name = "foo_table".to_string();
+        let schema = RegisterSchemaRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+        };
+        let table = RegisterTableRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+            table_name,
+            table_id: 0,
+            table: NumbersTable::table(0),
+        };
+        catalog
+            .clone()
+            .register_catalog(catalog_name.clone())
+            .await
+            .unwrap();
+        catalog.register_schema(schema).await.unwrap();
+        catalog.register_table(table).await.unwrap();
+
+        let request = DeregisterSchemaRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+        };
+
+        assert!(catalog.deregister_schema(request).await.unwrap());
+        assert!(!catalog
+            .schema_exist(&catalog_name, &schema_name)
+            .await
+            .unwrap());
+    }
 }

src/catalog/src/remote.rs

@@ -12,11 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 pub use client::{CachedMetaKvBackend, MetaKvBackend};
+pub use manager::RemoteCatalogManager;
 
 mod client;
 mod manager;
 
 #[cfg(feature = "testing")]
 pub mod mock;
-pub use manager::KvBackendCatalogManager;
+pub mod region_alive_keeper;
+
+#[async_trait::async_trait]
+pub trait KvCacheInvalidator: Send + Sync {
+    async fn invalidate_key(&self, key: &[u8]);
+}
+
+pub type KvCacheInvalidatorRef = Arc<dyn KvCacheInvalidator>;

src/catalog/src/remote/client.rs

@@ -18,9 +18,8 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use common_error::ext::BoxedError;
-use common_meta::cache_invalidator::KvCacheInvalidator;
 use common_meta::error::Error::{CacheNotGet, GetKvCache};
-use common_meta::error::{CacheNotGetSnafu, Error, ExternalSnafu, Result};
+use common_meta::error::{CacheNotGetSnafu, Error, MetaSrvSnafu, Result};
 use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
 use common_meta::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
@@ -29,11 +28,12 @@ use common_meta::rpc::store::{
     RangeRequest, RangeResponse,
 };
 use common_meta::rpc::KeyValue;
-use common_telemetry::{debug, timer};
+use common_telemetry::timer;
 use meta_client::client::MetaClient;
 use moka::future::{Cache, CacheBuilder};
 use snafu::{OptionExt, ResultExt};
 
+use super::KvCacheInvalidator;
 use crate::metrics::{METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET};
 
 const CACHE_MAX_CAPACITY: u64 = 10000;
@@ -197,8 +197,7 @@ impl KvBackend for CachedMetaKvBackend {
 #[async_trait::async_trait]
 impl KvCacheInvalidator for CachedMetaKvBackend {
     async fn invalidate_key(&self, key: &[u8]) {
-        self.cache.invalidate(key).await;
-        debug!("invalidated cache key: {}", String::from_utf8_lossy(key));
+        self.cache.invalidate(key).await
     }
 }
 
@@ -252,7 +251,7 @@ impl KvBackend for MetaKvBackend {
             .range(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
@@ -261,7 +260,7 @@ impl KvBackend for MetaKvBackend {
             .range(RangeRequest::new().with_key(key))
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)?;
+            .context(MetaSrvSnafu)?;
         Ok(response.take_kvs().get_mut(0).map(|kv| KeyValue {
             key: kv.take_key(),
             value: kv.take_value(),
@@ -273,7 +272,7 @@ impl KvBackend for MetaKvBackend {
             .batch_put(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn put(&self, req: PutRequest) -> Result<PutResponse> {
@@ -281,7 +280,7 @@ impl KvBackend for MetaKvBackend {
             .put(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
@@ -289,7 +288,7 @@ impl KvBackend for MetaKvBackend {
             .delete_range(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
@@ -297,7 +296,7 @@ impl KvBackend for MetaKvBackend {
             .batch_delete(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
@@ -305,7 +304,7 @@ impl KvBackend for MetaKvBackend {
             .batch_get(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn compare_and_put(
@@ -316,7 +315,7 @@ impl KvBackend for MetaKvBackend {
             .compare_and_put(request)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
@@ -324,7 +323,7 @@ impl KvBackend for MetaKvBackend {
             .move_value(req)
             .await
             .map_err(BoxedError::new)
-            .context(ExternalSnafu)
+            .context(MetaSrvSnafu)
     }
 
     fn as_any(&self) -> &dyn Any {

src/catalog/src/remote/manager.rs

@@ -0,0 +1,436 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_catalog::consts::MITO_ENGINE;
+use common_meta::ident::TableIdent;
+use common_meta::key::catalog_name::CatalogNameKey;
+use common_meta::key::datanode_table::DatanodeTableValue;
+use common_meta::key::schema_name::SchemaNameKey;
+use common_meta::key::TableMetadataManagerRef;
+use common_telemetry::{error, info, warn};
+use futures_util::TryStreamExt;
+use metrics::increment_gauge;
+use snafu::{ensure, OptionExt, ResultExt};
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::EngineContext;
+use table::requests::OpenTableRequest;
+use table::TableRef;
+use tokio::sync::Mutex;
+
+use crate::error::{
+    OpenTableSnafu, ParallelOpenTableSnafu, Result, TableEngineNotFoundSnafu, TableExistsSnafu,
+    TableMetadataManagerSnafu, TableNotFoundSnafu, UnimplementedSnafu,
+};
+use crate::local::MemoryCatalogManager;
+use crate::remote::region_alive_keeper::RegionAliveKeepers;
+use crate::{
+    handle_system_table_request, CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest,
+    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
+};
+
+/// Catalog manager based on metasrv.
+pub struct RemoteCatalogManager {
+    node_id: u64,
+    engine_manager: TableEngineManagerRef,
+    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
+    memory_catalog_manager: Arc<MemoryCatalogManager>,
+    table_metadata_manager: TableMetadataManagerRef,
+}
+
+impl RemoteCatalogManager {
+    pub fn new(
+        engine_manager: TableEngineManagerRef,
+        node_id: u64,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
+        table_metadata_manager: TableMetadataManagerRef,
+    ) -> Self {
+        Self {
+            engine_manager,
+            node_id,
+            system_table_requests: Default::default(),
+            region_alive_keepers,
+            memory_catalog_manager: MemoryCatalogManager::with_default_setup(),
+            table_metadata_manager,
+        }
+    }
+
+    async fn initiate_catalogs(&self) -> Result<()> {
+        let tables = self
+            .table_metadata_manager
+            .datanode_table_manager()
+            .tables(self.node_id)
+            .try_collect::<Vec<_>>()
+            .await
+            .context(TableMetadataManagerSnafu)?;
+
+        let joins = tables
+            .into_iter()
+            .map(|datanode_table_value| {
+                let engine_manager = self.engine_manager.clone();
+                let memory_catalog_manager = self.memory_catalog_manager.clone();
+                let table_metadata_manager = self.table_metadata_manager.clone();
+                let region_alive_keepers = self.region_alive_keepers.clone();
+                common_runtime::spawn_bg(async move {
+                    let table_id = datanode_table_value.table_id;
+                    if let Err(e) = open_and_register_table(
+                        engine_manager,
+                        datanode_table_value,
+                        memory_catalog_manager,
+                        table_metadata_manager,
+                        region_alive_keepers,
+                    )
+                    .await
+                    {
+                        // Note that we don't return error here if table opened failed. This is because
+                        // we don't want those broken tables to impede the startup of Datanode.
+                        // However, this could be changed in the future.
+                        error!(e; "Failed to open or register table, id = {table_id}")
+                    }
+                })
+            })
+            .collect::<Vec<_>>();
+
+        let _ = futures::future::try_join_all(joins)
+            .await
+            .context(ParallelOpenTableSnafu)?;
+        Ok(())
+    }
+}
+
+async fn open_and_register_table(
+    engine_manager: TableEngineManagerRef,
+    datanode_table_value: DatanodeTableValue,
+    memory_catalog_manager: Arc<MemoryCatalogManager>,
+    table_metadata_manager: TableMetadataManagerRef,
+    region_alive_keepers: Arc<RegionAliveKeepers>,
+) -> Result<()> {
+    let context = EngineContext {};
+
+    let table_id = datanode_table_value.table_id;
+    let region_numbers = datanode_table_value.regions;
+
+    let table_info_value = table_metadata_manager
+        .table_info_manager()
+        .get(table_id)
+        .await
+        .context(TableMetadataManagerSnafu)?
+        .context(TableNotFoundSnafu {
+            table_info: format!("table id: {table_id}"),
+        })?;
+    let table_info = &table_info_value.table_info;
+    let catalog_name = table_info.catalog_name.clone();
+    let schema_name = table_info.schema_name.clone();
+    let table_name = table_info.name.clone();
+
+    let request = OpenTableRequest {
+        catalog_name: catalog_name.clone(),
+        schema_name: schema_name.clone(),
+        table_name: table_name.clone(),
+        table_id,
+        region_numbers: region_numbers.clone(),
+    };
+    let engine =
+        engine_manager
+            .engine(&table_info.meta.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &table_info.meta.engine,
+            })?;
+
+    let table_ident = TableIdent {
+        catalog: catalog_name,
+        schema: schema_name,
+        table: table_name,
+        table_id,
+        engine: table_info.meta.engine.clone(),
+    };
+
+    let table = engine
+        .open_table(&context, request)
+        .await
+        .with_context(|_| OpenTableSnafu {
+            table_info: table_ident.to_string(),
+        })?
+        .with_context(|| TableNotFoundSnafu {
+            table_info: table_ident.to_string(),
+        })?;
+    info!("Successfully opened table, {table_ident}");
+
+    if !memory_catalog_manager
+        .catalog_exist(&table_ident.catalog)
+        .await?
+    {
+        memory_catalog_manager.register_catalog_sync(table_ident.catalog.clone())?;
+    }
+
+    if !memory_catalog_manager
+        .schema_exist(&table_ident.catalog, &table_ident.schema)
+        .await?
+    {
+        memory_catalog_manager.register_schema_sync(RegisterSchemaRequest {
+            catalog: table_ident.catalog.clone(),
+            schema: table_ident.schema.clone(),
+        })?;
+    }
+
+    let request = RegisterTableRequest {
+        catalog: table_ident.catalog.clone(),
+        schema: table_ident.schema.clone(),
+        table_name: table_ident.table.clone(),
+        table_id,
+        table,
+    };
+    let registered =
+        register_table(&memory_catalog_manager, &region_alive_keepers, request).await?;
+    ensure!(
+        registered,
+        TableExistsSnafu {
+            table: table_ident.to_string(),
+        }
+    );
+    info!("Successfully registered table, {table_ident}");
+    Ok(())
+}
+
+async fn register_table(
+    memory_catalog_manager: &Arc<MemoryCatalogManager>,
+    region_alive_keepers: &Arc<RegionAliveKeepers>,
+    request: RegisterTableRequest,
+) -> Result<bool> {
+    let table = request.table.clone();
+
+    let registered = memory_catalog_manager.register_table_sync(request)?;
+
+    if registered {
+        let table_info = table.table_info();
+        let table_ident = TableIdent {
+            catalog: table_info.catalog_name.clone(),
+            schema: table_info.schema_name.clone(),
+            table: table_info.name.clone(),
+            table_id: table_info.table_id(),
+            engine: table_info.meta.engine.clone(),
+        };
+        region_alive_keepers
+            .register_table(table_ident, table, memory_catalog_manager.clone())
+            .await?;
+    }
+
+    Ok(registered)
+}
+
+#[async_trait]
+impl CatalogManager for RemoteCatalogManager {
+    async fn start(&self) -> Result<()> {
+        self.initiate_catalogs().await?;
+
+        let mut system_table_requests = self.system_table_requests.lock().await;
+        let engine = self
+            .engine_manager
+            .engine(MITO_ENGINE)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: MITO_ENGINE,
+            })?;
+        handle_system_table_request(self, engine, &mut system_table_requests).await?;
+        info!("All system table opened");
+        Ok(())
+    }
+
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
+        register_table(
+            &self.memory_catalog_manager,
+            &self.region_alive_keepers,
+            request,
+        )
+        .await
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
+        let Some(table) = self
+            .memory_catalog_manager
+            .table(&request.catalog, &request.schema, &request.table_name)
+            .await?
+        else {
+            return Ok(());
+        };
+
+        let table_info = table.table_info();
+        let table_ident = TableIdent {
+            catalog: request.catalog.clone(),
+            schema: request.schema.clone(),
+            table: request.table_name.clone(),
+            table_id: table_info.ident.table_id,
+            engine: table_info.meta.engine.clone(),
+        };
+        if let Some(keeper) = self
+            .region_alive_keepers
+            .deregister_table(&table_ident)
+            .await
+        {
+            warn!(
+                "Table {} is deregistered from region alive keepers",
+                keeper.table_ident(),
+            );
+        }
+
+        self.memory_catalog_manager.deregister_table(request).await
+    }
+
+    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        self.memory_catalog_manager.register_schema_sync(request)
+    }
+
+    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
+        UnimplementedSnafu {
+            operation: "deregister schema",
+        }
+        .fail()
+    }
+
+    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
+        self.memory_catalog_manager.rename_table(request).await
+    }
+
+    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
+        let catalog_name = request.create_table_request.catalog_name.clone();
+        let schema_name = request.create_table_request.schema_name.clone();
+
+        let mut requests = self.system_table_requests.lock().await;
+        requests.push(request);
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&catalog_name, &schema_name)],
+        );
+        Ok(())
+    }
+
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
+        if !self.catalog_exist(catalog).await? {
+            return Ok(false);
+        }
+
+        if self
+            .memory_catalog_manager
+            .schema_exist(catalog, schema)
+            .await?
+        {
+            return Ok(true);
+        }
+
+        let remote_schema_exists = self
+            .table_metadata_manager
+            .schema_manager()
+            .exist(SchemaNameKey::new(catalog, schema))
+            .await
+            .context(TableMetadataManagerSnafu)?;
+        // Create schema locally if remote schema exists. Since local schema is managed by memory
+        // catalog manager, creating a local schema is relatively cheap (just a HashMap).
+        // Besides, if this method ("schema_exist) is called, it's very likely that someone wants to
+        // create a table in this schema. We should create the schema now.
+        if remote_schema_exists
+            && self
+                .memory_catalog_manager
+                .register_schema(RegisterSchemaRequest {
+                    catalog: catalog.to_string(),
+                    schema: schema.to_string(),
+                })
+                .await?
+        {
+            info!("register schema '{catalog}/{schema}' on demand");
+        }
+
+        Ok(remote_schema_exists)
+    }
+
+    async fn table(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        self.memory_catalog_manager
+            .table(catalog_name, schema_name, table_name)
+            .await
+    }
+
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
+        if self.memory_catalog_manager.catalog_exist(catalog).await? {
+            return Ok(true);
+        }
+
+        let key = CatalogNameKey::new(catalog);
+        let remote_catalog_exists = self
+            .table_metadata_manager
+            .catalog_manager()
+            .exist(key)
+            .await
+            .context(TableMetadataManagerSnafu)?;
+
+        // Create catalog locally if remote catalog exists. Since local catalog is managed by memory
+        // catalog manager, creating a local catalog is relatively cheap (just a HashMap).
+        // Besides, if this method ("catalog_exist) is called, it's very likely that someone wants to
+        // create a table in this catalog. We should create the catalog now.
+        if remote_catalog_exists
+            && self
+                .memory_catalog_manager
+                .clone()
+                .register_catalog(catalog.to_string())
+                .await?
+        {
+            info!("register catalog '{catalog}' on demand");
+        }
+
+        Ok(remote_catalog_exists)
+    }
+
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+        if !self.catalog_exist(catalog).await? {
+            return Ok(false);
+        }
+
+        if !self.schema_exist(catalog, schema).await? {
+            return Ok(false);
+        }
+
+        self.memory_catalog_manager
+            .table_exist(catalog, schema, table)
+            .await
+    }
+
+    async fn catalog_names(&self) -> Result<Vec<String>> {
+        self.memory_catalog_manager.catalog_names().await
+    }
+
+    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
+        self.memory_catalog_manager.schema_names(catalog_name).await
+    }
+
+    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
+        self.memory_catalog_manager
+            .table_names(catalog_name, schema_name)
+            .await
+    }
+
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
+        self.memory_catalog_manager.register_catalog_sync(name)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/catalog/src/remote/mock.rs

@@ -55,14 +55,14 @@ impl TableEngine for MockTableEngine {
 
         let data = vec![Arc::new(StringVector::from(vec!["a", "b", "c"])) as _];
         let record_batch = RecordBatch::new(schema, data).unwrap();
-        let table = MemTable::new_with_catalog(
+        let table: TableRef = Arc::new(MemTable::new_with_catalog(
             &request.table_name,
             record_batch,
             table_id,
             request.catalog_name,
             request.schema_name,
             vec![0],
-        );
+        )) as Arc<_>;
 
         let mut tables = self.tables.write().unwrap();
         let _ = tables.insert(table_id, table.clone() as TableRef);

src/catalog/src/remote/region_alive_keeper.rs

@@ -0,0 +1,867 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_meta::error::InvalidProtoMsgSnafu;
+use common_meta::heartbeat::handler::{
+    HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
+};
+use common_meta::ident::TableIdent;
+use common_meta::RegionIdent;
+use common_telemetry::{debug, error, info, warn};
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::RegionNumber;
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::{CloseTableResult, EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::CloseTableRequest;
+use table::TableRef;
+use tokio::sync::{mpsc, oneshot, Mutex};
+use tokio::task::JoinHandle;
+use tokio::time::{Duration, Instant};
+
+use crate::error::{Result, TableEngineNotFoundSnafu};
+use crate::local::MemoryCatalogManager;
+use crate::DeregisterTableRequest;
+
+/// [RegionAliveKeepers] manages all [RegionAliveKeeper] in a scope of tables.
+pub struct RegionAliveKeepers {
+    table_engine_manager: TableEngineManagerRef,
+    keepers: Arc<Mutex<HashMap<TableId, Arc<RegionAliveKeeper>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+
+    /// The epoch when [RegionAliveKeepers] is created. It's used to get a monotonically non-decreasing
+    /// elapsed time when submitting heartbeats to Metasrv (because [Instant] is monotonically
+    /// non-decreasing). The heartbeat request will carry the duration since this epoch, and the
+    /// duration acts like an "invariant point" for region's keep alive lease.
+    epoch: Instant,
+}
+
+impl RegionAliveKeepers {
+    pub fn new(
+        table_engine_manager: TableEngineManagerRef,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            table_engine_manager,
+            keepers: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+            epoch: Instant::now(),
+        }
+    }
+
+    pub async fn find_keeper(&self, table_id: TableId) -> Option<Arc<RegionAliveKeeper>> {
+        self.keepers.lock().await.get(&table_id).cloned()
+    }
+
+    pub async fn register_table(
+        &self,
+        table_ident: TableIdent,
+        table: TableRef,
+        catalog_manager: Arc<MemoryCatalogManager>,
+    ) -> Result<()> {
+        let table_id = table_ident.table_id;
+        let keeper = self.find_keeper(table_id).await;
+        if keeper.is_some() {
+            return Ok(());
+        }
+
+        let table_engine = self
+            .table_engine_manager
+            .engine(&table_ident.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &table_ident.engine,
+            })?;
+
+        let keeper = Arc::new(RegionAliveKeeper::new(
+            table_engine,
+            catalog_manager,
+            table_ident.clone(),
+            self.heartbeat_interval_millis,
+        ));
+        for r in table.table_info().meta.region_numbers.iter() {
+            keeper.register_region(*r).await;
+        }
+
+        let mut keepers = self.keepers.lock().await;
+        let _ = keepers.insert(table_id, keeper.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            keeper.start().await;
+
+            info!("RegionAliveKeeper for table {table_ident} is started!");
+        } else {
+            info!("RegionAliveKeeper for table {table_ident} is registered but not started yet!");
+        }
+        Ok(())
+    }
+
+    pub async fn deregister_table(
+        &self,
+        table_ident: &TableIdent,
+    ) -> Option<Arc<RegionAliveKeeper>> {
+        let table_id = table_ident.table_id;
+        self.keepers.lock().await.remove(&table_id).map(|x| {
+            info!("Deregister RegionAliveKeeper for table {table_ident}");
+            x
+        })
+    }
+
+    pub async fn register_region(&self, region_ident: &RegionIdent) {
+        let table_id = region_ident.table_ident.table_id;
+        let Some(keeper) = self.find_keeper(table_id).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        keeper.register_region(region_ident.region_number).await
+    }
+
+    pub async fn deregister_region(&self, region_ident: &RegionIdent) {
+        let table_id = region_ident.table_ident.table_id;
+        let Some(keeper) = self.find_keeper(table_id).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        let _ = keeper.deregister_region(region_ident.region_number).await;
+    }
+
+    pub async fn start(&self) {
+        let keepers = self.keepers.lock().await;
+        for keeper in keepers.values() {
+            keeper.start().await;
+        }
+        self.started.store(true, Ordering::Relaxed);
+
+        info!(
+            "RegionAliveKeepers for tables {:?} are started!",
+            keepers.keys().map(|x| x.to_string()).collect::<Vec<_>>(),
+        );
+    }
+
+    pub fn epoch(&self) -> Instant {
+        self.epoch
+    }
+}
+
+#[async_trait]
+impl HeartbeatResponseHandler for RegionAliveKeepers {
+    fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
+        !ctx.response.region_leases.is_empty()
+    }
+
+    async fn handle(
+        &self,
+        ctx: &mut HeartbeatResponseHandlerContext,
+    ) -> common_meta::error::Result<HandleControl> {
+        let leases = ctx.response.region_leases.drain(..).collect::<Vec<_>>();
+        for lease in leases {
+            let table_ident: TableIdent = match lease
+                .table_ident
+                .context(InvalidProtoMsgSnafu {
+                    err_msg: "'table_ident' is missing in RegionLease",
+                })
+                .and_then(|x| x.try_into())
+            {
+                Ok(x) => x,
+                Err(e) => {
+                    error!(e; "");
+                    continue;
+                }
+            };
+
+            let table_id = table_ident.table_id;
+            let Some(keeper) = self.keepers.lock().await.get(&table_id).cloned() else {
+                // Alive keeper could be affected by lagging msg, just warn and ignore.
+                warn!("Alive keeper for table {table_ident} is not found!");
+                continue;
+            };
+
+            let start_instant = self.epoch + Duration::from_millis(lease.duration_since_epoch);
+            let deadline = start_instant + Duration::from_secs(lease.lease_seconds);
+            keeper.keep_lived(lease.regions, deadline).await;
+        }
+        Ok(HandleControl::Continue)
+    }
+}
+
+/// [RegionAliveKeeper] starts a countdown for each region in a table. When deadline is reached,
+/// the region will be closed.
+/// The deadline is controlled by Metasrv. It works like "lease" for regions: a Datanode submits its
+/// opened regions to Metasrv, in heartbeats. If Metasrv decides some region could be resided in this
+/// Datanode, it will "extend" the region's "lease", with a deadline for [RegionAliveKeeper] to
+/// countdown.
+pub struct RegionAliveKeeper {
+    catalog_manager: Arc<MemoryCatalogManager>,
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    countdown_task_handles: Arc<Mutex<HashMap<RegionNumber, Arc<CountdownTaskHandle>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+}
+
+impl RegionAliveKeeper {
+    fn new(
+        table_engine: TableEngineRef,
+        catalog_manager: Arc<MemoryCatalogManager>,
+        table_ident: TableIdent,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            catalog_manager,
+            table_engine,
+            table_ident,
+            countdown_task_handles: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+        }
+    }
+
+    async fn find_handle(&self, region: &RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .get(region)
+            .cloned()
+    }
+
+    async fn register_region(&self, region: RegionNumber) {
+        if self.find_handle(&region).await.is_some() {
+            return;
+        }
+
+        let countdown_task_handles = Arc::downgrade(&self.countdown_task_handles);
+        let on_task_finished = async move {
+            if let Some(x) = countdown_task_handles.upgrade() {
+                let _ = x.lock().await.remove(&region);
+            } // Else the countdown task handles map could be dropped because the keeper is dropped.
+        };
+        let catalog_manager = self.catalog_manager.clone();
+        let ident = self.table_ident.clone();
+        let handle = Arc::new(CountdownTaskHandle::new(
+            self.table_engine.clone(),
+            self.table_ident.clone(),
+            region,
+            move |result: Option<CloseTableResult>| {
+                if matches!(result, Some(CloseTableResult::Released(_))) {
+                    let result = catalog_manager.deregister_table_sync(DeregisterTableRequest {
+                        catalog: ident.catalog.to_string(),
+                        schema: ident.schema.to_string(),
+                        table_name: ident.table.to_string(),
+                    });
+
+                    info!(
+                        "Deregister table: {} after countdown task finished, result: {result:?}",
+                        ident.table_id
+                    );
+                } else {
+                    debug!("Countdown task returns: {result:?}");
+                }
+                on_task_finished
+            },
+        ));
+
+        let mut handles = self.countdown_task_handles.lock().await;
+        let _ = handles.insert(region, handle.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            handle.start(self.heartbeat_interval_millis).await;
+
+            info!(
+                "Region alive countdown for region {region} in table {} is started!",
+                self.table_ident
+            );
+        } else {
+            info!(
+                "Region alive countdown for region {region} in table {} is registered but not started yet!",
+                self.table_ident
+            );
+        }
+    }
+
+    async fn deregister_region(&self, region: RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .remove(&region)
+            .map(|x| {
+                info!(
+                    "Deregister alive countdown for region {region} in table {}",
+                    self.table_ident
+                );
+                x
+            })
+    }
+
+    async fn start(&self) {
+        let handles = self.countdown_task_handles.lock().await;
+        for handle in handles.values() {
+            handle.start(self.heartbeat_interval_millis).await;
+        }
+
+        self.started.store(true, Ordering::Relaxed);
+        info!(
+            "Region alive countdowns for regions {:?} in table {} are started!",
+            handles.keys().copied().collect::<Vec<_>>(),
+            self.table_ident
+        );
+    }
+
+    async fn keep_lived(&self, designated_regions: Vec<RegionNumber>, deadline: Instant) {
+        for region in designated_regions {
+            if let Some(handle) = self.find_handle(&region).await {
+                handle.reset_deadline(deadline).await;
+            }
+            // Else the region alive keeper might be triggered by lagging messages, we can safely ignore it.
+        }
+    }
+
+    pub async fn deadline(&self, region: RegionNumber) -> Option<Instant> {
+        let mut deadline = None;
+        if let Some(handle) = self.find_handle(&region).await {
+            let (s, r) = oneshot::channel();
+            if handle.tx.send(CountdownCommand::Deadline(s)).await.is_ok() {
+                deadline = r.await.ok()
+            }
+        }
+        deadline
+    }
+
+    pub fn table_ident(&self) -> &TableIdent {
+        &self.table_ident
+    }
+}
+
+#[derive(Debug)]
+enum CountdownCommand {
+    Start(u64),
+    Reset(Instant),
+    Deadline(oneshot::Sender<Instant>),
+}
+
+struct CountdownTaskHandle {
+    tx: mpsc::Sender<CountdownCommand>,
+    handler: JoinHandle<()>,
+    table_ident: TableIdent,
+    region: RegionNumber,
+}
+
+impl CountdownTaskHandle {
+    /// Creates a new [CountdownTaskHandle] and starts the countdown task.
+    /// # Params
+    /// - `on_task_finished`: a callback to be invoked when the task is finished. Note that it will not
+    ///   be invoked if the task is cancelled (by dropping the handle). This is because we want something
+    ///   meaningful to be done when the task is finished, e.g. deregister the handle from the map.
+    ///   While dropping the handle does not necessarily mean the task is finished.
+    fn new<Fut>(
+        table_engine: TableEngineRef,
+        table_ident: TableIdent,
+        region: RegionNumber,
+        on_task_finished: impl FnOnce(Option<CloseTableResult>) -> Fut + Send + 'static,
+    ) -> Self
+    where
+        Fut: Future<Output = ()> + Send,
+    {
+        let (tx, rx) = mpsc::channel(1024);
+
+        let mut countdown_task = CountdownTask {
+            table_engine,
+            table_ident: table_ident.clone(),
+            region,
+            rx,
+        };
+        let handler = common_runtime::spawn_bg(async move {
+            let result = countdown_task.run().await;
+            on_task_finished(result).await;
+        });
+
+        Self {
+            tx,
+            handler,
+            table_ident,
+            region,
+        }
+    }
+
+    async fn start(&self, heartbeat_interval_millis: u64) {
+        if let Err(e) = self
+            .tx
+            .send(CountdownCommand::Start(heartbeat_interval_millis))
+            .await
+        {
+            warn!(
+                "Failed to start region alive keeper countdown: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+
+    async fn reset_deadline(&self, deadline: Instant) {
+        if let Err(e) = self.tx.send(CountdownCommand::Reset(deadline)).await {
+            warn!(
+                "Failed to reset region alive keeper deadline: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+}
+
+impl Drop for CountdownTaskHandle {
+    fn drop(&mut self) {
+        debug!(
+            "Aborting region alive countdown task for region {} in table {}",
+            self.region, self.table_ident,
+        );
+        self.handler.abort();
+    }
+}
+
+struct CountdownTask {
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    region: RegionNumber,
+    rx: mpsc::Receiver<CountdownCommand>,
+}
+
+impl CountdownTask {
+    // returns true if
+    async fn run(&mut self) -> Option<CloseTableResult> {
+        // 30 years. See `Instant::far_future`.
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
+
+        // Make sure the alive countdown is not gonna happen before heartbeat task is started (the
+        // "start countdown" command will be sent from heartbeat task).
+        let countdown = tokio::time::sleep_until(far_future);
+        tokio::pin!(countdown);
+
+        let region = &self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            tokio::select! {
+                command = self.rx.recv() => {
+                    match command {
+                        Some(CountdownCommand::Start(heartbeat_interval_millis)) => {
+                            // Set first deadline in 4 heartbeats (roughly after 20 seconds from now if heartbeat
+                            // interval is set to default 5 seconds), to make Datanode and Metasrv more tolerable to
+                            // network or other jitters during startup.
+                            let first_deadline = Instant::now() + Duration::from_millis(heartbeat_interval_millis) * 4;
+                            countdown.set(tokio::time::sleep_until(first_deadline));
+                        },
+                        Some(CountdownCommand::Reset(deadline)) => {
+                            if countdown.deadline() < deadline {
+                                debug!(
+                                    "Reset deadline of region {region} of table {table_ident} to approximately {} seconds later",
+                                    (deadline - Instant::now()).as_secs_f32(),
+                                );
+                                countdown.set(tokio::time::sleep_until(deadline));
+                            }
+                            // Else the countdown could be either:
+                            // - not started yet;
+                            // - during startup protection;
+                            // - received a lagging heartbeat message.
+                            // All can be safely ignored.
+                        },
+                        None => {
+                            info!(
+                                "The handle of countdown task for region {region} of table {table_ident} \
+                                is dropped, RegionAliveKeeper out."
+                            );
+                            break;
+                        },
+                        Some(CountdownCommand::Deadline(tx)) => {
+                            let _ = tx.send(countdown.deadline());
+                        }
+                    }
+                }
+                () = &mut countdown => {
+                    let result = self.close_region().await;
+                    warn!(
+                        "Region {region} of table {table_ident} is closed, result: {result:?}. \
+                        RegionAliveKeeper out.",
+                    );
+                    return Some(result);
+                }
+            }
+        }
+        None
+    }
+
+    async fn close_region(&self) -> CloseTableResult {
+        let ctx = EngineContext::default();
+        let region = self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            let request = CloseTableRequest {
+                catalog_name: table_ident.catalog.clone(),
+                schema_name: table_ident.schema.clone(),
+                table_name: table_ident.table.clone(),
+                table_id: table_ident.table_id,
+                region_numbers: vec![region],
+                flush: true,
+            };
+            match self.table_engine.close_table(&ctx, request).await {
+                Ok(result) => return result,
+                // If region is failed to close, immediately retry. Maybe we should panic instead?
+                Err(e) => error!(e;
+                    "Failed to close region {region} of table {table_ident}. \
+                    For the integrity of data, retry closing and retry without wait.",
+                ),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+
+    use api::v1::meta::{HeartbeatResponse, RegionLease};
+    use common_meta::heartbeat::mailbox::HeartbeatMailbox;
+    use datatypes::schema::RawSchema;
+    use table::engine::manager::MemoryTableEngineManager;
+    use table::engine::TableEngine;
+    use table::requests::{CreateTableRequest, TableOptions};
+    use table::test_util::EmptyTable;
+
+    use super::*;
+    use crate::remote::mock::MockTableEngine;
+
+    async fn prepare_keepers() -> (TableIdent, RegionAliveKeepers) {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_engine_manager = Arc::new(MemoryTableEngineManager::new(table_engine));
+        let keepers = RegionAliveKeepers::new(table_engine_manager, 5000);
+
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id: 1,
+            engine: "MockTableEngine".to_string(),
+        };
+        let table = Arc::new(EmptyTable::new(CreateTableRequest {
+            id: 1,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![1, 2, 3],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "MockTableEngine".to_string(),
+        }));
+        let catalog_manager = MemoryCatalogManager::new_with_table(table.clone());
+        keepers
+            .register_table(table_ident.clone(), table, catalog_manager)
+            .await
+            .unwrap();
+        assert!(keepers
+            .keepers
+            .lock()
+            .await
+            .contains_key(&table_ident.table_id));
+
+        (table_ident, keepers)
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_handle_heartbeat_response() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers.start().await;
+        let startup_protection_until = Instant::now() + Duration::from_secs(21);
+
+        let duration_since_epoch = (Instant::now() - keepers.epoch).as_millis() as _;
+        let lease_seconds = 100;
+        let response = HeartbeatResponse {
+            region_leases: vec![RegionLease {
+                table_ident: Some(table_ident.clone().into()),
+                regions: vec![1, 3], // Not extending region 2's lease time.
+                duration_since_epoch,
+                lease_seconds,
+            }],
+            ..Default::default()
+        };
+        let keep_alive_until = keepers.epoch
+            + Duration::from_millis(duration_since_epoch)
+            + Duration::from_secs(lease_seconds);
+
+        let (tx, _) = mpsc::channel(8);
+        let mailbox = Arc::new(HeartbeatMailbox::new(tx));
+        let mut ctx = HeartbeatResponseHandlerContext::new(mailbox, response);
+
+        assert!(keepers.handle(&mut ctx).await.unwrap() == HandleControl::Continue);
+
+        // sleep to wait for background task spawned in `handle`
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        async fn test(
+            keeper: &Arc<RegionAliveKeeper>,
+            region_number: RegionNumber,
+            startup_protection_until: Instant,
+            keep_alive_until: Instant,
+            is_kept_live: bool,
+        ) {
+            let deadline = keeper.deadline(region_number).await.unwrap();
+            if is_kept_live {
+                assert!(deadline > startup_protection_until && deadline == keep_alive_until);
+            } else {
+                assert!(deadline <= startup_protection_until);
+            }
+        }
+
+        let keeper = &keepers
+            .keepers
+            .lock()
+            .await
+            .get(&table_ident.table_id)
+            .cloned()
+            .unwrap();
+
+        // Test region 1 and 3 is kept lived. Their deadlines are updated to desired instant.
+        test(keeper, 1, startup_protection_until, keep_alive_until, true).await;
+        test(keeper, 3, startup_protection_until, keep_alive_until, true).await;
+
+        // Test region 2 is not kept lived. It's deadline is not updated: still during startup protection period.
+        test(keeper, 2, startup_protection_until, keep_alive_until, false).await;
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keepers() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers
+            .register_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 4,
+            })
+            .await;
+
+        keepers.start().await;
+        for keeper in keepers.keepers.lock().await.values() {
+            let regions = {
+                let handles = keeper.countdown_task_handles.lock().await;
+                handles.keys().copied().collect::<Vec<_>>()
+            };
+            for region in regions {
+                // assert countdown tasks are started
+                let deadline = keeper.deadline(region).await.unwrap();
+                assert!(deadline <= Instant::now() + Duration::from_secs(20));
+            }
+        }
+
+        keepers
+            .deregister_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 1,
+            })
+            .await;
+        let mut regions = keepers
+            .find_keeper(table_ident.table_id)
+            .await
+            .unwrap()
+            .countdown_task_handles
+            .lock()
+            .await
+            .keys()
+            .copied()
+            .collect::<Vec<_>>();
+        regions.sort();
+        assert_eq!(regions, vec![2, 3, 4]);
+
+        let keeper = keepers.deregister_table(&table_ident).await.unwrap();
+        assert!(Arc::try_unwrap(keeper).is_ok(), "keeper is not dropped");
+        assert!(keepers.keepers.lock().await.is_empty());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keeper() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let catalog_manager = MemoryCatalogManager::with_default_setup();
+        let keeper = RegionAliveKeeper::new(table_engine, catalog_manager, table_ident, 1000);
+
+        let region = 1;
+        assert!(keeper.find_handle(&region).await.is_none());
+        keeper.register_region(region).await;
+        let _ = keeper.find_handle(&region).await.unwrap();
+
+        let ten_seconds_later = || Instant::now() + Duration::from_secs(10);
+
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        assert!(keeper.find_handle(&2).await.is_none());
+        assert!(keeper.find_handle(&3).await.is_none());
+
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert if keeper is not started, keep_lived is of no use
+        assert!(keeper.deadline(region).await.unwrap() > far_future);
+
+        keeper.start().await;
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        // assert keep_lived works if keeper is started
+        assert!(keeper.deadline(region).await.unwrap() <= ten_seconds_later());
+
+        let handle = keeper.deregister_region(region).await.unwrap();
+        assert!(Arc::try_unwrap(handle).is_ok(), "handle is not dropped");
+        assert!(keeper.find_handle(&region).await.is_none());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_handle() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(
+            table_engine.clone(),
+            table_ident.clone(),
+            1,
+            |_| async move { finished_clone.store(true, Ordering::Relaxed) },
+        );
+        let tx = handle.tx.clone();
+
+        // assert countdown task is running
+        tx.send(CountdownCommand::Start(5000)).await.unwrap();
+        assert!(!finished.load(Ordering::Relaxed));
+
+        drop(handle);
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        // assert countdown task is stopped
+        assert!(tx
+            .try_send(CountdownCommand::Reset(
+                Instant::now() + Duration::from_secs(10)
+            ))
+            .is_err());
+        // assert `on_task_finished` is not called (because the task is aborted by the handle's drop)
+        assert!(!finished.load(Ordering::Relaxed));
+
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(table_engine, table_ident, 1, |_| async move {
+            finished_clone.store(true, Ordering::Relaxed)
+        });
+        handle.tx.send(CountdownCommand::Start(100)).await.unwrap();
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        // assert `on_task_finished` is called when task is finished normally
+        assert!(finished.load(Ordering::Relaxed));
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_run() {
+        let ctx = &EngineContext::default();
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let table_id = 1;
+        let request = CreateTableRequest {
+            id: table_id,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "mito".to_string(),
+        };
+
+        let table_engine = Arc::new(MockTableEngine::default());
+        let _ = table_engine.create_table(ctx, request).await.unwrap();
+
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id,
+            engine: "mito".to_string(),
+        };
+        let (tx, rx) = mpsc::channel(10);
+        let mut task = CountdownTask {
+            table_engine: table_engine.clone(),
+            table_ident,
+            region: 1,
+            rx,
+        };
+        let _handle = common_runtime::spawn_bg(async move {
+            task.run().await;
+        });
+
+        async fn deadline(tx: &mpsc::Sender<CountdownCommand>) -> Instant {
+            let (s, r) = oneshot::channel();
+            tx.send(CountdownCommand::Deadline(s)).await.unwrap();
+            r.await.unwrap()
+        }
+
+        // if countdown task is not started, its deadline is set to far future
+        assert!(deadline(&tx).await > Instant::now() + Duration::from_secs(86400 * 365 * 29));
+
+        // start countdown in 250ms * 4 = 1s
+        tx.send(CountdownCommand::Start(250)).await.unwrap();
+        // assert deadline is correctly set
+        assert!(deadline(&tx).await <= Instant::now() + Duration::from_secs(1));
+
+        // reset countdown in 1.5s
+        tx.send(CountdownCommand::Reset(
+            Instant::now() + Duration::from_millis(1500),
+        ))
+        .await
+        .unwrap();
+
+        // assert the table is closed after deadline is reached
+        assert!(table_engine.table_exists(ctx, table_id));
+        // spare 500ms for the task to close the table
+        tokio::time::sleep(Duration::from_millis(2000)).await;
+        assert!(!table_engine.table_exists(ctx, table_id));
+    }
+}

src/catalog/src/system.rs

@@ -0,0 +1,634 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
+    SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
+};
+use common_recordbatch::SendableRecordBatchStream;
+use common_telemetry::{debug, warn};
+use common_time::util;
+use datatypes::prelude::{ConcreteDataType, ScalarVector, VectorRef};
+use datatypes::schema::{ColumnSchema, RawSchema};
+use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
+use serde::{Deserialize, Serialize};
+use snafu::{ensure, OptionExt, ResultExt};
+use store_api::storage::ScanRequest;
+use table::engine::{EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::{CreateTableRequest, InsertRequest, OpenTableRequest, TableOptions};
+use table::TableRef;
+
+use crate::error::{
+    self, CreateSystemCatalogSnafu, DeregisterTableSnafu, EmptyValueSnafu, Error,
+    InsertCatalogRecordSnafu, InvalidEntryTypeSnafu, InvalidKeySnafu, OpenSystemCatalogSnafu,
+    Result, ValueDeserializeSnafu,
+};
+use crate::DeregisterTableRequest;
+
+pub const ENTRY_TYPE_INDEX: usize = 0;
+pub const KEY_INDEX: usize = 1;
+pub const VALUE_INDEX: usize = 3;
+
+pub struct SystemCatalogTable(TableRef);
+
+impl SystemCatalogTable {
+    pub async fn new(engine: TableEngineRef) -> Result<Self> {
+        let request = OpenTableRequest {
+            catalog_name: SYSTEM_CATALOG_NAME.to_string(),
+            schema_name: INFORMATION_SCHEMA_NAME.to_string(),
+            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+            table_id: SYSTEM_CATALOG_TABLE_ID,
+            region_numbers: vec![0],
+        };
+        let schema = build_system_catalog_schema();
+        let ctx = EngineContext::default();
+
+        if let Some(table) = engine
+            .open_table(&ctx, request)
+            .await
+            .context(OpenSystemCatalogSnafu)?
+        {
+            Ok(Self(table))
+        } else {
+            // system catalog table is not yet created, try to create
+            let request = CreateTableRequest {
+                id: SYSTEM_CATALOG_TABLE_ID,
+                catalog_name: SYSTEM_CATALOG_NAME.to_string(),
+                schema_name: INFORMATION_SCHEMA_NAME.to_string(),
+                table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+                desc: Some("System catalog table".to_string()),
+                schema,
+                region_numbers: vec![0],
+                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX],
+                create_if_not_exists: true,
+                table_options: TableOptions::default(),
+                engine: engine.name().to_string(),
+            };
+
+            let table = engine
+                .create_table(&ctx, request)
+                .await
+                .context(CreateSystemCatalogSnafu)?;
+            Ok(Self(table))
+        }
+    }
+
+    pub async fn register_table(
+        &self,
+        catalog: String,
+        schema: String,
+        table_name: String,
+        table_id: TableId,
+        engine: String,
+    ) -> Result<usize> {
+        let insert_request =
+            build_table_insert_request(catalog, schema, table_name, table_id, engine);
+        self.0
+            .insert(insert_request)
+            .await
+            .context(InsertCatalogRecordSnafu)
+    }
+
+    pub(crate) async fn deregister_table(
+        &self,
+        request: &DeregisterTableRequest,
+        table_id: TableId,
+    ) -> Result<()> {
+        let deletion_request = build_table_deletion_request(request, table_id);
+        self.0
+            .insert(deletion_request)
+            .await
+            .map(|x| {
+                if x != 1 {
+                    let table = common_catalog::format_full_table_name(
+                        &request.catalog,
+                        &request.schema,
+                        &request.table_name
+                    );
+                    warn!("Failed to delete table record from information_schema, unexpected returned result: {x}, table: {table}");
+                }
+            })
+            .with_context(|_| DeregisterTableSnafu {
+                request: request.clone(),
+            })
+    }
+
+    pub async fn register_schema(&self, catalog: String, schema: String) -> Result<usize> {
+        let insert_request = build_schema_insert_request(catalog, schema);
+        self.0
+            .insert(insert_request)
+            .await
+            .context(InsertCatalogRecordSnafu)
+    }
+
+    /// Create a stream of all entries inside system catalog table
+    pub async fn records(&self) -> Result<SendableRecordBatchStream> {
+        let full_projection = None;
+        let scan_req = ScanRequest {
+            sequence: None,
+            projection: full_projection,
+            filters: vec![],
+            output_ordering: None,
+            limit: None,
+        };
+        let stream = self
+            .0
+            .scan_to_stream(scan_req)
+            .await
+            .context(error::SystemCatalogTableScanSnafu)?;
+        Ok(stream)
+    }
+
+    pub fn as_table_ref(&self) -> TableRef {
+        self.0.clone()
+    }
+}
+
+/// Build system catalog table schema.
+/// A system catalog table consists of 6 columns, namely
+/// - entry_type: type of entry in current row, can be any variant of [EntryType].
+/// - key: a binary encoded key of entry, differs according to different entry type.
+/// - timestamp: currently not used.
+/// - value: JSON-encoded value of entry's metadata.
+/// - gmt_created: create time of this metadata.
+/// - gmt_modified: last updated time of this metadata.
+fn build_system_catalog_schema() -> RawSchema {
+    let cols = vec![
+        ColumnSchema::new(
+            "entry_type".to_string(),
+            ConcreteDataType::uint8_datatype(),
+            false,
+        ),
+        ColumnSchema::new(
+            "key".to_string(),
+            ConcreteDataType::binary_datatype(),
+            false,
+        ),
+        ColumnSchema::new(
+            "timestamp".to_string(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new(
+            "value".to_string(),
+            ConcreteDataType::binary_datatype(),
+            false,
+        ),
+        ColumnSchema::new(
+            "gmt_created".to_string(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        ),
+        ColumnSchema::new(
+            "gmt_modified".to_string(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        ),
+    ];
+
+    RawSchema::new(cols)
+}
+
+/// Formats key string for table entry in system catalog
+#[inline]
+pub fn format_table_entry_key(catalog: &str, schema: &str, table_id: TableId) -> String {
+    format!("{catalog}.{schema}.{table_id}")
+}
+
+pub fn build_table_insert_request(
+    catalog: String,
+    schema: String,
+    table_name: String,
+    table_id: TableId,
+    engine: String,
+) -> InsertRequest {
+    let entry_key = format_table_entry_key(&catalog, &schema, table_id);
+    build_insert_request(
+        EntryType::Table,
+        entry_key.as_bytes(),
+        serde_json::to_string(&TableEntryValue {
+            table_name,
+            engine,
+            is_deleted: false,
+        })
+        .unwrap()
+        .as_bytes(),
+    )
+}
+
+pub(crate) fn build_table_deletion_request(
+    request: &DeregisterTableRequest,
+    table_id: TableId,
+) -> InsertRequest {
+    let entry_key = format_table_entry_key(&request.catalog, &request.schema, table_id);
+    build_insert_request(
+        EntryType::Table,
+        entry_key.as_bytes(),
+        serde_json::to_string(&TableEntryValue {
+            table_name: "".to_string(),
+            engine: "".to_string(),
+            is_deleted: true,
+        })
+        .unwrap()
+        .as_bytes(),
+    )
+}
+
+fn build_primary_key_columns(entry_type: EntryType, key: &[u8]) -> HashMap<String, VectorRef> {
+    HashMap::from([
+        (
+            "entry_type".to_string(),
+            Arc::new(UInt8Vector::from_slice([entry_type as u8])) as VectorRef,
+        ),
+        (
+            "key".to_string(),
+            Arc::new(BinaryVector::from_slice(&[key])) as VectorRef,
+        ),
+        (
+            "timestamp".to_string(),
+            // Timestamp in key part is intentionally left to 0
+            Arc::new(TimestampMillisecondVector::from_slice([0])) as VectorRef,
+        ),
+    ])
+}
+
+pub fn build_schema_insert_request(catalog_name: String, schema_name: String) -> InsertRequest {
+    let full_schema_name = format!("{catalog_name}.{schema_name}");
+    build_insert_request(
+        EntryType::Schema,
+        full_schema_name.as_bytes(),
+        serde_json::to_string(&SchemaEntryValue {})
+            .unwrap()
+            .as_bytes(),
+    )
+}
+
+pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) -> InsertRequest {
+    let primary_key_columns = build_primary_key_columns(entry_type, key);
+
+    let mut columns_values = HashMap::with_capacity(6);
+    columns_values.extend(primary_key_columns);
+
+    let _ = columns_values.insert(
+        "value".to_string(),
+        Arc::new(BinaryVector::from_slice(&[value])) as _,
+    );
+
+    let now = util::current_time_millis();
+    let _ = columns_values.insert(
+        "gmt_created".to_string(),
+        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
+    );
+
+    let _ = columns_values.insert(
+        "gmt_modified".to_string(),
+        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
+    );
+
+    InsertRequest {
+        catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+        schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+        table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+        columns_values,
+        region_number: 0, // system catalog table has only one region
+    }
+}
+
+pub fn decode_system_catalog(
+    entry_type: Option<u8>,
+    key: Option<&[u8]>,
+    value: Option<&[u8]>,
+) -> Result<Entry> {
+    debug!(
+        "Decode system catalog entry: {:?}, {:?}, {:?}",
+        entry_type, key, value
+    );
+    let entry_type = entry_type.context(InvalidKeySnafu { key: None })?;
+    let key = String::from_utf8_lossy(key.context(InvalidKeySnafu { key: None })?);
+
+    match EntryType::try_from(entry_type)? {
+        EntryType::Catalog => {
+            // As for catalog entry, the key is a string with format: `<catalog_name>`
+            // and the value is current not used.
+            let catalog_name = key.to_string();
+            Ok(Entry::Catalog(CatalogEntry { catalog_name }))
+        }
+        EntryType::Schema => {
+            // As for schema entry, the key is a string with format: `<catalog_name>.<schema_name>`
+            // and the value is current not used.
+            let schema_parts = key.split('.').collect::<Vec<_>>();
+            ensure!(
+                schema_parts.len() == 2,
+                InvalidKeySnafu {
+                    key: Some(key.to_string())
+                }
+            );
+            Ok(Entry::Schema(SchemaEntry {
+                catalog_name: schema_parts[0].to_string(),
+                schema_name: schema_parts[1].to_string(),
+            }))
+        }
+
+        EntryType::Table => {
+            // As for table entry, the key is a string with format: `<catalog_name>.<schema_name>.<table_id>`
+            // and the value is a JSON string with format: `{"table_name": <table_name>}`
+            let table_parts = key.split('.').collect::<Vec<_>>();
+            ensure!(
+                table_parts.len() >= 3,
+                InvalidKeySnafu {
+                    key: Some(key.to_string())
+                }
+            );
+            let value = value.context(EmptyValueSnafu)?;
+            debug!("Table meta value: {}", String::from_utf8_lossy(value));
+            let table_meta: TableEntryValue =
+                serde_json::from_slice(value).context(ValueDeserializeSnafu)?;
+            let table_id = table_parts[2].parse::<TableId>().unwrap();
+            Ok(Entry::Table(TableEntry {
+                catalog_name: table_parts[0].to_string(),
+                schema_name: table_parts[1].to_string(),
+                table_name: table_meta.table_name,
+                table_id,
+                engine: table_meta.engine,
+                is_deleted: table_meta.is_deleted,
+            }))
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum EntryType {
+    Catalog = 1,
+    Schema = 2,
+    Table = 3,
+}
+
+impl TryFrom<u8> for EntryType {
+    type Error = Error;
+
+    fn try_from(value: u8) -> std::result::Result<Self, Self::Error> {
+        match value {
+            b if b == Self::Catalog as u8 => Ok(Self::Catalog),
+            b if b == Self::Schema as u8 => Ok(Self::Schema),
+            b if b == Self::Table as u8 => Ok(Self::Table),
+            b => InvalidEntryTypeSnafu {
+                entry_type: Some(b),
+            }
+            .fail(),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
+pub enum Entry {
+    Catalog(CatalogEntry),
+    Schema(SchemaEntry),
+    Table(TableEntry),
+}
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
+pub struct CatalogEntry {
+    pub catalog_name: String,
+}
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
+pub struct SchemaEntry {
+    pub catalog_name: String,
+    pub schema_name: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SchemaEntryValue;
+
+#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
+pub struct TableEntry {
+    pub catalog_name: String,
+    pub schema_name: String,
+    pub table_name: String,
+    pub table_id: TableId,
+    pub engine: String,
+    pub is_deleted: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TableEntryValue {
+    pub table_name: String,
+
+    #[serde(default = "mito_engine")]
+    pub engine: String,
+
+    #[serde(default = "not_deleted")]
+    pub is_deleted: bool,
+}
+
+fn mito_engine() -> String {
+    MITO_ENGINE.to_string()
+}
+
+fn not_deleted() -> bool {
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use common_recordbatch::RecordBatches;
+    use common_test_util::temp_dir::{create_temp_dir, TempDir};
+    use datatypes::value::Value;
+    use log_store::NoopLogStore;
+    use mito::config::EngineConfig;
+    use mito::engine::{MitoEngine, MITO_ENGINE};
+    use object_store::ObjectStore;
+    use storage::compaction::noop::NoopCompactionScheduler;
+    use storage::config::EngineConfig as StorageEngineConfig;
+    use storage::EngineImpl;
+    use table::metadata::TableType;
+    use table::metadata::TableType::Base;
+
+    use super::*;
+
+    #[test]
+    pub fn test_decode_catalog_entry() {
+        let entry = decode_system_catalog(
+            Some(EntryType::Catalog as u8),
+            Some("some_catalog".as_bytes()),
+            None,
+        )
+        .unwrap();
+        if let Entry::Catalog(e) = entry {
+            assert_eq!("some_catalog", e.catalog_name);
+        } else {
+            panic!("Unexpected type: {entry:?}");
+        }
+    }
+
+    #[test]
+    pub fn test_decode_schema_entry() {
+        let entry = decode_system_catalog(
+            Some(EntryType::Schema as u8),
+            Some("some_catalog.some_schema".as_bytes()),
+            None,
+        )
+        .unwrap();
+
+        if let Entry::Schema(e) = entry {
+            assert_eq!("some_catalog", e.catalog_name);
+            assert_eq!("some_schema", e.schema_name);
+        } else {
+            panic!("Unexpected type: {entry:?}");
+        }
+    }
+
+    #[test]
+    pub fn test_decode_table() {
+        let entry = decode_system_catalog(
+            Some(EntryType::Table as u8),
+            Some("some_catalog.some_schema.42".as_bytes()),
+            Some("{\"table_name\":\"some_table\"}".as_bytes()),
+        )
+        .unwrap();
+
+        if let Entry::Table(e) = entry {
+            assert_eq!("some_catalog", e.catalog_name);
+            assert_eq!("some_schema", e.schema_name);
+            assert_eq!("some_table", e.table_name);
+            assert_eq!(42, e.table_id);
+        } else {
+            panic!("Unexpected type: {entry:?}");
+        }
+    }
+
+    #[test]
+    pub fn test_decode_mismatch() {
+        assert!(decode_system_catalog(
+            Some(EntryType::Table as u8),
+            Some("some_catalog.some_schema.42".as_bytes()),
+            None,
+        )
+        .is_err());
+    }
+
+    #[test]
+    pub fn test_entry_type() {
+        assert_eq!(EntryType::Catalog, EntryType::try_from(1).unwrap());
+        assert_eq!(EntryType::Schema, EntryType::try_from(2).unwrap());
+        assert_eq!(EntryType::Table, EntryType::try_from(3).unwrap());
+        assert!(EntryType::try_from(4).is_err());
+    }
+
+    pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
+        let dir = create_temp_dir("system-table-test");
+        let store_dir = dir.path().to_string_lossy();
+        let mut builder = object_store::services::Fs::default();
+        let _ = builder.root(&store_dir);
+        let object_store = ObjectStore::new(builder).unwrap().finish();
+        let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
+        let table_engine = Arc::new(MitoEngine::new(
+            EngineConfig::default(),
+            EngineImpl::new(
+                StorageEngineConfig::default(),
+                Arc::new(NoopLogStore),
+                object_store.clone(),
+                noop_compaction_scheduler,
+            )
+            .unwrap(),
+            object_store,
+        ));
+        (dir, table_engine)
+    }
+
+    #[tokio::test]
+    async fn test_system_table_type() {
+        let (_dir, table_engine) = prepare_table_engine().await;
+        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
+        assert_eq!(Base, system_table.as_table_ref().table_type());
+    }
+
+    #[tokio::test]
+    async fn test_system_table_info() {
+        let (_dir, table_engine) = prepare_table_engine().await;
+        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
+        let info = system_table.as_table_ref().table_info();
+        assert_eq!(TableType::Base, info.table_type);
+        assert_eq!(SYSTEM_CATALOG_TABLE_NAME, info.name);
+        assert_eq!(SYSTEM_CATALOG_TABLE_ID, info.ident.table_id);
+        assert_eq!(SYSTEM_CATALOG_NAME, info.catalog_name);
+        assert_eq!(INFORMATION_SCHEMA_NAME, info.schema_name);
+    }
+
+    #[tokio::test]
+    async fn test_system_catalog_table_records() {
+        let (_, table_engine) = prepare_table_engine().await;
+        let catalog_table = SystemCatalogTable::new(table_engine).await.unwrap();
+
+        let result = catalog_table
+            .register_table(
+                DEFAULT_CATALOG_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string(),
+                "my_table".to_string(),
+                1,
+                MITO_ENGINE.to_string(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(result, 1);
+
+        let records = catalog_table.records().await.unwrap();
+        let mut batches = RecordBatches::try_collect(records).await.unwrap().take();
+        assert_eq!(batches.len(), 1);
+        let batch = batches.remove(0);
+        assert_eq!(batch.num_rows(), 1);
+
+        let row = batch.rows().next().unwrap();
+        let Value::UInt8(entry_type) = row[0] else {
+            unreachable!()
+        };
+        let Value::Binary(key) = row[1].clone() else {
+            unreachable!()
+        };
+        let Value::Binary(value) = row[3].clone() else {
+            unreachable!()
+        };
+        let entry = decode_system_catalog(Some(entry_type), Some(&*key), Some(&*value)).unwrap();
+        let expected = Entry::Table(TableEntry {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "my_table".to_string(),
+            table_id: 1,
+            engine: MITO_ENGINE.to_string(),
+            is_deleted: false,
+        });
+        assert_eq!(entry, expected);
+
+        catalog_table
+            .deregister_table(
+                &DeregisterTableRequest {
+                    catalog: DEFAULT_CATALOG_NAME.to_string(),
+                    schema: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "my_table".to_string(),
+                },
+                1,
+            )
+            .await
+            .unwrap();
+
+        let records = catalog_table.records().await.unwrap();
+        let batches = RecordBatches::try_collect(records).await.unwrap().take();
+        assert_eq!(batches.len(), 1);
+    }
+}

src/catalog/src/table_source.rs

@@ -123,7 +123,7 @@ mod tests {
     use session::context::QueryContext;
 
     use super::*;
-    use crate::memory::MemoryCatalogManager;
+    use crate::local::MemoryCatalogManager;
 
     #[test]
     fn test_validate_table_ref() {

src/catalog/src/tables.rs

@@ -0,0 +1,77 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The `tables` table in system catalog keeps a record of all tables created by user.
+
+use std::sync::Arc;
+
+use table::metadata::TableId;
+
+use crate::system::SystemCatalogTable;
+use crate::DeregisterTableRequest;
+
+pub struct InformationSchema {
+    pub system: Arc<SystemCatalogTable>,
+}
+
+pub struct SystemCatalog {
+    pub information_schema: Arc<InformationSchema>,
+}
+
+impl SystemCatalog {
+    pub(crate) fn new(system: SystemCatalogTable) -> Self {
+        let schema = InformationSchema {
+            system: Arc::new(system),
+        };
+        Self {
+            information_schema: Arc::new(schema),
+        }
+    }
+
+    pub async fn register_table(
+        &self,
+        catalog: String,
+        schema: String,
+        table_name: String,
+        table_id: TableId,
+        engine: String,
+    ) -> crate::error::Result<usize> {
+        self.information_schema
+            .system
+            .register_table(catalog, schema, table_name, table_id, engine)
+            .await
+    }
+
+    pub(crate) async fn deregister_table(
+        &self,
+        request: &DeregisterTableRequest,
+        table_id: TableId,
+    ) -> crate::error::Result<()> {
+        self.information_schema
+            .system
+            .deregister_table(request, table_id)
+            .await
+    }
+
+    pub async fn register_schema(
+        &self,
+        catalog: String,
+        schema: String,
+    ) -> crate::error::Result<usize> {
+        self.information_schema
+            .system
+            .register_schema(catalog, schema)
+            .await
+    }
+}

src/catalog/tests/local_catalog_tests.rs

@@ -0,0 +1,175 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use catalog::local::LocalCatalogManager;
+    use catalog::{CatalogManager, RegisterTableRequest, RenameTableRequest};
+    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use common_telemetry::{error, info};
+    use common_test_util::temp_dir::TempDir;
+    use mito::config::EngineConfig;
+    use table::engine::manager::MemoryTableEngineManager;
+    use table::table::numbers::NumbersTable;
+    use tokio::sync::Mutex;
+
+    async fn create_local_catalog_manager(
+    ) -> Result<(TempDir, LocalCatalogManager), catalog::error::Error> {
+        let (dir, object_store) =
+            mito::table::test_util::new_test_object_store("setup_mock_engine_and_table").await;
+        let mock_engine = Arc::new(mito::table::test_util::MockMitoEngine::new(
+            EngineConfig::default(),
+            mito::table::test_util::MockEngine::default(),
+            object_store,
+        ));
+        let engine_manager = Arc::new(MemoryTableEngineManager::new(mock_engine.clone()));
+        let catalog_manager = LocalCatalogManager::try_new(engine_manager).await.unwrap();
+        catalog_manager.start().await?;
+        Ok((dir, catalog_manager))
+    }
+
+    #[tokio::test]
+    async fn test_rename_table() {
+        common_telemetry::init_default_ut_logging();
+        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
+        // register table
+        let table_name = "test_table";
+        let table_id = 42;
+        let request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table: NumbersTable::table(table_id),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        // rename table
+        let new_table_name = "table_t";
+        let rename_table_req = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        assert!(catalog_manager
+            .rename_table(rename_table_req)
+            .await
+            .unwrap());
+
+        let registered_table = catalog_manager
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
+    }
+
+    #[tokio::test]
+    async fn test_duplicate_register() {
+        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
+        let request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: "test_table".to_string(),
+            table_id: 42,
+            table: NumbersTable::table(42),
+        };
+        assert!(catalog_manager
+            .register_table(request.clone())
+            .await
+            .unwrap());
+
+        // register table with same table id will succeed with 0 as return val.
+        assert!(!catalog_manager.register_table(request).await.unwrap());
+
+        let err = catalog_manager
+            .register_table(RegisterTableRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+                table_name: "test_table".to_string(),
+                table_id: 43,
+                table: NumbersTable::table(43),
+            })
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("Table `greptime.public.test_table` already exists"),
+            "Actual error message: {err}",
+        );
+    }
+
+    #[test]
+    fn test_concurrent_register() {
+        common_telemetry::init_default_ut_logging();
+        let rt = Arc::new(tokio::runtime::Builder::new_multi_thread().build().unwrap());
+        let (_dir, catalog_manager) =
+            rt.block_on(async { create_local_catalog_manager().await.unwrap() });
+        let catalog_manager = Arc::new(catalog_manager);
+
+        let succeed = Arc::new(Mutex::new(None));
+
+        let mut handles = Vec::with_capacity(8);
+        for i in 0..8 {
+            let catalog = catalog_manager.clone();
+            let succeed = succeed.clone();
+            let handle = rt.spawn(async move {
+                let table_id = 42 + i;
+                let table = NumbersTable::table(table_id);
+                let table_info = table.table_info();
+                let req = RegisterTableRequest {
+                    catalog: DEFAULT_CATALOG_NAME.to_string(),
+                    schema: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "test_table".to_string(),
+                    table_id,
+                    table,
+                };
+                match catalog.register_table(req).await {
+                    Ok(res) => {
+                        if res {
+                            let mut succeed = succeed.lock().await;
+                            info!("Successfully registered table: {}", table_id);
+                            *succeed = Some(table_info);
+                        }
+                    }
+                    Err(_) => {
+                        error!("Failed to register table {}", table_id);
+                    }
+                }
+            });
+            handles.push(handle);
+        }
+
+        rt.block_on(async move {
+            for handle in handles {
+                handle.await.unwrap();
+            }
+            let guard = succeed.lock().await;
+            let table_info = guard.as_ref().unwrap();
+            let table_registered = catalog_manager
+                .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "test_table")
+                .await
+                .unwrap()
+                .unwrap();
+            assert_eq!(
+                table_registered.table_info().ident.table_id,
+                table_info.ident.table_id
+            );
+        });
+    }
+}

src/catalog/tests/remote_catalog_tests.rs

@@ -0,0 +1,453 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(assert_matches)]
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+    use std::collections::HashSet;
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    use catalog::remote::mock::MockTableEngine;
+    use catalog::remote::region_alive_keeper::RegionAliveKeepers;
+    use catalog::remote::{CachedMetaKvBackend, RemoteCatalogManager};
+    use catalog::{CatalogManager, RegisterSchemaRequest, RegisterTableRequest};
+    use common_catalog::consts::{
+        DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
+    };
+    use common_meta::helper::CatalogValue;
+    use common_meta::ident::TableIdent;
+    use common_meta::key::catalog_name::CatalogNameKey;
+    use common_meta::key::TableMetadataManager;
+    use common_meta::kv_backend::memory::MemoryKvBackend;
+    use common_meta::kv_backend::KvBackend;
+    use common_meta::rpc::store::{CompareAndPutRequest, PutRequest};
+    use datatypes::schema::RawSchema;
+    use table::engine::manager::{MemoryTableEngineManager, TableEngineManagerRef};
+    use table::engine::{EngineContext, TableEngineRef};
+    use table::requests::CreateTableRequest;
+    use table::test_util::EmptyTable;
+    use tokio::time::Instant;
+
+    struct TestingComponents {
+        catalog_manager: Arc<RemoteCatalogManager>,
+        table_engine_manager: TableEngineManagerRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
+    }
+
+    impl TestingComponents {
+        fn table_engine(&self) -> TableEngineRef {
+            self.table_engine_manager.engine(MITO_ENGINE).unwrap()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_cached_backend() {
+        let backend = CachedMetaKvBackend::wrap(Arc::new(MemoryKvBackend::default()));
+
+        let default_catalog_key = CatalogNameKey::new(DEFAULT_CATALOG_NAME).to_string();
+
+        let req = PutRequest::new()
+            .with_key(default_catalog_key.as_bytes())
+            .with_value(CatalogValue.as_bytes().unwrap());
+        backend.put(req).await.unwrap();
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        let _ = ret.unwrap();
+
+        let req = CompareAndPutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_expect(CatalogValue.as_bytes().unwrap())
+            .with_value(b"123".to_vec());
+        let _ = backend.compare_and_put(req).await.unwrap();
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        assert_eq!(b"123", ret.as_ref().unwrap().value.as_slice());
+
+        let req = PutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_value(b"1234".to_vec());
+        let _ = backend.put(req).await;
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        assert_eq!(b"1234", ret.unwrap().value.as_slice());
+
+        backend
+            .delete(b"__catalog_name/greptime", false)
+            .await
+            .unwrap();
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        assert!(ret.is_none());
+    }
+
+    async fn prepare_components(node_id: u64) -> TestingComponents {
+        let backend = Arc::new(MemoryKvBackend::default());
+
+        let req = PutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_value(b"".to_vec());
+        backend.put(req).await.unwrap();
+
+        let req = PutRequest::new()
+            .with_key(b"__schema_name/greptime-public".to_vec())
+            .with_value(b"".to_vec());
+        backend.put(req).await.unwrap();
+
+        let cached_backend = Arc::new(CachedMetaKvBackend::wrap(backend));
+
+        let table_engine = Arc::new(MockTableEngine::default());
+        let engine_manager = Arc::new(MemoryTableEngineManager::alias(
+            MITO_ENGINE.to_string(),
+            table_engine,
+        ));
+
+        let region_alive_keepers = Arc::new(RegionAliveKeepers::new(engine_manager.clone(), 5000));
+
+        let catalog_manager = RemoteCatalogManager::new(
+            engine_manager.clone(),
+            node_id,
+            region_alive_keepers.clone(),
+            Arc::new(TableMetadataManager::new(cached_backend)),
+        );
+        catalog_manager.start().await.unwrap();
+
+        TestingComponents {
+            catalog_manager: Arc::new(catalog_manager),
+            table_engine_manager: engine_manager,
+            region_alive_keepers,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_remote_catalog_default() {
+        common_telemetry::init_default_ut_logging();
+        let node_id = 42;
+        let TestingComponents {
+            catalog_manager, ..
+        } = prepare_components(node_id).await;
+        assert_eq!(
+            vec![DEFAULT_CATALOG_NAME.to_string()],
+            catalog_manager.catalog_names().await.unwrap()
+        );
+
+        let mut schema_names = catalog_manager
+            .schema_names(DEFAULT_CATALOG_NAME)
+            .await
+            .unwrap();
+        schema_names.sort_unstable();
+        assert_eq!(
+            vec![
+                INFORMATION_SCHEMA_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string()
+            ],
+            schema_names
+        );
+    }
+
+    #[tokio::test]
+    async fn test_remote_catalog_register_nonexistent() {
+        common_telemetry::init_default_ut_logging();
+        let node_id = 42;
+        let components = prepare_components(node_id).await;
+
+        // register a new table with an nonexistent catalog
+        let catalog_name = "nonexistent_catalog".to_string();
+        let schema_name = "nonexistent_schema".to_string();
+        let table_name = "fail_table".to_string();
+        // this schema has no effect
+        let table_schema = RawSchema::new(vec![]);
+        let table = components
+            .table_engine()
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: 1,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    desc: None,
+                    schema: table_schema,
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
+                },
+            )
+            .await
+            .unwrap();
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name,
+            schema: schema_name,
+            table_name,
+            table_id: 1,
+            table,
+        };
+        let res = components.catalog_manager.register_table(reg_req).await;
+
+        // because nonexistent_catalog does not exist yet.
+        assert_matches!(
+            res.err().unwrap(),
+            catalog::error::Error::CatalogNotFound { .. }
+        );
+    }
+
+    #[tokio::test]
+    async fn test_register_table() {
+        let node_id = 42;
+        let components = prepare_components(node_id).await;
+        let mut schema_names = components
+            .catalog_manager
+            .schema_names(DEFAULT_CATALOG_NAME)
+            .await
+            .unwrap();
+        schema_names.sort_unstable();
+        assert_eq!(
+            vec![
+                INFORMATION_SCHEMA_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string(),
+            ],
+            schema_names
+        );
+
+        // register a new table with an nonexistent catalog
+        let catalog_name = DEFAULT_CATALOG_NAME.to_string();
+        let schema_name = DEFAULT_SCHEMA_NAME.to_string();
+        let table_name = "test_table".to_string();
+        let table_id = 1;
+        // this schema has no effect
+        let table_schema = RawSchema::new(vec![]);
+        let table = components
+            .table_engine()
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: table_id,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: table_name.clone(),
+                    desc: None,
+                    schema: table_schema,
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
+                },
+            )
+            .await
+            .unwrap();
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name,
+            schema: schema_name,
+            table_name: table_name.clone(),
+            table_id,
+            table,
+        };
+        assert!(components
+            .catalog_manager
+            .register_table(reg_req)
+            .await
+            .unwrap());
+        assert_eq!(
+            vec![table_name],
+            components
+                .catalog_manager
+                .table_names(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
+                .await
+                .unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_register_catalog_schema_table() {
+        let node_id = 42;
+        let components = prepare_components(node_id).await;
+
+        let catalog_name = "test_catalog".to_string();
+        let schema_name = "nonexistent_schema".to_string();
+
+        // register catalog to catalog manager
+        assert!(components
+            .catalog_manager
+            .clone()
+            .register_catalog(catalog_name.clone())
+            .await
+            .is_ok());
+        assert_eq!(
+            HashSet::<String>::from_iter(vec![
+                DEFAULT_CATALOG_NAME.to_string(),
+                catalog_name.clone()
+            ]),
+            HashSet::from_iter(components.catalog_manager.catalog_names().await.unwrap())
+        );
+
+        let table_to_register = components
+            .table_engine()
+            .create_table(
+                &EngineContext {},
+                CreateTableRequest {
+                    id: 2,
+                    catalog_name: catalog_name.clone(),
+                    schema_name: schema_name.clone(),
+                    table_name: "".to_string(),
+                    desc: None,
+                    schema: RawSchema::new(vec![]),
+                    region_numbers: vec![0],
+                    primary_key_indices: vec![],
+                    create_if_not_exists: false,
+                    table_options: Default::default(),
+                    engine: MITO_ENGINE.to_string(),
+                },
+            )
+            .await
+            .unwrap();
+
+        let reg_req = RegisterTableRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+            table_name: " fail_table".to_string(),
+            table_id: 2,
+            table: table_to_register,
+        };
+        // this register will fail since schema does not exist yet
+        assert_matches!(
+            components
+                .catalog_manager
+                .register_table(reg_req.clone())
+                .await
+                .unwrap_err(),
+            catalog::error::Error::SchemaNotFound { .. }
+        );
+
+        let register_schema_request = RegisterSchemaRequest {
+            catalog: catalog_name.to_string(),
+            schema: schema_name.to_string(),
+        };
+        assert!(components
+            .catalog_manager
+            .register_schema(register_schema_request)
+            .await
+            .expect("Register schema should not fail"));
+        assert!(components
+            .catalog_manager
+            .register_table(reg_req)
+            .await
+            .unwrap());
+
+        assert_eq!(
+            HashSet::from([schema_name.clone(), INFORMATION_SCHEMA_NAME.to_string()]),
+            components
+                .catalog_manager
+                .schema_names(&catalog_name)
+                .await
+                .unwrap()
+                .into_iter()
+                .collect()
+        )
+    }
+
+    #[tokio::test]
+    async fn test_register_table_before_and_after_region_alive_keeper_started() {
+        let components = prepare_components(42).await;
+        let catalog_manager = &components.catalog_manager;
+        let region_alive_keepers = &components.region_alive_keepers;
+
+        let table_before = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_before".to_string(),
+            table_id: 1,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_before.catalog.clone(),
+            schema: table_before.schema.clone(),
+            table_name: table_before.table.clone(),
+            table_id: table_before.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_before.table_id,
+                catalog_name: table_before.catalog.clone(),
+                schema_name: table_before.schema.clone(),
+                table_name: table_before.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_before.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert region alive countdown is not started
+        assert!(deadline > far_future);
+
+        region_alive_keepers.start().await;
+
+        let table_after = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_after".to_string(),
+            table_id: 2,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_after.catalog.clone(),
+            schema: table_after.schema.clone(),
+            table_name: table_after.table.clone(),
+            table_id: table_after.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_after.table_id,
+                catalog_name: table_after.catalog.clone(),
+                schema_name: table_after.schema.clone(),
+                table_name: table_after.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_after.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered after [RegionAliveKeepers] started
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_before.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered before [RegionAliveKeepers] started, too
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+    }
+}

src/client/Cargo.toml

@@ -11,12 +11,10 @@ testing = []
 api = { workspace = true }
 arrow-flight.workspace = true
 async-stream.workspace = true
-async-trait.workspace = true
 common-base = { workspace = true }
 common-catalog = { workspace = true }
 common-error = { workspace = true }
 common-grpc = { workspace = true }
-common-macro = { workspace = true }
 common-meta = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
@@ -27,11 +25,10 @@ datatypes = { workspace = true }
 derive_builder.workspace = true
 enum_dispatch = "0.3"
 futures-util.workspace = true
-moka = { workspace = true, features = ["future"] }
+moka = { version = "0.9", features = ["future"] }
 parking_lot = "0.12"
 prost.workspace = true
 rand.workspace = true
-session = { workspace = true }
 snafu.workspace = true
 tokio-stream = { version = "0.1", features = ["net"] }
 tokio.workspace = true

src/client/examples/logical.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, SemanticType, TableId};
+use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, TableId};
 use client::{Client, Database};
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
 use prost::Message;
@@ -41,27 +41,21 @@ async fn run() {
         column_defs: vec![
             ColumnDef {
                 name: "timestamp".to_string(),
-                data_type: ColumnDataType::TimestampMillisecond as i32,
+                datatype: ColumnDataType::TimestampMillisecond as i32,
                 is_nullable: false,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Timestamp as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "key".to_string(),
-                data_type: ColumnDataType::Uint64 as i32,
+                datatype: ColumnDataType::Uint64 as i32,
                 is_nullable: false,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Tag as i32,
-                comment: String::new(),
             },
             ColumnDef {
                 name: "value".to_string(),
-                data_type: ColumnDataType::Uint64 as i32,
+                datatype: ColumnDataType::Uint64 as i32,
                 is_nullable: false,
                 default_constraint: vec![],
-                semantic_type: SemanticType::Field as i32,
-                comment: String::new(),
             },
         ],
         time_index: "timestamp".to_string(),
@@ -69,6 +63,7 @@ async fn run() {
         create_if_not_exists: false,
         table_options: Default::default(),
         table_id: Some(TableId { id: 1024 }),
+        region_numbers: vec![0],
         engine: MITO_ENGINE.to_string(),
     };
 
@@ -78,7 +73,7 @@ async fn run() {
 
     let logical = mock_logical_plan();
     event!(Level::INFO, "plan size: {:#?}", logical.len());
-    let result = db.logical_plan(logical, 0).await.unwrap();
+    let result = db.logical_plan(logical, None).await.unwrap();
 
     event!(Level::INFO, "result: {:#?}", result);
 }

src/client/examples/stream_ingest.rs

@@ -42,14 +42,14 @@ async fn run() {
         .insert(vec![to_insert_request(weather_records_1())])
         .await
     {
-        error!("Error: {e:?}");
+        error!("Error: {e}");
     }
 
     if let Err(e) = stream_inserter
         .insert(vec![to_insert_request(weather_records_2())])
         .await
     {
-        error!("Error: {e:?}");
+        error!("Error: {e}");
     }
 
     let result = stream_inserter.finish().await;
@@ -59,7 +59,7 @@ async fn run() {
             info!("Rows written: {rows}");
         }
         Err(e) => {
-            error!("Error: {e:?}");
+            error!("Error: {e}");
         }
     };
 }
@@ -131,7 +131,7 @@ fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
         Column {
             column_name: "ts".to_owned(),
             values: Some(column::Values {
-                timestamp_millisecond_values: timestamp_millis,
+                ts_millisecond_values: timestamp_millis,
                 ..Default::default()
             }),
             semantic_type: SemanticType::Timestamp as i32,
@@ -177,5 +177,6 @@ fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
         table_name: "weather_demo".to_owned(),
         columns,
         row_count: rows as u32,
+        ..Default::default()
     }
 }

src/client/src/client.rs

@@ -138,46 +138,24 @@ impl Client {
         Ok((addr, channel))
     }
 
-    fn max_grpc_recv_message_size(&self) -> usize {
-        self.inner
-            .channel_manager
-            .config()
-            .max_recv_message_size
-            .as_bytes() as usize
-    }
-
-    fn max_grpc_send_message_size(&self) -> usize {
-        self.inner
-            .channel_manager
-            .config()
-            .max_send_message_size
-            .as_bytes() as usize
-    }
-
     pub(crate) fn make_flight_client(&self) -> Result<FlightClient> {
         let (addr, channel) = self.find_channel()?;
         Ok(FlightClient {
             addr,
-            client: FlightServiceClient::new(channel)
-                .max_decoding_message_size(self.max_grpc_recv_message_size())
-                .max_encoding_message_size(self.max_grpc_send_message_size()),
+            client: FlightServiceClient::new(channel),
         })
     }
 
     pub(crate) fn make_database_client(&self) -> Result<DatabaseClient> {
         let (_, channel) = self.find_channel()?;
         Ok(DatabaseClient {
-            inner: GreptimeDatabaseClient::new(channel)
-                .max_decoding_message_size(self.max_grpc_recv_message_size())
-                .max_encoding_message_size(self.max_grpc_send_message_size()),
+            inner: GreptimeDatabaseClient::new(channel),
         })
     }
 
     pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
         let (_, channel) = self.find_channel()?;
-        Ok(PbRegionClient::new(channel)
-            .max_decoding_message_size(self.max_grpc_recv_message_size())
-            .max_encoding_message_size(self.max_grpc_send_message_size()))
+        Ok(PbRegionClient::new(channel))
     }
 
     pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {

src/client/src/client_manager.rs

@@ -13,15 +13,12 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
-use std::sync::Arc;
 use std::time::Duration;
 
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
-use common_meta::datanode_manager::{Datanode, DatanodeManager};
 use common_meta::peer::Peer;
 use moka::future::{Cache, CacheBuilder};
 
-use crate::region::RegionRequester;
 use crate::Client;
 
 pub struct DatanodeClients {
@@ -43,15 +40,6 @@ impl Debug for DatanodeClients {
     }
 }
 
-#[async_trait::async_trait]
-impl DatanodeManager for DatanodeClients {
-    async fn datanode(&self, datanode: &Peer) -> Arc<dyn Datanode> {
-        let client = self.get_client(datanode).await;
-
-        Arc::new(RegionRequester::new(client))
-    }
-}
-
 impl DatanodeClients {
     pub fn new(config: ChannelConfig) -> Self {
         Self {

src/client/src/database.rs

@@ -17,9 +17,9 @@ use api::v1::ddl_request::Expr as DdlExpr;
 use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
-    AlterExpr, AuthHeader, CreateTableExpr, DdlRequest, DeleteRequests, DropTableExpr,
-    GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest, RequestHeader,
-    RowInsertRequests, TruncateTableExpr,
+    AlterExpr, AuthHeader, CompactTableExpr, CreateTableExpr, DdlRequest, DeleteRequests,
+    DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest,
+    RequestHeader, RowInsertRequests, TruncateTableExpr,
 };
 use arrow_flight::Ticket;
 use async_stream::stream;
@@ -147,13 +147,13 @@ impl Database {
 
     async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = self.client.make_database_client()?.inner;
-        let request = self.to_rpc_request(request, 0);
+        let request = self.to_rpc_request(request, None);
         let response = client.handle(request).await?.into_inner();
         from_grpc_response(response)
     }
 
     #[inline]
-    fn to_rpc_request(&self, request: Request, trace_id: u64) -> GreptimeRequest {
+    fn to_rpc_request(&self, request: Request, trace_id: Option<u64>) -> GreptimeRequest {
         GreptimeRequest {
             header: Some(RequestHeader {
                 catalog: self.catalog.clone(),
@@ -161,27 +161,28 @@ impl Database {
                 authorization: self.ctx.auth_header.clone(),
                 dbname: self.dbname.clone(),
                 trace_id,
-                span_id: 0,
+                span_id: None,
             }),
             request: Some(request),
         }
     }
 
-    pub async fn sql<S>(&self, sql: S) -> Result<Output>
-    where
-        S: AsRef<str>,
-    {
+    pub async fn sql(&self, sql: &str) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_SQL);
         self.do_get(
             Request::Query(QueryRequest {
-                query: Some(Query::Sql(sql.as_ref().to_string())),
+                query: Some(Query::Sql(sql.to_string())),
             }),
-            0,
+            None,
         )
         .await
     }
 
-    pub async fn logical_plan(&self, logical_plan: Vec<u8>, trace_id: u64) -> Result<Output> {
+    pub async fn logical_plan(
+        &self,
+        logical_plan: Vec<u8>,
+        trace_id: Option<u64>,
+    ) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_LOGICAL_PLAN);
         self.do_get(
             Request::Query(QueryRequest {
@@ -209,7 +210,7 @@ impl Database {
                     step: step.to_string(),
                 })),
             }),
-            0,
+            None,
         )
         .await
     }
@@ -220,7 +221,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::CreateTable(expr)),
             }),
-            0,
+            None,
         )
         .await
     }
@@ -231,7 +232,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::Alter(expr)),
             }),
-            0,
+            None,
         )
         .await
     }
@@ -242,7 +243,29 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::DropTable(expr)),
             }),
-            0,
+            None,
+        )
+        .await
+    }
+
+    pub async fn flush_table(&self, expr: FlushTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_FLUSH_TABLE);
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::FlushTable(expr)),
+            }),
+            None,
+        )
+        .await
+    }
+
+    pub async fn compact_table(&self, expr: CompactTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_COMPACT_TABLE);
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::CompactTable(expr)),
+            }),
+            None,
         )
         .await
     }
@@ -253,12 +276,12 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::TruncateTable(expr)),
             }),
-            0,
+            None,
         )
         .await
     }
 
-    async fn do_get(&self, request: Request, trace_id: u64) -> Result<Output> {
+    async fn do_get(&self, request: Request, trace_id: Option<u64>) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
         let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
         let request = self.to_rpc_request(request, trace_id);
@@ -279,7 +302,7 @@ impl Database {
                 source: BoxedError::new(ServerSnafu { code, msg }.build()),
             };
             logging::error!(
-                "Failed to do Flight get, addr: {}, code: {}, source: {:?}",
+                "Failed to do Flight get, addr: {}, code: {}, source: {}",
                 client.addr(),
                 tonic_code,
                 error

src/client/src/error.rs

@@ -17,37 +17,29 @@ use std::any::Any;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
-use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use tonic::{Code, Status};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display("Illegal Flight messages, reason: {}", reason))]
     IllegalFlightMessages { reason: String, location: Location },
 
-    #[snafu(display("Failed to do Flight get, code: {}", tonic_code))]
+    #[snafu(display("Failed to do Flight get, code: {}, source: {}", tonic_code, source))]
     FlightGet {
         addr: String,
         tonic_code: Code,
         source: BoxedError,
     },
 
-    #[snafu(display("Failure occurs during handling request"))]
-    HandleRequest {
-        location: Location,
-        source: BoxedError,
-    },
-
-    #[snafu(display("Failed to convert FlightData"))]
+    #[snafu(display("Failed to convert FlightData, source: {}", source))]
     ConvertFlightData {
         location: Location,
         source: common_grpc::Error,
     },
 
-    #[snafu(display("Column datatype error"))]
+    #[snafu(display("Column datatype error, source: {}", source))]
     ColumnDataType {
         location: Location,
         source: api::error::Error,
@@ -59,16 +51,17 @@ pub enum Error {
     #[snafu(display("Missing required field in protobuf, field: {}", field))]
     MissingField { field: String, location: Location },
 
-    #[snafu(display("Failed to create gRPC channel, peer address: {}", addr))]
+    #[snafu(display(
+        "Failed to create gRPC channel, peer address: {}, source: {}",
+        addr,
+        source
+    ))]
     CreateChannel {
         addr: String,
         location: Location,
         source: common_grpc::error::Error,
     },
 
-    #[snafu(display("Failed to request RegionServer, code: {}", code))]
-    RegionServer { code: Code, source: BoxedError },
-
     // Server error carried in Tonic Status's metadata.
     #[snafu(display("{}", msg))]
     Server { code: StatusCode, msg: String },
@@ -92,9 +85,7 @@ impl ErrorExt for Error {
             | Error::ClientStreaming { .. } => StatusCode::Internal,
 
             Error::Server { code, .. } => *code,
-            Error::FlightGet { source, .. }
-            | Error::HandleRequest { source, .. }
-            | Error::RegionServer { source, .. } => source.status_code(),
+            Error::FlightGet { source, .. } => source.status_code(),
             Error::CreateChannel { source, .. } | Error::ConvertFlightData { source, .. } => {
                 source.status_code()
             }

src/client/src/lib.rs

@@ -26,8 +26,6 @@ use api::v1::greptime_response::Response;
 use api::v1::{AffectedRows, GreptimeResponse};
 pub use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_error::status_code::StatusCode;
-pub use common_query::Output;
-pub use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
 use snafu::OptionExt;
 
 pub use self::client::Client;

src/client/src/metrics.rs

@@ -21,6 +21,8 @@ pub const METRIC_GRPC_SQL: &str = "grpc.sql";
 pub const METRIC_GRPC_LOGICAL_PLAN: &str = "grpc.logical_plan";
 pub const METRIC_GRPC_ALTER: &str = "grpc.alter";
 pub const METRIC_GRPC_DROP_TABLE: &str = "grpc.drop_table";
+pub const METRIC_GRPC_FLUSH_TABLE: &str = "grpc.flush_table";
+pub const METRIC_GRPC_COMPACT_TABLE: &str = "grpc.compact_table";
 pub const METRIC_GRPC_TRUNCATE_TABLE: &str = "grpc.truncate_table";
 pub const METRIC_GRPC_DO_GET: &str = "grpc.do_get";
 pub(crate) const METRIC_REGION_REQUEST_GRPC: &str = "grpc.region_request";

src/client/src/region.rs

@@ -12,146 +12,44 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::region::{QueryRequest, RegionRequest, RegionResponse};
+use api::v1::region::{region_request, RegionRequest, RegionRequestHeader, RegionResponse};
 use api::v1::ResponseHeader;
-use arrow_flight::Ticket;
-use async_stream::stream;
-use async_trait::async_trait;
-use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_grpc::flight::{FlightDecoder, FlightMessage};
-use common_meta::datanode_manager::{AffectedRows, Datanode};
-use common_meta::error::{self as meta_error, Result as MetaResult};
-use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
-use common_telemetry::{error, timer};
-use prost::Message;
-use snafu::{location, Location, OptionExt, ResultExt};
-use tokio_stream::StreamExt;
+use common_telemetry::timer;
+use snafu::OptionExt;
 
-use crate::error::Error::RegionServer;
-use crate::error::{
-    self, ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
-    MissingFieldSnafu, Result, ServerSnafu,
-};
-use crate::{metrics, Client, Error};
+use crate::error::{IllegalDatabaseResponseSnafu, Result, ServerSnafu};
+use crate::{metrics, Client};
+
+type AffectedRows = u64;
 
 #[derive(Debug)]
 pub struct RegionRequester {
+    trace_id: Option<u64>,
+    span_id: Option<u64>,
     client: Client,
 }
 
-#[async_trait]
-impl Datanode for RegionRequester {
-    async fn handle(&self, request: RegionRequest) -> MetaResult<AffectedRows> {
-        self.handle_inner(request).await.map_err(|err| {
-            if matches!(err, RegionServer { .. }) {
-                meta_error::Error::RetryLater {
-                    source: BoxedError::new(err),
-                }
-            } else {
-                meta_error::Error::External {
-                    source: BoxedError::new(err),
-                    location: location!(),
-                }
-            }
-        })
-    }
-
-    async fn handle_query(&self, request: QueryRequest) -> MetaResult<SendableRecordBatchStream> {
-        let ticket = Ticket {
-            ticket: request.encode_to_vec().into(),
-        };
-        self.do_get_inner(ticket)
-            .await
-            .map_err(BoxedError::new)
-            .context(meta_error::ExternalSnafu)
-    }
-}
-
 impl RegionRequester {
     pub fn new(client: Client) -> Self {
-        Self { client }
+        // TODO(LFC): Pass in trace_id and span_id from some context when we have it.
+        Self {
+            trace_id: None,
+            span_id: None,
+            client,
+        }
     }
 
-    pub async fn do_get_inner(&self, ticket: Ticket) -> Result<SendableRecordBatchStream> {
-        let mut flight_client = self.client.make_flight_client()?;
-        let response = flight_client
-            .mut_inner()
-            .do_get(ticket)
-            .await
-            .map_err(|e| {
-                let tonic_code = e.code();
-                let e: error::Error = e.into();
-                let code = e.status_code();
-                let msg = e.to_string();
-                let error = Error::FlightGet {
-                    tonic_code,
-                    addr: flight_client.addr().to_string(),
-                    source: BoxedError::new(ServerSnafu { code, msg }.build()),
-                };
-                error!(
-                    e; "Failed to do Flight get, addr: {}, code: {}",
-                    flight_client.addr(),
-                    tonic_code
-                );
-                error
-            })?;
+    pub async fn handle(self, request: region_request::Body) -> Result<AffectedRows> {
+        let request_type = request.as_ref().to_string();
 
-        let flight_data_stream = response.into_inner();
-        let mut decoder = FlightDecoder::default();
-
-        let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
-            flight_data
-                .map_err(Error::from)
-                .and_then(|data| decoder.try_decode(data).context(ConvertFlightDataSnafu))
-        });
-
-        let Some(first_flight_message) = flight_message_stream.next().await else {
-            return IllegalFlightMessagesSnafu {
-                reason: "Expect the response not to be empty",
-            }
-            .fail();
+        let request = RegionRequest {
+            header: Some(RegionRequestHeader {
+                trace_id: self.trace_id,
+                span_id: self.span_id,
+            }),
+            body: Some(request),
         };
-        let FlightMessage::Schema(schema) = first_flight_message? else {
-            return IllegalFlightMessagesSnafu {
-                reason: "Expect schema to be the first flight message",
-            }
-            .fail();
-        };
-
-        let stream = Box::pin(stream!({
-            while let Some(flight_message) = flight_message_stream.next().await {
-                let flight_message = flight_message
-                    .map_err(BoxedError::new)
-                    .context(ExternalSnafu)?;
-                let FlightMessage::Recordbatch(record_batch) = flight_message else {
-                    yield IllegalFlightMessagesSnafu {
-                            reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"
-                        }
-                        .fail()
-                        .map_err(BoxedError::new)
-                        .context(ExternalSnafu);
-                    break;
-                };
-                yield Ok(record_batch);
-            }
-        }));
-        let record_batch_stream = RecordBatchStreamAdaptor {
-            schema,
-            stream,
-            output_ordering: None,
-        };
-        Ok(Box::pin(record_batch_stream))
-    }
-
-    async fn handle_inner(&self, request: RegionRequest) -> Result<AffectedRows> {
-        let request_type = request
-            .body
-            .as_ref()
-            .with_context(|| MissingFieldSnafu { field: "body" })?
-            .as_ref()
-            .to_string();
 
         let _timer = timer!(
             metrics::METRIC_REGION_REQUEST_GRPC,
@@ -163,31 +61,15 @@ impl RegionRequester {
         let RegionResponse {
             header,
             affected_rows,
-        } = client
-            .handle(request)
-            .await
-            .map_err(|e| {
-                let code = e.code();
-                let err: error::Error = e.into();
-                // Uses `Error::RegionServer` instead of `Error::Server`
-                error::Error::RegionServer {
-                    code,
-                    source: BoxedError::new(err),
-                }
-            })?
-            .into_inner();
+        } = client.handle(request).await?.into_inner();
 
         check_response_header(header)?;
 
         Ok(affected_rows)
     }
-
-    pub async fn handle(&self, request: RegionRequest) -> Result<AffectedRows> {
-        self.handle_inner(request).await
-    }
 }
 
-pub fn check_response_header(header: Option<ResponseHeader>) -> Result<()> {
+fn check_response_header(header: Option<ResponseHeader>) -> Result<()> {
     let status = header
         .and_then(|header| header.status)
         .context(IllegalDatabaseResponseSnafu {

src/cmd/Cargo.toml

@@ -23,12 +23,8 @@ chrono.workspace = true
 clap = { version = "3.1", features = ["derive"] }
 client = { workspace = true }
 common-base = { workspace = true }
-common-catalog = { workspace = true }
-common-config = { workspace = true }
 common-error = { workspace = true }
-common-macro = { workspace = true }
 common-meta = { workspace = true }
-common-procedure = { workspace = true }
 common-query = { workspace = true }
 common-recordbatch = { workspace = true }
 common-telemetry = { workspace = true, features = [
@@ -39,24 +35,18 @@ datanode = { workspace = true }
 datatypes = { workspace = true }
 either = "1.8"
 etcd-client.workspace = true
-file-engine = { workspace = true }
 frontend = { workspace = true }
 futures.workspace = true
-lazy_static.workspace = true
 meta-client = { workspace = true }
 meta-srv = { workspace = true }
 metrics.workspace = true
-mito2 = { workspace = true }
 nu-ansi-term = "0.46"
 partition = { workspace = true }
-plugins.workspace = true
 prost.workspace = true
 query = { workspace = true }
 rand.workspace = true
-regex.workspace = true
 rustyline = "10.1"
 serde.workspace = true
-serde_json.workspace = true
 servers = { workspace = true }
 session = { workspace = true }
 snafu.workspace = true

src/cmd/src/bin/greptime.rs

@@ -116,7 +116,7 @@ impl SubCommand {
                 Ok(Application::Metasrv(app))
             }
             (SubCommand::Standalone(cmd), Options::Standalone(opts)) => {
-                let app = cmd.build(*opts).await?;
+                let app = cmd.build(opts.fe_opts, opts.dn_opts).await?;
                 Ok(Application::Standalone(app))
             }
             (SubCommand::Cli(cmd), Options::Cli(_)) => {

src/cmd/src/cli.rs

@@ -14,7 +14,6 @@
 
 mod bench;
 mod cmd;
-mod export;
 mod helper;
 mod repl;
 // TODO(weny): Removes it
@@ -28,7 +27,6 @@ use common_telemetry::logging::LoggingOptions;
 pub use repl::Repl;
 use upgrade::UpgradeCommand;
 
-use self::export::ExportCommand;
 use crate::error::Result;
 use crate::options::{Options, TopLevelOptions};
 
@@ -80,19 +78,17 @@ impl Command {
 
 #[derive(Parser)]
 enum SubCommand {
-    // Attach(AttachCommand),
+    Attach(AttachCommand),
     Upgrade(UpgradeCommand),
     Bench(BenchTableMetadataCommand),
-    Export(ExportCommand),
 }
 
 impl SubCommand {
     async fn build(self) -> Result<Instance> {
         match self {
-            // SubCommand::Attach(cmd) => cmd.build().await,
+            SubCommand::Attach(cmd) => cmd.build().await,
             SubCommand::Upgrade(cmd) => cmd.build().await,
             SubCommand::Bench(cmd) => cmd.build().await,
-            SubCommand::Export(cmd) => cmd.build().await,
         }
     }
 }
@@ -108,9 +104,51 @@ pub(crate) struct AttachCommand {
 }
 
 impl AttachCommand {
-    #[allow(dead_code)]
     async fn build(self) -> Result<Instance> {
         let repl = Repl::try_new(&self).await?;
         Ok(Instance::Repl(repl))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_load_options() {
+        let cmd = Command {
+            cmd: SubCommand::Attach(AttachCommand {
+                grpc_addr: String::from(""),
+                meta_addr: None,
+                disable_helper: false,
+            }),
+        };
+
+        let opts = cmd.load_options(TopLevelOptions::default()).unwrap();
+        let logging_opts = opts.logging_options();
+        assert_eq!("/tmp/greptimedb/logs", logging_opts.dir);
+        assert!(logging_opts.level.is_none());
+        assert!(!logging_opts.enable_jaeger_tracing);
+    }
+
+    #[test]
+    fn test_top_level_options() {
+        let cmd = Command {
+            cmd: SubCommand::Attach(AttachCommand {
+                grpc_addr: String::from(""),
+                meta_addr: None,
+                disable_helper: false,
+            }),
+        };
+
+        let opts = cmd
+            .load_options(TopLevelOptions {
+                log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
+                log_level: Some("debug".to_string()),
+            })
+            .unwrap();
+        let logging_opts = opts.logging_options();
+        assert_eq!("/tmp/greptimedb/test/logs", logging_opts.dir);
+        assert_eq!("debug", logging_opts.level.as_ref().unwrap());
+    }
+}

src/cmd/src/cli/bench.rs

@@ -120,6 +120,7 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
         created_on: chrono::DateTime::default(),
         primary_key_indices: vec![],
         next_column_id: columns as u32 + 1,
+        engine_options: Default::default(),
         value_indices: vec![],
         options: Default::default(),
         region_numbers: (1..=100).collect(),

src/cmd/src/cli/export.rs

@@ -1,395 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::path::Path;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use clap::{Parser, ValueEnum};
-use client::{Client, Database, DEFAULT_SCHEMA_NAME};
-use common_query::Output;
-use common_recordbatch::util::collect;
-use common_telemetry::{debug, error, info, warn};
-use datatypes::scalars::ScalarVector;
-use datatypes::vectors::{StringVector, Vector};
-use snafu::{OptionExt, ResultExt};
-use tokio::fs::File;
-use tokio::io::AsyncWriteExt;
-use tokio::sync::Semaphore;
-
-use crate::cli::{Instance, Tool};
-use crate::error::{
-    CollectRecordBatchesSnafu, ConnectServerSnafu, EmptyResultSnafu, Error, FileIoSnafu,
-    InvalidDatabaseNameSnafu, NotDataFromOutputSnafu, RequestDatabaseSnafu, Result,
-};
-
-type TableReference = (String, String, String);
-
-#[derive(Debug, Default, Clone, ValueEnum)]
-enum ExportTarget {
-    /// Corresponding to `SHOW CREATE TABLE`
-    #[default]
-    CreateTable,
-    /// Corresponding to `EXPORT TABLE`
-    TableData,
-}
-
-#[derive(Debug, Default, Parser)]
-pub struct ExportCommand {
-    /// Server address to connect
-    #[clap(long)]
-    addr: String,
-
-    /// Directory to put the exported data. E.g.: /tmp/greptimedb-export
-    #[clap(long)]
-    output_dir: String,
-
-    /// The name of the catalog to export. Default to "greptime-*"".
-    #[clap(long, default_value = "")]
-    database: String,
-
-    /// Parallelism of the export.
-    #[clap(long, short = 'j', default_value = "1")]
-    export_jobs: usize,
-
-    /// Max retry times for each job.
-    #[clap(long, default_value = "3")]
-    max_retry: usize,
-
-    /// Things to export
-    #[clap(long, short = 't', value_enum)]
-    target: ExportTarget,
-}
-
-impl ExportCommand {
-    pub async fn build(&self) -> Result<Instance> {
-        let client = Client::with_urls([self.addr.clone()]);
-        client
-            .health_check()
-            .await
-            .with_context(|_| ConnectServerSnafu {
-                addr: self.addr.clone(),
-            })?;
-        let (catalog, schema) = split_database(&self.database)?;
-        let database_client = Database::new(
-            catalog.clone(),
-            schema.clone().unwrap_or(DEFAULT_SCHEMA_NAME.to_string()),
-            client,
-        );
-
-        Ok(Instance::Tool(Box::new(Export {
-            client: database_client,
-            catalog,
-            schema,
-            output_dir: self.output_dir.clone(),
-            parallelism: self.export_jobs,
-            target: self.target.clone(),
-        })))
-    }
-}
-
-pub struct Export {
-    client: Database,
-    catalog: String,
-    schema: Option<String>,
-    output_dir: String,
-    parallelism: usize,
-    target: ExportTarget,
-}
-
-impl Export {
-    /// Iterate over all db names.
-    ///
-    /// Newbie: `db_name` is catalog + schema.
-    async fn iter_db_names(&self) -> Result<Vec<(String, String)>> {
-        if let Some(schema) = &self.schema {
-            Ok(vec![(self.catalog.clone(), schema.clone())])
-        } else {
-            let mut client = self.client.clone();
-            client.set_catalog(self.catalog.clone());
-            let result =
-                client
-                    .sql("show databases")
-                    .await
-                    .with_context(|_| RequestDatabaseSnafu {
-                        sql: "show databases".to_string(),
-                    })?;
-            let Output::Stream(stream) = result else {
-                NotDataFromOutputSnafu.fail()?
-            };
-            let record_batch = collect(stream)
-                .await
-                .context(CollectRecordBatchesSnafu)?
-                .pop()
-                .context(EmptyResultSnafu)?;
-            let schemas = record_batch
-                .column(0)
-                .as_any()
-                .downcast_ref::<StringVector>()
-                .unwrap();
-            let mut result = Vec::with_capacity(schemas.len());
-            for i in 0..schemas.len() {
-                let schema = schemas.get_data(i).unwrap().to_owned();
-                result.push((self.catalog.clone(), schema));
-            }
-            Ok(result)
-        }
-    }
-
-    /// Return a list of [`TableReference`] to be exported.
-    /// Includes all tables under the given `catalog` and `schema`
-    async fn get_table_list(&self, catalog: &str, schema: &str) -> Result<Vec<TableReference>> {
-        // TODO: SQL injection hurts
-        let sql = format!(
-            "select table_catalog, table_schema, table_name from \
-            information_schema.tables where table_type = \'BASE TABLE\'\
-            and table_catalog = \'{catalog}\' and table_schema = \'{schema}\'",
-        );
-        let mut client = self.client.clone();
-        client.set_catalog(catalog);
-        client.set_schema(schema);
-        let result = client
-            .sql(&sql)
-            .await
-            .with_context(|_| RequestDatabaseSnafu { sql })?;
-        let Output::Stream(stream) = result else {
-            NotDataFromOutputSnafu.fail()?
-        };
-        let Some(record_batch) = collect(stream)
-            .await
-            .context(CollectRecordBatchesSnafu)?
-            .pop()
-        else {
-            return Ok(vec![]);
-        };
-
-        debug!("Fetched table list: {}", record_batch.pretty_print());
-
-        if record_batch.num_rows() == 0 {
-            return Ok(vec![]);
-        }
-
-        let mut result = Vec::with_capacity(record_batch.num_rows());
-        let catalog_column = record_batch
-            .column(0)
-            .as_any()
-            .downcast_ref::<StringVector>()
-            .unwrap();
-        let schema_column = record_batch
-            .column(1)
-            .as_any()
-            .downcast_ref::<StringVector>()
-            .unwrap();
-        let table_column = record_batch
-            .column(2)
-            .as_any()
-            .downcast_ref::<StringVector>()
-            .unwrap();
-        for i in 0..record_batch.num_rows() {
-            let catalog = catalog_column.get_data(i).unwrap().to_owned();
-            let schema = schema_column.get_data(i).unwrap().to_owned();
-            let table = table_column.get_data(i).unwrap().to_owned();
-            result.push((catalog, schema, table));
-        }
-
-        Ok(result)
-    }
-
-    async fn show_create_table(&self, catalog: &str, schema: &str, table: &str) -> Result<String> {
-        let sql = format!("show create table {}.{}.{}", catalog, schema, table);
-        let mut client = self.client.clone();
-        client.set_catalog(catalog);
-        client.set_schema(schema);
-        let result = client
-            .sql(&sql)
-            .await
-            .with_context(|_| RequestDatabaseSnafu { sql })?;
-        let Output::Stream(stream) = result else {
-            NotDataFromOutputSnafu.fail()?
-        };
-        let record_batch = collect(stream)
-            .await
-            .context(CollectRecordBatchesSnafu)?
-            .pop()
-            .context(EmptyResultSnafu)?;
-        let create_table = record_batch
-            .column(1)
-            .as_any()
-            .downcast_ref::<StringVector>()
-            .unwrap()
-            .get_data(0)
-            .unwrap();
-
-        Ok(format!("{create_table};\n"))
-    }
-
-    async fn export_create_table(&self) -> Result<()> {
-        let semaphore = Arc::new(Semaphore::new(self.parallelism));
-        let db_names = self.iter_db_names().await?;
-        let db_count = db_names.len();
-        let mut tasks = Vec::with_capacity(db_names.len());
-        for (catalog, schema) in db_names {
-            let semaphore_moved = semaphore.clone();
-            tasks.push(async move {
-                let _permit = semaphore_moved.acquire().await.unwrap();
-                let table_list = self.get_table_list(&catalog, &schema).await?;
-                let table_count = table_list.len();
-                tokio::fs::create_dir_all(&self.output_dir)
-                    .await
-                    .context(FileIoSnafu)?;
-                let output_file =
-                    Path::new(&self.output_dir).join(format!("{catalog}-{schema}.sql"));
-                let mut file = File::create(output_file).await.context(FileIoSnafu)?;
-                for (c, s, t) in table_list {
-                    match self.show_create_table(&c, &s, &t).await {
-                        Err(e) => {
-                            error!(e; "Failed to export table {}.{}.{}", c, s, t)
-                        }
-                        Ok(create_table) => {
-                            file.write_all(create_table.as_bytes())
-                                .await
-                                .context(FileIoSnafu)?;
-                        }
-                    }
-                }
-                info!("finished exporting {catalog}.{schema} with {table_count} tables",);
-                Ok::<(), Error>(())
-            });
-        }
-
-        let success = futures::future::join_all(tasks)
-            .await
-            .into_iter()
-            .filter(|r| match r {
-                Ok(_) => true,
-                Err(e) => {
-                    error!(e; "export job failed");
-                    false
-                }
-            })
-            .count();
-
-        info!("success {success}/{db_count} jobs");
-
-        Ok(())
-    }
-
-    async fn export_table_data(&self) -> Result<()> {
-        let semaphore = Arc::new(Semaphore::new(self.parallelism));
-        let db_names = self.iter_db_names().await?;
-        let db_count = db_names.len();
-        let mut tasks = Vec::with_capacity(db_names.len());
-        for (catalog, schema) in db_names {
-            let semaphore_moved = semaphore.clone();
-            tasks.push(async move {
-                let _permit = semaphore_moved.acquire().await.unwrap();
-                tokio::fs::create_dir_all(&self.output_dir)
-                    .await
-                    .context(FileIoSnafu)?;
-                let output_dir = Path::new(&self.output_dir).join(format!("{catalog}-{schema}/"));
-
-                let mut client = self.client.clone();
-                client.set_catalog(catalog.clone());
-                client.set_schema(schema.clone());
-
-                // copy database to
-                let sql = format!(
-                    "copy database {} to '{}' with (format='parquet');",
-                    schema,
-                    output_dir.to_str().unwrap()
-                );
-                client
-                    .sql(sql.clone())
-                    .await
-                    .context(RequestDatabaseSnafu { sql })?;
-                info!("finished exporting {catalog}.{schema} data");
-
-                // export copy from sql
-                let dir_filenames = match output_dir.read_dir() {
-                    Ok(dir) => dir,
-                    Err(_) => {
-                        warn!("empty database {catalog}.{schema}");
-                        return Ok(());
-                    }
-                };
-
-                let copy_from_file =
-                    Path::new(&self.output_dir).join(format!("{catalog}-{schema}_copy_from.sql"));
-                let mut file = File::create(copy_from_file).await.context(FileIoSnafu)?;
-
-                let copy_from_sql = dir_filenames
-                    .into_iter()
-                    .map(|file| {
-                        let file = file.unwrap();
-                        let filename = file.file_name().into_string().unwrap();
-
-                        format!(
-                            "copy {} from '{}' with (format='parquet');\n",
-                            filename.replace(".parquet", ""),
-                            file.path().to_str().unwrap()
-                        )
-                    })
-                    .collect::<Vec<_>>()
-                    .join("");
-                file.write_all(copy_from_sql.as_bytes())
-                    .await
-                    .context(FileIoSnafu)?;
-
-                info!("finished exporting {catalog}.{schema} copy_from.sql");
-
-                Ok::<(), Error>(())
-            });
-        }
-
-        let success = futures::future::join_all(tasks)
-            .await
-            .into_iter()
-            .filter(|r| match r {
-                Ok(_) => true,
-                Err(e) => {
-                    error!(e; "export job failed");
-                    false
-                }
-            })
-            .count();
-
-        info!("success {success}/{db_count} jobs");
-
-        Ok(())
-    }
-}
-
-#[async_trait]
-impl Tool for Export {
-    async fn do_work(&self) -> Result<()> {
-        match self.target {
-            ExportTarget::CreateTable => self.export_create_table().await,
-            ExportTarget::TableData => self.export_table_data().await,
-        }
-    }
-}
-
-/// Split at `-`.
-fn split_database(database: &str) -> Result<(String, Option<String>)> {
-    let (catalog, schema) = database
-        .split_once('-')
-        .with_context(|| InvalidDatabaseNameSnafu {
-            database: database.to_string(),
-        })?;
-    if schema == "*" {
-        Ok((catalog.to_string(), None))
-    } else {
-        Ok((catalog.to_string(), Some(schema.to_string())))
-    }
-}

src/cmd/src/cli/repl.rs

@@ -16,16 +16,20 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
 
-use catalog::kvbackend::{CachedMetaKvBackend, KvBackendCatalogManager};
+use catalog::remote::CachedMetaKvBackend;
 use client::client_manager::DatanodeClients;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_base::Plugins;
 use common_error::ext::ErrorExt;
+use common_meta::key::TableMetadataManager;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
 use common_telemetry::logging;
 use either::Either;
+use frontend::catalog::FrontendCatalogManager;
 use meta_client::client::MetaClientBuilder;
+use partition::manager::PartitionRuleManager;
+use partition::route::TableRoutes;
 use query::datafusion::DatafusionQueryEngine;
 use query::logical_optimizer::LogicalOptimizer;
 use query::parser::QueryLanguageParser;
@@ -35,7 +39,7 @@ use query::QueryEngine;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 use session::context::QueryContext;
-use snafu::ResultExt;
+use snafu::{ErrorCompat, ResultExt};
 use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 
 use crate::cli::cmd::ReplCommand;
@@ -148,7 +152,7 @@ impl Repl {
             .await
             .map_err(|e| {
                 let status_code = e.status_code();
-                let root_cause = e.output_msg();
+                let root_cause = e.iter_chain().last().unwrap();
                 println!("Error: {}({status_code}), {root_cause}", status_code as u32)
             })
             .is_ok()
@@ -176,7 +180,7 @@ impl Repl {
                 .encode(&plan)
                 .context(SubstraitEncodeLogicalPlanSnafu)?;
 
-            self.database.logical_plan(plan.to_vec(), 0).await
+            self.database.logical_plan(plan.to_vec(), None).await
         } else {
             self.database.sql(&sql).await
         }
@@ -250,19 +254,24 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
 
     let cached_meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
 
+    let table_routes = Arc::new(TableRoutes::new(meta_client));
+    let partition_manager = Arc::new(PartitionRuleManager::new(table_routes));
+
     let datanode_clients = Arc::new(DatanodeClients::default());
 
-    let catalog_list = KvBackendCatalogManager::new(
+    let catalog_list = Arc::new(FrontendCatalogManager::new(
         cached_meta_backend.clone(),
         cached_meta_backend.clone(),
+        partition_manager,
         datanode_clients,
-    );
-    let plugins: Plugins = Default::default();
+        Arc::new(TableMetadataManager::new(cached_meta_backend)),
+    ));
+    let plugins: Arc<Plugins> = Default::default();
     let state = Arc::new(QueryEngineState::new(
         catalog_list,
-        None,
-        None,
         false,
+        None,
+        None,
         plugins.clone(),
     ));
 

src/cmd/src/cli/upgrade.rs

@@ -17,15 +17,15 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use clap::Parser;
 use client::api::v1::meta::TableRouteValue;
-use common_meta::ddl::utils::region_storage_path;
 use common_meta::error as MetaError;
+use common_meta::helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue};
 use common_meta::key::catalog_name::{CatalogNameKey, CatalogNameValue};
-use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue, RegionInfo};
+use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue};
 use common_meta::key::schema_name::{SchemaNameKey, SchemaNameValue};
 use common_meta::key::table_info::{TableInfoKey, TableInfoValue};
 use common_meta::key::table_name::{TableNameKey, TableNameValue};
 use common_meta::key::table_region::{TableRegionKey, TableRegionValue};
-use common_meta::key::table_route::{TableRouteKey, TableRouteValue as NextTableRouteValue};
+use common_meta::key::table_route::{NextTableRouteKey, TableRouteValue as NextTableRouteValue};
 use common_meta::key::{RegionDistribution, TableMetaKey};
 use common_meta::range_stream::PaginationStream;
 use common_meta::rpc::router::TableRoute;
@@ -39,7 +39,6 @@ use meta_srv::service::store::etcd::EtcdStore;
 use meta_srv::service::store::kv::{KvBackendAdapter, KvStoreRef};
 use prost::Message;
 use snafu::ResultExt;
-use v1_helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue};
 
 use crate::cli::{Instance, Tool};
 use crate::error::{self, ConnectEtcdSnafu, Result};
@@ -155,7 +154,7 @@ impl MigrateTableMetadata {
         let new_table_value = NextTableRouteValue::new(table_route.region_routes);
 
         let table_id = table_route.table.id as u32;
-        let new_key = TableRouteKey::new(table_id);
+        let new_key = NextTableRouteKey::new(table_id);
         info!("Creating '{new_key}'");
 
         if self.dryrun {
@@ -387,11 +386,6 @@ impl MigrateTableMetadata {
 
     async fn create_datanode_table_keys(&self, value: &TableGlobalValue) {
         let table_id = value.table_id();
-        let engine = value.table_info.meta.engine.as_str();
-        let region_storage_path = region_storage_path(
-            &value.table_info.catalog_name,
-            &value.table_info.schema_name,
-        );
         let region_distribution: RegionDistribution =
             value.regions_id_map.clone().into_iter().collect();
 
@@ -400,18 +394,7 @@ impl MigrateTableMetadata {
             .map(|(datanode_id, regions)| {
                 let k = DatanodeTableKey::new(datanode_id, table_id);
                 info!("Creating DatanodeTableKey '{k}' => {regions:?}");
-                (
-                    k,
-                    DatanodeTableValue::new(
-                        table_id,
-                        regions,
-                        RegionInfo {
-                            engine: engine.to_string(),
-                            region_storage_path: region_storage_path.clone(),
-                            region_options: (&value.table_info.meta.options).into(),
-                        },
-                    ),
-                )
+                (k, DatanodeTableValue::new(table_id, regions))
             })
             .collect::<Vec<_>>();
 
@@ -426,148 +409,3 @@ impl MigrateTableMetadata {
         }
     }
 }
-
-#[deprecated(since = "0.4.0", note = "Used for migrate old version(v0.3) metadata")]
-mod v1_helper {
-    use std::collections::HashMap;
-    use std::fmt::{Display, Formatter};
-
-    use err::{DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu};
-    use lazy_static::lazy_static;
-    use regex::Regex;
-    use serde::{Deserialize, Serialize};
-    use snafu::{ensure, OptionExt, ResultExt};
-    use table::metadata::{RawTableInfo, TableId};
-
-    pub const CATALOG_KEY_PREFIX: &str = "__c";
-    pub const SCHEMA_KEY_PREFIX: &str = "__s";
-
-    /// The pattern of a valid catalog, schema or table name.
-    const NAME_PATTERN: &str = "[a-zA-Z_:][a-zA-Z0-9_:]*";
-
-    lazy_static! {
-        static ref CATALOG_KEY_PATTERN: Regex =
-            Regex::new(&format!("^{CATALOG_KEY_PREFIX}-({NAME_PATTERN})$")).unwrap();
-    }
-
-    lazy_static! {
-        static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!(
-            "^{SCHEMA_KEY_PREFIX}-({NAME_PATTERN})-({NAME_PATTERN})$"
-        ))
-        .unwrap();
-    }
-
-    /// Table global info contains necessary info for a datanode to create table regions, including
-    /// table id, table meta(schema...), region id allocation across datanodes.
-    #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-    pub struct TableGlobalValue {
-        /// Id of datanode that created the global table info kv. only for debugging.
-        pub node_id: u64,
-        /// Allocation of region ids across all datanodes.
-        pub regions_id_map: HashMap<u64, Vec<u32>>,
-        pub table_info: RawTableInfo,
-    }
-
-    impl TableGlobalValue {
-        pub fn table_id(&self) -> TableId {
-            self.table_info.ident.table_id
-        }
-    }
-
-    pub struct CatalogKey {
-        pub catalog_name: String,
-    }
-
-    impl Display for CatalogKey {
-        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-            f.write_str(CATALOG_KEY_PREFIX)?;
-            f.write_str("-")?;
-            f.write_str(&self.catalog_name)
-        }
-    }
-
-    impl CatalogKey {
-        pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-            let key = s.as_ref();
-            let captures = CATALOG_KEY_PATTERN
-                .captures(key)
-                .context(InvalidCatalogSnafu { key })?;
-            ensure!(captures.len() == 2, InvalidCatalogSnafu { key });
-            Ok(Self {
-                catalog_name: captures[1].to_string(),
-            })
-        }
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    pub struct CatalogValue;
-
-    pub struct SchemaKey {
-        pub catalog_name: String,
-        pub schema_name: String,
-    }
-
-    impl Display for SchemaKey {
-        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-            f.write_str(SCHEMA_KEY_PREFIX)?;
-            f.write_str("-")?;
-            f.write_str(&self.catalog_name)?;
-            f.write_str("-")?;
-            f.write_str(&self.schema_name)
-        }
-    }
-
-    impl SchemaKey {
-        pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-            let key = s.as_ref();
-            let captures = SCHEMA_KEY_PATTERN
-                .captures(key)
-                .context(InvalidCatalogSnafu { key })?;
-            ensure!(captures.len() == 3, InvalidCatalogSnafu { key });
-            Ok(Self {
-                catalog_name: captures[1].to_string(),
-                schema_name: captures[2].to_string(),
-            })
-        }
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    pub struct SchemaValue;
-
-    macro_rules! define_catalog_value {
-    ( $($val_ty: ty), *) => {
-            $(
-                impl $val_ty {
-                    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-                        serde_json::from_str(s.as_ref())
-                            .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() })
-                    }
-
-                    pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self, Error> {
-                         Self::parse(&String::from_utf8_lossy(bytes.as_ref()))
-                    }
-                }
-            )*
-        }
-    }
-
-    define_catalog_value!(TableGlobalValue);
-
-    mod err {
-        use snafu::{Location, Snafu};
-
-        #[derive(Debug, Snafu)]
-        #[snafu(visibility(pub))]
-        pub enum Error {
-            #[snafu(display("Invalid catalog info: {}", key))]
-            InvalidCatalog { key: String, location: Location },
-
-            #[snafu(display("Failed to deserialize catalog entry value: {}", raw))]
-            DeserializeCatalogEntryValue {
-                raw: String,
-                location: Location,
-                source: serde_json::error::Error,
-            },
-        }
-    }
-}

src/cmd/src/datanode.rs

@@ -16,8 +16,7 @@ use std::time::Duration;
 
 use clap::Parser;
 use common_telemetry::logging;
-use datanode::config::DatanodeOptions;
-use datanode::datanode::{Datanode, DatanodeBuilder};
+use datanode::datanode::{Datanode, DatanodeOptions};
 use meta_client::MetaClientOptions;
 use servers::Mode;
 use snafu::ResultExt;
@@ -31,10 +30,6 @@ pub struct Instance {
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        plugins::start_datanode_plugins(self.datanode.plugins())
-            .await
-            .context(StartDatanodeSnafu)?;
-
         self.datanode.start().await.context(StartDatanodeSnafu)
     }
 
@@ -134,7 +129,7 @@ impl StartCommand {
         }
 
         if let Some(metasrv_addrs) = &self.metasrv_addr {
-            opts.meta_client
+            opts.meta_client_options
                 .get_or_insert_with(MetaClientOptions::default)
                 .metasrv_addrs = metasrv_addrs.clone();
             opts.mode = Mode::Distributed;
@@ -156,29 +151,24 @@ impl StartCommand {
         }
 
         if let Some(http_addr) = &self.http_addr {
-            opts.http.addr = http_addr.clone();
+            opts.http_opts.addr = http_addr.clone();
         }
 
         if let Some(http_timeout) = self.http_timeout {
-            opts.http.timeout = Duration::from_secs(http_timeout)
+            opts.http_opts.timeout = Duration::from_secs(http_timeout)
         }
 
         // Disable dashboard in datanode.
-        opts.http.disable_dashboard = true;
+        opts.http_opts.disable_dashboard = true;
 
         Ok(Options::Datanode(Box::new(opts)))
     }
 
-    async fn build(self, mut opts: DatanodeOptions) -> Result<Instance> {
-        let plugins = plugins::setup_datanode_plugins(&mut opts)
-            .await
-            .context(StartDatanodeSnafu)?;
-
+    async fn build(self, opts: DatanodeOptions) -> Result<Instance> {
         logging::info!("Datanode start command: {:#?}", self);
         logging::info!("Datanode options: {:#?}", opts);
 
-        let datanode = DatanodeBuilder::new(opts, None, plugins)
-            .build()
+        let datanode = Datanode::new(opts, Default::default())
             .await
             .context(StartDatanodeSnafu)?;
 
@@ -193,8 +183,9 @@ mod tests {
 
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::config::{CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig};
-    use servers::heartbeat_options::HeartbeatOptions;
+    use datanode::datanode::{
+        CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig,
+    };
     use servers::Mode;
 
     use super::*;
@@ -211,14 +202,11 @@ mod tests {
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
 
-            [heartbeat]
-            interval = "300ms"
-
-            [meta_client]
+            [meta_client_options]
             metasrv_addrs = ["127.0.0.1:3002"]
-            timeout = "3s"
-            connect_timeout = "5s"
-            ddl_timeout = "10s"
+            timeout_millis = 3000
+            connect_timeout_millis = 5000
+            ddl_timeout_millis= 10000
             tcp_nodelay = true
 
             [wal]
@@ -261,33 +249,25 @@ mod tests {
 
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
-        assert_eq!("/other/wal", options.wal.dir.unwrap());
 
+        assert_eq!("/other/wal", options.wal.dir.unwrap());
         assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
         assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
         assert_eq!(1024 * 1024 * 1024 * 50, options.wal.purge_threshold.0);
         assert!(!options.wal.sync_write);
 
-        let HeartbeatOptions {
-            interval: heart_beat_interval,
-            ..
-        } = options.heartbeat;
-
-        assert_eq!(300, heart_beat_interval.as_millis());
-
         let MetaClientOptions {
             metasrv_addrs: metasrv_addr,
-            timeout,
-            connect_timeout,
-            ddl_timeout,
+            timeout_millis,
+            connect_timeout_millis,
             tcp_nodelay,
-            ..
-        } = options.meta_client.unwrap();
+            ddl_timeout_millis,
+        } = options.meta_client_options.unwrap();
 
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
-        assert_eq!(5000, connect_timeout.as_millis());
-        assert_eq!(10000, ddl_timeout.as_millis());
-        assert_eq!(3000, timeout.as_millis());
+        assert_eq!(5000, connect_timeout_millis);
+        assert_eq!(10000, ddl_timeout_millis);
+        assert_eq!(3000, timeout_millis);
         assert!(tcp_nodelay);
         assert_eq!("/tmp/greptimedb/", options.storage.data_home);
         assert!(matches!(
@@ -380,9 +360,9 @@ mod tests {
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
 
-            [meta_client]
-            timeout = "3s"
-            connect_timeout = "5s"
+            [meta_client_options]
+            timeout_millis = 3000
+            connect_timeout_millis = 5000
             tcp_nodelay = true
 
             [wal]
@@ -433,10 +413,10 @@ mod tests {
                     Some("99"),
                 ),
                 (
-                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
                     [
                         env_prefix.to_string(),
-                        "meta_client".to_uppercase(),
+                        "meta_client_options".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -463,7 +443,7 @@ mod tests {
                     Some(Duration::from_secs(9))
                 );
                 assert_eq!(
-                    opts.meta_client.unwrap().metasrv_addrs,
+                    opts.meta_client_options.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),

src/cmd/src/error.rs

@@ -16,76 +16,56 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use config::ConfigError;
 use rustyline::error::ReadlineError;
 use snafu::{Location, Snafu};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to create default catalog and schema"))]
-    InitMetadata {
-        location: Location,
-        source: common_meta::error::Error,
-    },
-
-    #[snafu(display("Failed to iter stream"))]
+    #[snafu(display("Failed to iter stream, source: {}", source))]
     IterStream {
         location: Location,
         source: common_meta::error::Error,
     },
 
-    #[snafu(display("Failed to start procedure manager"))]
-    StartProcedureManager {
-        location: Location,
-        source: common_procedure::error::Error,
-    },
-
-    #[snafu(display("Failed to stop procedure manager"))]
-    StopProcedureManager {
-        location: Location,
-        source: common_procedure::error::Error,
-    },
-
-    #[snafu(display("Failed to start datanode"))]
+    #[snafu(display("Failed to start datanode, source: {}", source))]
     StartDatanode {
         location: Location,
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown datanode"))]
+    #[snafu(display("Failed to shutdown datanode, source: {}", source))]
     ShutdownDatanode {
         location: Location,
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Failed to start frontend"))]
+    #[snafu(display("Failed to start frontend, source: {}", source))]
     StartFrontend {
         location: Location,
         source: frontend::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown frontend"))]
+    #[snafu(display("Failed to shutdown frontend, source: {}", source))]
     ShutdownFrontend {
         location: Location,
         source: frontend::error::Error,
     },
 
-    #[snafu(display("Failed to build meta server"))]
+    #[snafu(display("Failed to build meta server, source: {}", source))]
     BuildMetaServer {
         location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to start meta server"))]
+    #[snafu(display("Failed to start meta server, source: {}", source))]
     StartMetaServer {
         location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown meta server"))]
+    #[snafu(display("Failed to shutdown meta server, source: {}", source))]
     ShutdownMetaServer {
         location: Location,
         source: meta_srv::error::Error,
@@ -97,7 +77,13 @@ pub enum Error {
     #[snafu(display("Illegal config: {}", msg))]
     IllegalConfig { msg: String, location: Location },
 
-    #[snafu(display("Unsupported selector type: {}", selector_type))]
+    #[snafu(display("Illegal auth config: {}", source))]
+    IllegalAuthConfig {
+        location: Location,
+        source: auth::error::Error,
+    },
+
+    #[snafu(display("Unsupported selector type, {} source: {}", selector_type, source))]
     UnsupportedSelectorType {
         selector_type: String,
         location: Location,
@@ -107,124 +93,80 @@ pub enum Error {
     #[snafu(display("Invalid REPL command: {reason}"))]
     InvalidReplCommand { reason: String },
 
-    #[snafu(display("Cannot create REPL"))]
+    #[snafu(display("Cannot create REPL: {}", source))]
     ReplCreation {
-        #[snafu(source)]
-        error: ReadlineError,
+        source: ReadlineError,
         location: Location,
     },
 
-    #[snafu(display("Error reading command"))]
+    #[snafu(display("Error reading command: {}", source))]
     Readline {
-        #[snafu(source)]
-        error: ReadlineError,
+        source: ReadlineError,
         location: Location,
     },
 
-    #[snafu(display("Failed to request database, sql: {sql}"))]
+    #[snafu(display("Failed to request database, sql: {sql}, source: {source}"))]
     RequestDatabase {
         sql: String,
         location: Location,
         source: client::Error,
     },
 
-    #[snafu(display("Failed to collect RecordBatches"))]
+    #[snafu(display("Failed to collect RecordBatches, source: {source}"))]
     CollectRecordBatches {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to pretty print Recordbatches"))]
+    #[snafu(display("Failed to pretty print Recordbatches, source: {source}"))]
     PrettyPrintRecordBatches {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to start Meta client"))]
+    #[snafu(display("Failed to start Meta client, source: {}", source))]
     StartMetaClient {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Failed to parse SQL: {}", sql))]
+    #[snafu(display("Failed to parse SQL: {}, source: {}", sql, source))]
     ParseSql {
         sql: String,
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to plan statement"))]
+    #[snafu(display("Failed to plan statement, source: {}", source))]
     PlanStatement {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to encode logical plan in substrait"))]
+    #[snafu(display("Failed to encode logical plan in substrait, source: {}", source))]
     SubstraitEncodeLogicalPlan {
         location: Location,
         source: substrait::error::Error,
     },
 
-    #[snafu(display("Failed to load layered config"))]
+    #[snafu(display("Failed to load layered config, source: {}", source))]
     LoadLayeredConfig {
-        #[snafu(source)]
-        error: ConfigError,
+        source: ConfigError,
         location: Location,
     },
 
-    #[snafu(display("Failed to start catalog manager"))]
+    #[snafu(display("Failed to start catalog manager, source: {}", source))]
     StartCatalogManager {
         location: Location,
         source: catalog::error::Error,
     },
 
-    #[snafu(display("Failed to connect to Etcd at {etcd_addr}"))]
+    #[snafu(display("Failed to connect to Etcd at {etcd_addr}, source: {}", source))]
     ConnectEtcd {
         etcd_addr: String,
-        #[snafu(source)]
-        error: etcd_client::Error,
+        source: etcd_client::Error,
         location: Location,
     },
-
-    #[snafu(display("Failed to connect server at {addr}"))]
-    ConnectServer {
-        addr: String,
-        source: client::error::Error,
-        location: Location,
-    },
-
-    #[snafu(display("Failed to serde json"))]
-    SerdeJson {
-        #[snafu(source)]
-        error: serde_json::error::Error,
-        location: Location,
-    },
-
-    #[snafu(display("Expect data from output, but got another thing"))]
-    NotDataFromOutput { location: Location },
-
-    #[snafu(display("Empty result from output"))]
-    EmptyResult { location: Location },
-
-    #[snafu(display("Failed to manipulate file"))]
-    FileIo {
-        location: Location,
-        #[snafu(source)]
-        error: std::io::Error,
-    },
-
-    #[snafu(display("Invalid database name: {}", database))]
-    InvalidDatabaseName {
-        location: Location,
-        database: String,
-    },
-
-    #[snafu(display("Failed to create directory {}", dir))]
-    CreateDir {
-        dir: String,
-        #[snafu(source)]
-        error: std::io::Error,
-    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -240,21 +182,14 @@ impl ErrorExt for Error {
             Error::ShutdownMetaServer { source, .. } => source.status_code(),
             Error::BuildMetaServer { source, .. } => source.status_code(),
             Error::UnsupportedSelectorType { source, .. } => source.status_code(),
-            Error::IterStream { source, .. } | Error::InitMetadata { source, .. } => {
-                source.status_code()
-            }
-            Error::ConnectServer { source, .. } => source.status_code(),
+            Error::IterStream { source, .. } => source.status_code(),
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }
             | Error::InvalidReplCommand { .. }
-            | Error::ConnectEtcd { .. }
-            | Error::NotDataFromOutput { .. }
-            | Error::CreateDir { .. }
-            | Error::EmptyResult { .. }
-            | Error::InvalidDatabaseName { .. } => StatusCode::InvalidArguments,
-            Error::StartProcedureManager { source, .. }
-            | Error::StopProcedureManager { source, .. } => source.status_code(),
+            | Error::IllegalAuthConfig { .. }
+            | Error::ConnectEtcd { .. } => StatusCode::InvalidArguments,
+
             Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
             Error::RequestDatabase { source, .. } => source.status_code(),
             Error::CollectRecordBatches { source, .. }
@@ -265,8 +200,6 @@ impl ErrorExt for Error {
             }
             Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
             Error::StartCatalogManager { source, .. } => source.status_code(),
-
-            Error::SerdeJson { .. } | Error::FileIo { .. } => StatusCode::Unexpected,
         }
     }
 

src/cmd/src/frontend.rs

@@ -12,18 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::time::Duration;
+use std::sync::Arc;
 
+use auth::UserProviderRef;
 use clap::Parser;
+use common_base::Plugins;
 use common_telemetry::logging;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
+use frontend::service_config::InfluxdbOptions;
 use meta_client::MetaClientOptions;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::ResultExt;
 
-use crate::error::{self, Result, StartFrontendSnafu};
+use crate::error::{self, IllegalAuthConfigSnafu, Result, StartCatalogManagerSnafu};
 use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
@@ -32,11 +35,16 @@ pub struct Instance {
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        plugins::start_frontend_plugins(self.frontend.plugins().clone())
+        self.frontend
+            .catalog_manager()
+            .start()
             .await
-            .context(StartFrontendSnafu)?;
+            .context(StartCatalogManagerSnafu)?;
 
-        self.frontend.start().await.context(StartFrontendSnafu)
+        self.frontend
+            .start()
+            .await
+            .context(error::StartFrontendSnafu)
     }
 
     pub async fn stop(&self) -> Result<()> {
@@ -87,9 +95,7 @@ pub struct StartCommand {
     #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
-    http_timeout: Option<u64>,
-    #[clap(long)]
-    rpc_addr: Option<String>,
+    grpc_addr: Option<String>,
     #[clap(long)]
     mysql_addr: Option<String>,
     #[clap(long)]
@@ -139,75 +145,86 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            opts.http.addr = addr.clone()
-        }
-
-        if let Some(http_timeout) = self.http_timeout {
-            opts.http.timeout = Duration::from_secs(http_timeout)
+            if let Some(http_opts) = &mut opts.http_options {
+                http_opts.addr = addr.clone()
+            }
         }
 
         if let Some(disable_dashboard) = self.disable_dashboard {
-            opts.http.disable_dashboard = disable_dashboard;
+            opts.http_options
+                .get_or_insert_with(Default::default)
+                .disable_dashboard = disable_dashboard;
         }
 
-        if let Some(addr) = &self.rpc_addr {
-            opts.grpc.addr = addr.clone()
+        if let Some(addr) = &self.grpc_addr {
+            if let Some(grpc_opts) = &mut opts.grpc_options {
+                grpc_opts.addr = addr.clone()
+            }
         }
 
         if let Some(addr) = &self.mysql_addr {
-            opts.mysql.enable = true;
-            opts.mysql.addr = addr.clone();
-            opts.mysql.tls = tls_opts.clone();
+            if let Some(mysql_opts) = &mut opts.mysql_options {
+                mysql_opts.addr = addr.clone();
+                mysql_opts.tls = tls_opts.clone();
+            }
         }
 
         if let Some(addr) = &self.postgres_addr {
-            opts.postgres.enable = true;
-            opts.postgres.addr = addr.clone();
-            opts.postgres.tls = tls_opts;
+            if let Some(postgres_opts) = &mut opts.postgres_options {
+                postgres_opts.addr = addr.clone();
+                postgres_opts.tls = tls_opts;
+            }
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            opts.opentsdb.enable = true;
-            opts.opentsdb.addr = addr.clone();
+            if let Some(opentsdb_addr) = &mut opts.opentsdb_options {
+                opentsdb_addr.addr = addr.clone();
+            }
         }
 
         if let Some(enable) = self.influxdb_enable {
-            opts.influxdb.enable = enable;
+            opts.influxdb_options = Some(InfluxdbOptions { enable });
         }
 
         if let Some(metasrv_addrs) = &self.metasrv_addr {
-            opts.meta_client
+            opts.meta_client_options
                 .get_or_insert_with(MetaClientOptions::default)
                 .metasrv_addrs = metasrv_addrs.clone();
             opts.mode = Mode::Distributed;
         }
 
-        opts.user_provider = self.user_provider.clone();
-
         Ok(Options::Frontend(Box::new(opts)))
     }
 
-    async fn build(self, mut opts: FrontendOptions) -> Result<Instance> {
-        let plugins = plugins::setup_frontend_plugins(&mut opts)
-            .await
-            .context(StartFrontendSnafu)?;
-
+    async fn build(self, opts: FrontendOptions) -> Result<Instance> {
         logging::info!("Frontend start command: {:#?}", self);
         logging::info!("Frontend options: {:#?}", opts);
 
+        let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
+
         let mut instance = FeInstance::try_new_distributed(&opts, plugins.clone())
             .await
-            .context(StartFrontendSnafu)?;
+            .context(error::StartFrontendSnafu)?;
 
         instance
             .build_servers(&opts)
             .await
-            .context(StartFrontendSnafu)?;
+            .context(error::StartFrontendSnafu)?;
 
         Ok(Instance { frontend: instance })
     }
 }
 
+pub fn load_frontend_plugins(user_provider: &Option<String>) -> Result<Plugins> {
+    let plugins = Plugins::new();
+
+    if let Some(provider) = user_provider {
+        let provider = auth::user_provider_from_option(provider).context(IllegalAuthConfigSnafu)?;
+        plugins.insert::<UserProviderRef>(provider);
+    }
+    Ok(plugins)
+}
+
 #[cfg(test)]
 mod tests {
     use std::io::Write;
@@ -217,7 +234,6 @@ mod tests {
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
     use frontend::service_config::GrpcOptions;
-    use servers::http::HttpOptions;
 
     use super::*;
     use crate::options::ENV_VAR_SEP;
@@ -239,29 +255,40 @@ mod tests {
             unreachable!()
         };
 
-        assert_eq!(opts.http.addr, "127.0.0.1:1234");
-        assert_eq!(ReadableSize::mb(64), opts.http.body_limit);
-        assert_eq!(opts.mysql.addr, "127.0.0.1:5678");
-        assert_eq!(opts.postgres.addr, "127.0.0.1:5432");
-        assert_eq!(opts.opentsdb.addr, "127.0.0.1:4321");
+        assert_eq!(opts.http_options.as_ref().unwrap().addr, "127.0.0.1:1234");
+        assert_eq!(
+            ReadableSize::mb(64),
+            opts.http_options.as_ref().unwrap().body_limit
+        );
+        assert_eq!(opts.mysql_options.as_ref().unwrap().addr, "127.0.0.1:5678");
+        assert_eq!(
+            opts.postgres_options.as_ref().unwrap().addr,
+            "127.0.0.1:5432"
+        );
+        assert_eq!(
+            opts.opentsdb_options.as_ref().unwrap().addr,
+            "127.0.0.1:4321"
+        );
 
         let default_opts = FrontendOptions::default();
-
-        assert_eq!(opts.grpc.addr, default_opts.grpc.addr);
-        assert!(opts.mysql.enable);
-        assert_eq!(opts.mysql.runtime_size, default_opts.mysql.runtime_size);
-        assert!(opts.postgres.enable);
         assert_eq!(
-            opts.postgres.runtime_size,
-            default_opts.postgres.runtime_size
+            opts.grpc_options.unwrap().addr,
+            default_opts.grpc_options.unwrap().addr
         );
-        assert!(opts.opentsdb.enable);
         assert_eq!(
-            opts.opentsdb.runtime_size,
-            default_opts.opentsdb.runtime_size
+            opts.mysql_options.as_ref().unwrap().runtime_size,
+            default_opts.mysql_options.as_ref().unwrap().runtime_size
+        );
+        assert_eq!(
+            opts.postgres_options.as_ref().unwrap().runtime_size,
+            default_opts.postgres_options.as_ref().unwrap().runtime_size
+        );
+        assert_eq!(
+            opts.opentsdb_options.as_ref().unwrap().runtime_size,
+            default_opts.opentsdb_options.as_ref().unwrap().runtime_size
         );
 
-        assert!(!opts.influxdb.enable);
+        assert!(!opts.influxdb_options.unwrap().enable);
     }
 
     #[test]
@@ -270,7 +297,7 @@ mod tests {
         let toml_str = r#"
             mode = "distributed"
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
             timeout = "30s"
             body_limit = "2GB"
@@ -292,10 +319,19 @@ mod tests {
             unreachable!()
         };
         assert_eq!(Mode::Distributed, fe_opts.mode);
-        assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
-        assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
+        assert_eq!(
+            "127.0.0.1:4000".to_string(),
+            fe_opts.http_options.as_ref().unwrap().addr
+        );
+        assert_eq!(
+            Duration::from_secs(30),
+            fe_opts.http_options.as_ref().unwrap().timeout
+        );
 
-        assert_eq!(ReadableSize::gb(2), fe_opts.http.body_limit);
+        assert_eq!(
+            ReadableSize::gb(2),
+            fe_opts.http_options.as_ref().unwrap().body_limit
+        );
 
         assert_eq!("debug", fe_opts.logging.level.as_ref().unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), fe_opts.logging.dir);
@@ -303,17 +339,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_try_from_start_command_to_anymap() {
-        let mut fe_opts = FrontendOptions {
-            http: HttpOptions {
-                disable_dashboard: false,
-                ..Default::default()
-            },
+        let command = StartCommand {
             user_provider: Some("static_user_provider:cmd:test=test".to_string()),
+            disable_dashboard: Some(false),
             ..Default::default()
         };
 
-        let plugins = plugins::setup_frontend_plugins(&mut fe_opts).await.unwrap();
-
+        let plugins = load_frontend_plugins(&command.user_provider);
+        let plugins = plugins.unwrap();
         let provider = plugins.get::<UserProviderRef>().unwrap();
         let result = provider
             .authenticate(
@@ -349,15 +382,15 @@ mod tests {
         let toml_str = r#"
             mode = "distributed"
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
 
-            [meta_client]
-            timeout = "3s"
-            connect_timeout = "5s"
+            [meta_client_options]
+            timeout_millis = 3000
+            connect_timeout_millis = 5000
             tcp_nodelay = true
 
-            [mysql]
+            [mysql_options]
             addr = "127.0.0.1:4002"
         "#;
         write!(file, "{}", toml_str).unwrap();
@@ -366,40 +399,40 @@ mod tests {
         temp_env::with_vars(
             [
                 (
-                    // mysql.addr = 127.0.0.1:14002
+                    // mysql_options.addr = 127.0.0.1:14002
                     [
                         env_prefix.to_string(),
-                        "mysql".to_uppercase(),
+                        "mysql_options".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("127.0.0.1:14002"),
                 ),
                 (
-                    // mysql.runtime_size = 11
+                    // mysql_options.runtime_size = 11
                     [
                         env_prefix.to_string(),
-                        "mysql".to_uppercase(),
+                        "mysql_options".to_uppercase(),
                         "runtime_size".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("11"),
                 ),
                 (
-                    // http.addr = 127.0.0.1:24000
+                    // http_options.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http".to_uppercase(),
+                        "http_options".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("127.0.0.1:24000"),
                 ),
                 (
-                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
                     [
                         env_prefix.to_string(),
-                        "meta_client".to_uppercase(),
+                        "meta_client_options".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -424,9 +457,9 @@ mod tests {
                 };
 
                 // Should be read from env, env > default values.
-                assert_eq!(fe_opts.mysql.runtime_size, 11);
+                assert_eq!(fe_opts.mysql_options.as_ref().unwrap().runtime_size, 11);
                 assert_eq!(
-                    fe_opts.meta_client.unwrap().metasrv_addrs,
+                    fe_opts.meta_client_options.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),
@@ -435,13 +468,22 @@ mod tests {
                 );
 
                 // Should be read from config file, config file > env > default values.
-                assert_eq!(fe_opts.mysql.addr, "127.0.0.1:4002");
+                assert_eq!(
+                    fe_opts.mysql_options.as_ref().unwrap().addr,
+                    "127.0.0.1:4002"
+                );
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(fe_opts.http.addr, "127.0.0.1:14000");
+                assert_eq!(
+                    fe_opts.http_options.as_ref().unwrap().addr,
+                    "127.0.0.1:14000"
+                );
 
                 // Should be default value.
-                assert_eq!(fe_opts.grpc.addr, GrpcOptions::default().addr);
+                assert_eq!(
+                    fe_opts.grpc_options.as_ref().unwrap().addr,
+                    GrpcOptions::default().addr
+                );
             },
         );
     }

src/cmd/src/metasrv.rs

@@ -20,7 +20,7 @@ use meta_srv::bootstrap::MetaSrvInstance;
 use meta_srv::metasrv::MetaSrvOptions;
 use snafu::ResultExt;
 
-use crate::error::{self, Result, StartMetaServerSnafu};
+use crate::error::{self, Result};
 use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
@@ -29,10 +29,10 @@ pub struct Instance {
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        plugins::start_meta_srv_plugins(self.instance.plugins())
+        self.instance
+            .start()
             .await
-            .context(StartMetaServerSnafu)?;
-        self.instance.start().await.context(StartMetaServerSnafu)
+            .context(error::StartMetaServerSnafu)
     }
 
     pub async fn stop(&self) -> Result<()> {
@@ -93,7 +93,7 @@ struct StartCommand {
     #[clap(long)]
     use_memory_store: Option<bool>,
     #[clap(long)]
-    enable_region_failover: Option<bool>,
+    disable_region_failover: Option<bool>,
     #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
@@ -140,33 +140,30 @@ impl StartCommand {
             opts.use_memory_store = use_memory_store;
         }
 
-        if let Some(enable_region_failover) = self.enable_region_failover {
-            opts.enable_region_failover = enable_region_failover;
+        if let Some(disable_region_failover) = self.disable_region_failover {
+            opts.disable_region_failover = disable_region_failover;
         }
 
         if let Some(http_addr) = &self.http_addr {
-            opts.http.addr = http_addr.clone();
+            opts.http_opts.addr = http_addr.clone();
         }
 
         if let Some(http_timeout) = self.http_timeout {
-            opts.http.timeout = Duration::from_secs(http_timeout);
+            opts.http_opts.timeout = Duration::from_secs(http_timeout);
         }
 
         // Disable dashboard in metasrv.
-        opts.http.disable_dashboard = true;
+        opts.http_opts.disable_dashboard = true;
 
         Ok(Options::Metasrv(Box::new(opts)))
     }
 
-    async fn build(self, mut opts: MetaSrvOptions) -> Result<Instance> {
-        let plugins = plugins::setup_meta_srv_plugins(&mut opts)
-            .await
-            .context(StartMetaServerSnafu)?;
-
+    async fn build(self, opts: MetaSrvOptions) -> Result<Instance> {
         logging::info!("MetaSrv start command: {:#?}", self);
+
         logging::info!("MetaSrv options: {:#?}", opts);
 
-        let instance = MetaSrvInstance::new(opts, plugins)
+        let instance = MetaSrvInstance::new(opts)
             .await
             .context(error::BuildMetaServerSnafu)?;
 
@@ -210,6 +207,7 @@ mod tests {
             bind_addr = "127.0.0.1:3002"
             server_addr = "127.0.0.1:3002"
             store_addr = "127.0.0.1:2379"
+            datanode_lease_secs = 15
             selector = "LeaseBased"
             use_memory_store = false
 
@@ -231,6 +229,7 @@ mod tests {
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
         assert_eq!("127.0.0.1:2379".to_string(), options.store_addr);
+        assert_eq!(15, options.datanode_lease_secs);
         assert_eq!(SelectorType::LeaseBased, options.selector);
         assert_eq!("debug", options.logging.level.as_ref().unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), options.logging.dir);
@@ -267,7 +266,7 @@ mod tests {
             selector = "LeaseBased"
             use_memory_store = false
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
 
             [logging]
@@ -290,10 +289,10 @@ mod tests {
                     Some("127.0.0.1:13002"),
                 ),
                 (
-                    // http.addr = 127.0.0.1:24000
+                    // http_options.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http".to_uppercase(),
+                        "http_options".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -321,7 +320,7 @@ mod tests {
                 assert_eq!(opts.server_addr, "127.0.0.1:3002");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.http.addr, "127.0.0.1:14000");
+                assert_eq!(opts.http_opts.addr, "127.0.0.1:14000");
 
                 // Should be default value.
                 assert_eq!(opts.store_addr, "127.0.0.1:2379");

src/cmd/src/options.rs

@@ -12,27 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_config::KvStoreConfig;
 use common_telemetry::logging::LoggingOptions;
 use config::{Config, Environment, File, FileFormat};
-use datanode::config::{DatanodeOptions, ProcedureConfig};
+use datanode::datanode::DatanodeOptions;
 use frontend::frontend::FrontendOptions;
 use meta_srv::metasrv::MetaSrvOptions;
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 
-use crate::error::{LoadLayeredConfigSnafu, Result, SerdeJsonSnafu};
+use crate::error::{LoadLayeredConfigSnafu, Result};
 
 pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
 
-/// Options mixed up from datanode, frontend and metasrv.
 pub struct MixOptions {
-    pub data_home: String,
-    pub procedure: ProcedureConfig,
-    pub metadata_store: KvStoreConfig,
-    pub frontend: FrontendOptions,
-    pub datanode: DatanodeOptions,
+    pub fe_opts: FrontendOptions,
+    pub dn_opts: DatanodeOptions,
     pub logging: LoggingOptions,
 }
 
@@ -94,16 +89,9 @@ impl Options {
                 .ignore_empty(true)
         };
 
-        // Workaround: Replacement for `Config::try_from(&default_opts)` due to
-        // `ConfigSerializer` cannot handle the case of an empty struct contained
-        // within an iterative structure.
-        // See: https://github.com/mehcode/config-rs/issues/461
-        let json_str = serde_json::to_string(&default_opts).context(SerdeJsonSnafu)?;
-        let default_config = File::from_str(&json_str, FileFormat::Json);
-
         // Add default values and environment variables as the sources of the configuration.
         let mut layered_config = Config::builder()
-            .add_source(default_config)
+            .add_source(Config::try_from(&default_opts).context(LoadLayeredConfigSnafu)?)
             .add_source(env_source);
 
         // Add config file as the source of the configuration if it is specified.
@@ -127,7 +115,7 @@ mod tests {
     use std::time::Duration;
 
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::config::{DatanodeOptions, ObjectStoreConfig};
+    use datanode::datanode::{DatanodeOptions, ObjectStoreConfig};
 
     use super::*;
 
@@ -143,9 +131,9 @@ mod tests {
             mysql_addr = "127.0.0.1:4406"
             mysql_runtime_size = 2
 
-            [meta_client]
-            timeout = "3s"
-            connect_timeout = "5s"
+            [meta_client_options]
+            timeout_millis = 3000
+            connect_timeout_millis = 5000
             tcp_nodelay = true
 
             [wal]
@@ -224,10 +212,10 @@ mod tests {
                     Some("/other/wal/dir"),
                 ),
                 (
-                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
                     [
                         env_prefix.to_string(),
-                        "meta_client".to_uppercase(),
+                        "meta_client_options".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -255,7 +243,7 @@ mod tests {
                     Some(Duration::from_secs(42))
                 );
                 assert_eq!(
-                    opts.meta_client.unwrap().metasrv_addrs,
+                    opts.meta_client_options.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),

src/cmd/src/standalone.rs

@@ -13,28 +13,18 @@
 // limitations under the License.
 
 use std::sync::Arc;
-use std::{fs, path};
 
-use catalog::kvbackend::KvBackendCatalogManager;
-use catalog::CatalogManagerRef;
 use clap::Parser;
 use common_base::Plugins;
-use common_config::{metadata_store_dir, KvStoreConfig, WalConfig};
-use common_meta::cache_invalidator::DummyKvCacheInvalidator;
-use common_meta::kv_backend::KvBackendRef;
-use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
-use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
-use datanode::datanode::{Datanode, DatanodeBuilder};
-use datanode::region_server::RegionServer;
-use file_engine::config::EngineConfig as FileEngineConfig;
+use datanode::datanode::{Datanode, DatanodeOptions, ProcedureConfig, StorageConfig, WalConfig};
+use datanode::instance::InstanceRef;
 use frontend::frontend::FrontendOptions;
-use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
+use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
-use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -42,10 +32,10 @@ use servers::Mode;
 use snafu::ResultExt;
 
 use crate::error::{
-    CreateDirSnafu, IllegalConfigSnafu, InitMetadataSnafu, Result, ShutdownDatanodeSnafu,
-    ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
-    StopProcedureManagerSnafu,
+    IllegalConfigSnafu, Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu,
+    StartFrontendSnafu,
 };
+use crate::frontend::load_frontend_plugins;
 use crate::options::{MixOptions, Options, TopLevelOptions};
 
 #[derive(Parser)]
@@ -55,8 +45,12 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(self, opts: MixOptions) -> Result<Instance> {
-        self.subcmd.build(opts).await
+    pub async fn build(
+        self,
+        fe_opts: FrontendOptions,
+        dn_opts: DatanodeOptions,
+    ) -> Result<Instance> {
+        self.subcmd.build(fe_opts, dn_opts).await
     }
 
     pub fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
@@ -70,9 +64,9 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, opts: MixOptions) -> Result<Instance> {
+    async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
         match self {
-            SubCommand::Start(cmd) => cmd.build(opts).await,
+            SubCommand::Start(cmd) => cmd.build(fe_opts, dn_opts).await,
         }
     }
 
@@ -87,46 +81,38 @@ impl SubCommand {
 #[serde(default)]
 pub struct StandaloneOptions {
     pub mode: Mode,
+    pub enable_memory_catalog: bool,
     pub enable_telemetry: bool,
-    pub http: HttpOptions,
-    pub grpc: GrpcOptions,
-    pub mysql: MysqlOptions,
-    pub postgres: PostgresOptions,
-    pub opentsdb: OpentsdbOptions,
-    pub influxdb: InfluxdbOptions,
-    pub prom_store: PromStoreOptions,
+    pub http_options: Option<HttpOptions>,
+    pub grpc_options: Option<GrpcOptions>,
+    pub mysql_options: Option<MysqlOptions>,
+    pub postgres_options: Option<PostgresOptions>,
+    pub opentsdb_options: Option<OpentsdbOptions>,
+    pub influxdb_options: Option<InfluxdbOptions>,
+    pub prom_store_options: Option<PromStoreOptions>,
     pub wal: WalConfig,
     pub storage: StorageConfig,
-    pub metadata_store: KvStoreConfig,
     pub procedure: ProcedureConfig,
     pub logging: LoggingOptions,
-    pub user_provider: Option<String>,
-    /// Options for different store engines.
-    pub region_engine: Vec<RegionEngineConfig>,
 }
 
 impl Default for StandaloneOptions {
     fn default() -> Self {
         Self {
             mode: Mode::Standalone,
+            enable_memory_catalog: false,
             enable_telemetry: true,
-            http: HttpOptions::default(),
-            grpc: GrpcOptions::default(),
-            mysql: MysqlOptions::default(),
-            postgres: PostgresOptions::default(),
-            opentsdb: OpentsdbOptions::default(),
-            influxdb: InfluxdbOptions::default(),
-            prom_store: PromStoreOptions::default(),
+            http_options: Some(HttpOptions::default()),
+            grpc_options: Some(GrpcOptions::default()),
+            mysql_options: Some(MysqlOptions::default()),
+            postgres_options: Some(PostgresOptions::default()),
+            opentsdb_options: Some(OpentsdbOptions::default()),
+            influxdb_options: Some(InfluxdbOptions::default()),
+            prom_store_options: Some(PromStoreOptions::default()),
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
-            metadata_store: KvStoreConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
-            user_provider: None,
-            region_engine: vec![
-                RegionEngineConfig::Mito(MitoConfig::default()),
-                RegionEngineConfig::File(FileEngineConfig::default()),
-            ],
         }
     }
 }
@@ -135,27 +121,26 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
-            http: self.http,
-            grpc: self.grpc,
-            mysql: self.mysql,
-            postgres: self.postgres,
-            opentsdb: self.opentsdb,
-            influxdb: self.influxdb,
-            prom_store: self.prom_store,
-            meta_client: None,
+            http_options: self.http_options,
+            grpc_options: self.grpc_options,
+            mysql_options: self.mysql_options,
+            postgres_options: self.postgres_options,
+            opentsdb_options: self.opentsdb_options,
+            influxdb_options: self.influxdb_options,
+            prom_store_options: self.prom_store_options,
+            meta_client_options: None,
             logging: self.logging,
-            user_provider: self.user_provider,
             ..Default::default()
         }
     }
 
     fn datanode_options(self) -> DatanodeOptions {
         DatanodeOptions {
-            node_id: Some(0),
+            enable_memory_catalog: self.enable_memory_catalog,
             enable_telemetry: self.enable_telemetry,
             wal: self.wal,
             storage: self.storage,
-            region_engine: self.region_engine,
+            procedure: self.procedure,
             ..Default::default()
         }
     }
@@ -164,19 +149,16 @@ impl StandaloneOptions {
 pub struct Instance {
     datanode: Datanode,
     frontend: FeInstance,
-    procedure_manager: ProcedureManagerRef,
 }
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
         // Start datanode instance before starting services, to avoid requests come in before internal components are started.
-        self.datanode.start().await.context(StartDatanodeSnafu)?;
-        info!("Datanode instance started");
-
-        self.procedure_manager
-            .start()
+        self.datanode
+            .start_instance()
             .await
-            .context(StartProcedureManagerSnafu)?;
+            .context(StartDatanodeSnafu)?;
+        info!("Datanode instance started");
 
         self.frontend.start().await.context(StartFrontendSnafu)?;
         Ok(())
@@ -188,13 +170,8 @@ impl Instance {
             .await
             .context(ShutdownFrontendSnafu)?;
 
-        self.procedure_manager
-            .stop()
-            .await
-            .context(StopProcedureManagerSnafu)?;
-
         self.datanode
-            .shutdown()
+            .shutdown_instance()
             .await
             .context(ShutdownDatanodeSnafu)?;
         info!("Datanode instance stopped.");
@@ -219,6 +196,8 @@ struct StartCommand {
     influxdb_enable: bool,
     #[clap(short, long)]
     config_file: Option<String>,
+    #[clap(short = 'm', long = "memory-catalog")]
+    enable_memory_catalog: bool,
     #[clap(long)]
     tls_mode: Option<TlsMode>,
     #[clap(long)]
@@ -239,6 +218,8 @@ impl StartCommand {
             None,
         )?;
 
+        opts.enable_memory_catalog = self.enable_memory_catalog;
+
         opts.mode = Mode::Standalone;
 
         if let Some(dir) = top_level_options.log_dir {
@@ -256,7 +237,9 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            opts.http.addr = addr.clone()
+            if let Some(http_opts) = &mut opts.http_options {
+                http_opts.addr = addr.clone()
+            }
         }
 
         if let Some(addr) = &self.rpc_addr {
@@ -270,58 +253,48 @@ impl StartCommand {
                 }
                 .fail();
             }
-            opts.grpc.addr = addr.clone()
+            if let Some(grpc_opts) = &mut opts.grpc_options {
+                grpc_opts.addr = addr.clone()
+            }
         }
 
         if let Some(addr) = &self.mysql_addr {
-            opts.mysql.enable = true;
-            opts.mysql.addr = addr.clone();
-            opts.mysql.tls = tls_opts.clone();
+            if let Some(mysql_opts) = &mut opts.mysql_options {
+                mysql_opts.addr = addr.clone();
+                mysql_opts.tls = tls_opts.clone();
+            }
         }
 
         if let Some(addr) = &self.postgres_addr {
-            opts.postgres.enable = true;
-            opts.postgres.addr = addr.clone();
-            opts.postgres.tls = tls_opts;
+            if let Some(postgres_opts) = &mut opts.postgres_options {
+                postgres_opts.addr = addr.clone();
+                postgres_opts.tls = tls_opts;
+            }
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            opts.opentsdb.enable = true;
-            opts.opentsdb.addr = addr.clone();
+            if let Some(opentsdb_addr) = &mut opts.opentsdb_options {
+                opentsdb_addr.addr = addr.clone();
+            }
         }
 
         if self.influxdb_enable {
-            opts.influxdb.enable = self.influxdb_enable;
+            opts.influxdb_options = Some(InfluxdbOptions { enable: true });
         }
 
-        opts.user_provider = self.user_provider.clone();
-
-        let metadata_store = opts.metadata_store.clone();
-        let procedure = opts.procedure.clone();
-        let frontend = opts.clone().frontend_options();
+        let fe_opts = opts.clone().frontend_options();
         let logging = opts.logging.clone();
-        let datanode = opts.datanode_options();
+        let dn_opts = opts.datanode_options();
 
         Ok(Options::Standalone(Box::new(MixOptions {
-            procedure,
-            metadata_store,
-            data_home: datanode.storage.data_home.to_string(),
-            frontend,
-            datanode,
+            fe_opts,
+            dn_opts,
             logging,
         })))
     }
 
-    #[allow(unreachable_code)]
-    #[allow(unused_variables)]
-    #[allow(clippy::diverging_sub_expression)]
-    async fn build(self, opts: MixOptions) -> Result<Instance> {
-        let mut fe_opts = opts.frontend;
-        let fe_plugins = plugins::setup_frontend_plugins(&mut fe_opts)
-            .await
-            .context(StartFrontendSnafu)?;
-
-        let dn_opts = opts.datanode;
+    async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
+        let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
 
         info!("Standalone start command: {:#?}", self);
         info!(
@@ -329,79 +302,30 @@ impl StartCommand {
             fe_opts, dn_opts
         );
 
-        // Ensure the data_home directory exists.
-        fs::create_dir_all(path::Path::new(&opts.data_home)).context(CreateDirSnafu {
-            dir: &opts.data_home,
-        })?;
-
-        let metadata_dir = metadata_store_dir(&opts.data_home);
-        let (kv_store, procedure_manager) = FeInstance::try_build_standalone_components(
-            metadata_dir,
-            opts.metadata_store,
-            opts.procedure,
-        )
-        .await
-        .context(StartFrontendSnafu)?;
-
-        let datanode =
-            DatanodeBuilder::new(dn_opts.clone(), Some(kv_store.clone()), Default::default())
-                .build()
-                .await
-                .context(StartDatanodeSnafu)?;
-        let region_server = datanode.region_server();
-
-        let catalog_manager = KvBackendCatalogManager::new(
-            kv_store.clone(),
-            Arc::new(DummyKvCacheInvalidator),
-            Arc::new(StandaloneDatanodeManager(region_server.clone())),
-        );
-
-        catalog_manager
-            .table_metadata_manager_ref()
-            .init()
+        let datanode = Datanode::new(dn_opts.clone(), Default::default())
             .await
-            .context(InitMetadataSnafu)?;
+            .context(StartDatanodeSnafu)?;
 
-        // TODO: build frontend instance like in distributed mode
-        let mut frontend = build_frontend(
-            fe_plugins,
-            kv_store,
-            procedure_manager.clone(),
-            catalog_manager,
-            region_server,
-        )
-        .await?;
+        let mut frontend = build_frontend(plugins.clone(), datanode.get_instance()).await?;
 
         frontend
             .build_servers(&fe_opts)
             .await
             .context(StartFrontendSnafu)?;
 
-        Ok(Instance {
-            datanode,
-            frontend,
-            procedure_manager,
-        })
+        Ok(Instance { datanode, frontend })
     }
 }
 
 /// Build frontend instance in standalone mode
 async fn build_frontend(
-    plugins: Plugins,
-    kv_store: KvBackendRef,
-    procedure_manager: ProcedureManagerRef,
-    catalog_manager: CatalogManagerRef,
-    region_server: RegionServer,
+    plugins: Arc<Plugins>,
+    datanode_instance: InstanceRef,
 ) -> Result<FeInstance> {
-    let frontend_instance = FeInstance::try_new_standalone(
-        kv_store,
-        procedure_manager,
-        catalog_manager,
-        plugins,
-        region_server,
-    )
-    .await
-    .context(StartFrontendSnafu)?;
+    let mut frontend_instance = FeInstance::try_new_standalone(datanode_instance.clone())
+        .await
+        .context(StartFrontendSnafu)?;
+    frontend_instance.set_plugins(plugins.clone());
     Ok(frontend_instance)
 }
 
@@ -421,13 +345,13 @@ mod tests {
 
     #[tokio::test]
     async fn test_try_from_start_command_to_anymap() {
-        let mut fe_opts = FrontendOptions {
+        let command = StartCommand {
             user_provider: Some("static_user_provider:cmd:test=test".to_string()),
             ..Default::default()
         };
 
-        let plugins = plugins::setup_frontend_plugins(&mut fe_opts).await.unwrap();
-
+        let plugins = load_frontend_plugins(&command.user_provider);
+        let plugins = plugins.unwrap();
         let provider = plugins.get::<UserProviderRef>().unwrap();
         let result = provider
             .authenticate(
@@ -475,9 +399,9 @@ mod tests {
             checkpoint_margin = 9
             gc_duration = '7s'
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
-            timeout = "33s"
+            timeout = "30s"
             body_limit = "128MB"
 
             [logging]
@@ -495,24 +419,40 @@ mod tests {
         else {
             unreachable!()
         };
-        let fe_opts = options.frontend;
-        let dn_opts = options.datanode;
+        let fe_opts = options.fe_opts;
+        let dn_opts = options.dn_opts;
         let logging_opts = options.logging;
         assert_eq!(Mode::Standalone, fe_opts.mode);
-        assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
-        assert_eq!(Duration::from_secs(33), fe_opts.http.timeout);
-        assert_eq!(ReadableSize::mb(128), fe_opts.http.body_limit);
-        assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.addr);
-        assert!(fe_opts.mysql.enable);
-        assert_eq!("127.0.0.1:4002", fe_opts.mysql.addr);
-        assert_eq!(2, fe_opts.mysql.runtime_size);
-        assert_eq!(None, fe_opts.mysql.reject_no_database);
-        assert!(fe_opts.influxdb.enable);
+        assert_eq!(
+            "127.0.0.1:4000".to_string(),
+            fe_opts.http_options.as_ref().unwrap().addr
+        );
+        assert_eq!(
+            Duration::from_secs(30),
+            fe_opts.http_options.as_ref().unwrap().timeout
+        );
+        assert_eq!(
+            ReadableSize::mb(128),
+            fe_opts.http_options.as_ref().unwrap().body_limit
+        );
+        assert_eq!(
+            "127.0.0.1:4001".to_string(),
+            fe_opts.grpc_options.unwrap().addr
+        );
+        assert_eq!(
+            "127.0.0.1:4002",
+            fe_opts.mysql_options.as_ref().unwrap().addr
+        );
+        assert_eq!(2, fe_opts.mysql_options.as_ref().unwrap().runtime_size);
+        assert_eq!(
+            None,
+            fe_opts.mysql_options.as_ref().unwrap().reject_no_database
+        );
+        assert!(fe_opts.influxdb_options.as_ref().unwrap().enable);
 
         assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap());
-
         match &dn_opts.storage.store {
-            datanode::config::ObjectStoreConfig::S3(s3_config) => {
+            datanode::datanode::ObjectStoreConfig::S3(s3_config) => {
                 assert_eq!(
                     "Secret([REDACTED alloc::string::String])".to_string(),
                     format!("{:?}", s3_config.access_key_id)
@@ -554,7 +494,7 @@ mod tests {
         let toml_str = r#"
             mode = "standalone"
 
-            [http]
+            [http_options]
             addr = "127.0.0.1:4000"
 
             [logging]
@@ -586,10 +526,10 @@ mod tests {
                     Some("info"),
                 ),
                 (
-                    // http.addr = 127.0.0.1:24000
+                    // http_options.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http".to_uppercase(),
+                        "http_options".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -620,33 +560,21 @@ mod tests {
                 assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.frontend.http.addr, "127.0.0.1:14000");
-                assert_eq!(ReadableSize::mb(64), opts.frontend.http.body_limit);
+                assert_eq!(
+                    opts.fe_opts.http_options.as_ref().unwrap().addr,
+                    "127.0.0.1:14000"
+                );
+                assert_eq!(
+                    ReadableSize::mb(64),
+                    opts.fe_opts.http_options.as_ref().unwrap().body_limit
+                );
 
                 // Should be default value.
-                assert_eq!(opts.frontend.grpc.addr, GrpcOptions::default().addr);
+                assert_eq!(
+                    opts.fe_opts.grpc_options.unwrap().addr,
+                    GrpcOptions::default().addr
+                );
             },
         );
     }
-
-    #[test]
-    fn test_load_default_standalone_options() {
-        let options: StandaloneOptions =
-            Options::load_layered_options(None, "GREPTIMEDB_FRONTEND", None).unwrap();
-        let default_options = StandaloneOptions::default();
-        assert_eq!(options.mode, default_options.mode);
-        assert_eq!(options.enable_telemetry, default_options.enable_telemetry);
-        assert_eq!(options.http, default_options.http);
-        assert_eq!(options.grpc, default_options.grpc);
-        assert_eq!(options.mysql, default_options.mysql);
-        assert_eq!(options.postgres, default_options.postgres);
-        assert_eq!(options.opentsdb, default_options.opentsdb);
-        assert_eq!(options.influxdb, default_options.influxdb);
-        assert_eq!(options.prom_store, default_options.prom_store);
-        assert_eq!(options.wal, default_options.wal);
-        assert_eq!(options.metadata_store, default_options.metadata_store);
-        assert_eq!(options.procedure, default_options.procedure);
-        assert_eq!(options.logging, default_options.logging);
-        assert_eq!(options.region_engine, default_options.region_engine);
-    }
 }

src/common/base/Cargo.toml

@@ -9,7 +9,6 @@ anymap = "1.0.0-beta.2"
 bitvec = "1.0"
 bytes = { version = "1.1", features = ["serde"] }
 common-error = { workspace = true }
-common-macro = { workspace = true }
 paste = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 snafu.workspace = true

src/common/base/src/buffer.rs

@@ -17,13 +17,11 @@ use std::io::{Read, Write};
 
 use bytes::{Buf, BufMut, BytesMut};
 use common_error::ext::ErrorExt;
-use common_macro::stack_trace_debug;
 use paste::paste;
 use snafu::{ensure, Location, ResultExt, Snafu};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
     #[snafu(display(
         "Destination buffer overflow, src_len: {}, dst_len: {}",
@@ -39,10 +37,9 @@ pub enum Error {
     #[snafu(display("Buffer underflow"))]
     Underflow { location: Location },
 
-    #[snafu(display("IO operation reach EOF"))]
+    #[snafu(display("IO operation reach EOF, source: {}", source))]
     Eof {
-        #[snafu(source)]
-        error: std::io::Error,
+        source: std::io::Error,
         location: Location,
     },
 }

src/common/base/src/lib.rs

@@ -15,6 +15,7 @@
 pub mod bit_vec;
 pub mod buffer;
 pub mod bytes;
+pub mod paths;
 #[allow(clippy::all)]
 pub mod readable_size;
 
@@ -23,8 +24,6 @@ use std::sync::{Arc, Mutex, MutexGuard};
 
 pub use bit_vec::BitVec;
 
-/// [`Plugins`] is a wrapper of Arc contents.
-/// Make it Cloneable and we can treat it like an Arc struct.
 #[derive(Default, Clone)]
 pub struct Plugins {
     inner: Arc<Mutex<anymap::Map<dyn Any + Send + Sync>>>,

src/common/base/src/paths.rs

@@ -12,14 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_base::Plugins;
-use datanode::config::DatanodeOptions;
-use datanode::error::Result;
+//! Path constants for table engines, cluster states and WAL
+/// All paths relative to data_home(file storage) or root path(S3, OSS etc).
 
-pub async fn setup_datanode_plugins(_opts: &mut DatanodeOptions) -> Result<Plugins> {
-    Ok(Plugins::new())
-}
+/// WAL dir for local file storage
+pub const WAL_DIR: &str = "wal/";
 
-pub async fn start_datanode_plugins(_plugins: Plugins) -> Result<()> {
-    Ok(())
-}
+/// Data dir for table engines
+pub const DATA_DIR: &str = "data/";
+
+/// Cluster state dir
+pub const CLUSTER_DIR: &str = "cluster/";

src/common/base/src/readable_size.rs

@@ -14,9 +14,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This file is copied from https://github.com/tikv/raft-engine/blob/8dd2a39f359ff16f5295f35343f626e0c10132fa/src/util.rs
+// This file is copied from https://github.com/tikv/raft-engine/blob/8dd2a39f359ff16f5295f35343f626e0c10132fa/src/util.rs without any modification.
 
-use std::fmt::{self, Debug, Display, Write};
+use std::fmt;
+use std::fmt::{Display, Write};
 use std::ops::{Div, Mul};
 use std::str::FromStr;
 
@@ -33,7 +34,7 @@ pub const GIB: u64 = MIB * BINARY_DATA_MAGNITUDE;
 pub const TIB: u64 = GIB * BINARY_DATA_MAGNITUDE;
 pub const PIB: u64 = TIB * BINARY_DATA_MAGNITUDE;
 
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd)]
+#[derive(Clone, Debug, Copy, PartialEq, Eq, PartialOrd)]
 pub struct ReadableSize(pub u64);
 
 impl ReadableSize {
@@ -154,12 +155,6 @@ impl FromStr for ReadableSize {
     }
 }
 
-impl Debug for ReadableSize {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self)
-    }
-}
-
 impl Display for ReadableSize {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         if self.0 >= PIB {

src/common/catalog/Cargo.toml

@@ -6,7 +6,6 @@ license.workspace = true
 
 [dependencies]
 common-error = { workspace = true }
-common-macro = { workspace = true }
 serde.workspace = true
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }

src/common/catalog/src/consts.rs

@@ -41,7 +41,7 @@ pub fn default_engine() -> &'static str {
     MITO_ENGINE
 }
 
-pub const FILE_ENGINE: &str = "file";
+pub const IMMUTABLE_FILE_ENGINE: &str = "file";
 
 pub const SEMANTIC_TYPE_PRIMARY_KEY: &str = "TAG";
 pub const SEMANTIC_TYPE_FIELD: &str = "FIELD";

src/common/catalog/src/error.rs

@@ -16,24 +16,34 @@ use std::any::Any;
 
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
-use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 
-#[derive(Snafu)]
+#[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
-#[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Invalid full table name: {}", table_name))]
-    InvalidFullTableName {
-        table_name: String,
+    #[snafu(display("Invalid catalog info: {}", key))]
+    InvalidCatalog { key: String, location: Location },
+
+    #[snafu(display("Failed to deserialize catalog entry value: {}", raw))]
+    DeserializeCatalogEntryValue {
+        raw: String,
         location: Location,
+        source: serde_json::error::Error,
+    },
+
+    #[snafu(display("Failed to serialize catalog entry value"))]
+    SerializeCatalogEntryValue {
+        location: Location,
+        source: serde_json::error::Error,
     },
 }
 
 impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
-            Error::InvalidFullTableName { .. } => StatusCode::Unexpected,
+            Error::InvalidCatalog { .. }
+            | Error::DeserializeCatalogEntryValue { .. }
+            | Error::SerializeCatalogEntryValue { .. } => StatusCode::Unexpected,
         }
     }
 

src/common/config/Cargo.toml

@@ -1,10 +0,0 @@
-[package]
-name = "common-config"
-version.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-common-base.workspace = true
-humantime-serde.workspace = true
-serde.workspace = true

src/common/config/src/lib.rs

@@ -1,73 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::Duration;
-
-use common_base::readable_size::ReadableSize;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(default)]
-pub struct WalConfig {
-    // wal directory
-    pub dir: Option<String>,
-    // wal file size in bytes
-    pub file_size: ReadableSize,
-    // wal purge threshold in bytes
-    pub purge_threshold: ReadableSize,
-    // purge interval in seconds
-    #[serde(with = "humantime_serde")]
-    pub purge_interval: Duration,
-    // read batch size
-    pub read_batch_size: usize,
-    // whether to sync log file after every write
-    pub sync_write: bool,
-}
-
-impl Default for WalConfig {
-    fn default() -> Self {
-        Self {
-            dir: None,
-            file_size: ReadableSize::mb(256), // log file size 256MB
-            purge_threshold: ReadableSize::gb(4), // purge threshold 4GB
-            purge_interval: Duration::from_secs(600),
-            read_batch_size: 128,
-            sync_write: false,
-        }
-    }
-}
-
-pub fn metadata_store_dir(store_dir: &str) -> String {
-    format!("{store_dir}/metadata")
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(default)]
-pub struct KvStoreConfig {
-    // Kv file size in bytes
-    pub file_size: ReadableSize,
-    // Kv purge threshold in bytes
-    pub purge_threshold: ReadableSize,
-}
-
-impl Default for KvStoreConfig {
-    fn default() -> Self {
-        Self {
-            // log file size 256MB
-            file_size: ReadableSize::mb(256),
-            // purge threshold 4GB
-            purge_threshold: ReadableSize::gb(4),
-        }
-    }
-}

src/common/datasource/Cargo.toml

@@ -18,17 +18,14 @@ async-compression = { version = "0.3", features = [
 async-trait.workspace = true
 bytes = "1.1"
 common-error = { workspace = true }
-common-macro = { workspace = true }
 common-runtime = { workspace = true }
 datafusion.workspace = true
 derive_builder.workspace = true
 futures.workspace = true
-lazy_static.workspace = true
 object-store = { workspace = true }
 orc-rust = "0.2"
 paste = "1.0"
 regex = "1.7"
-serde.workspace = true
 snafu.workspace = true
 strum.workspace = true
 tokio-util.workspace = true