chore: fix typo (#2477)

fix: normalize region dir in RegionOpener

.cargo/config.toml

@@ -12,5 +12,9 @@ rustflags = [
     "-Wclippy::print_stdout",
     "-Wclippy::print_stderr",
     "-Wclippy::implicit_clone",
-    "-Aclippy::items_after_test_module",
+
+    # It seems clippy has made a false positive decision here when upgrading rust toolchain to
+    # nightly-2023-08-07, we do need it to be borrowed mutably.
+    # Allow it for now; try disallow it when the toolchain is upgraded in the future.
+    "-Aclippy::needless_pass_by_ref_mut",
 ]

.config/nextest.toml

@@ -1,2 +1,3 @@
 [profile.default]
 slow-timeout = { period = "60s", terminate-after = 3, grace-period = "30s" }
+retries = { backoff = "exponential", count = 3, delay = "10s", jitter = true }

.github/actions/build-dev-builder-image/action.yml

@@ -8,8 +8,8 @@ inputs:
   dockerhub-image-registry-username:
     description: The dockerhub username to login to the image registry
     required: true
-  dockerhub-image-registry-password:
-    description: The dockerhub password to login to the image registry
+  dockerhub-image-registry-token:
+    description: The dockerhub token to login to the image registry
     required: true
   dockerhub-image-namespace:
     description: The dockerhub namespace of the image registry to store the images
@@ -40,12 +40,23 @@ runs:
       with:
         registry: ${{ inputs.dockerhub-image-registry }}
         username: ${{ inputs.dockerhub-image-registry-username }}
-        password: ${{ inputs.dockerhub-image-registry-password }}
+        password: ${{ inputs.dockerhub-image-registry-token }}
 
-    - name: Build and push dev builder image to dockerhub
+    - name: Build and push ubuntu dev builder image to dockerhub
       shell: bash
       run:
         make dev-builder \
+        BASE_IMAGE=ubuntu \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push centos dev builder image to dockerhub
+      shell: bash
+      run:
+        make dev-builder \
+        BASE_IMAGE=centos \
         BUILDX_MULTI_PLATFORM_BUILD=true \
         IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
         IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
@@ -59,11 +70,23 @@ runs:
         username: ${{ inputs.acr-image-registry-username }}
         password: ${{ inputs.acr-image-registry-password }}
 
-    - name: Build and push dev builder image to ACR
+    - name: Build and push ubuntu dev builder image to ACR
       shell: bash
       continue-on-error: true
       run: # buildx will cache the images that already built, so it will not take long time to build the images again.
         make dev-builder \
+        BASE_IMAGE=ubuntu \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push centos dev builder image to ACR
+      shell: bash
+      continue-on-error: true
+      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
+        make dev-builder \
+        BASE_IMAGE=centos \
         BUILDX_MULTI_PLATFORM_BUILD=true \
         IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
         IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \

.github/actions/build-greptime-binary/action.yml

@@ -28,13 +28,26 @@ inputs:
   aws-region:
     description: AWS region
     required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
+  working-dir:
+    description: Working directory to build the artifacts
+    required: false
+    default: .
 runs:
   using: composite
   steps:
     - name: Build greptime binary
       shell: bash
       run: |
-        make build-greptime-by-buildx \
+        cd ${{ inputs.working-dir }} && \
+        make build-by-dev-builder \
           CARGO_PROFILE=${{ inputs.cargo-profile }} \
           FEATURES=${{ inputs.features }} \
           BASE_IMAGE=${{ inputs.base-image }}
@@ -43,9 +56,12 @@ runs:
       uses: ./.github/actions/upload-artifacts
       with:
         artifacts-dir: ${{ inputs.artifacts-dir }}
-        target-file: ./greptime
+        target-file: ./target/${{ inputs.cargo-profile }}/greptime
         version: ${{ inputs.version }}
         release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
         aws-access-key-id: ${{ inputs.aws-access-key-id }}
         aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
         aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}

.github/actions/build-greptime-images/action.yml

@@ -32,6 +32,10 @@ inputs:
   platforms:
     description: The supported platforms to build the image
     required: true
+  push-latest-tag:
+    description: Whether to push the latest tag
+    required: false
+    default: 'true'
 runs:
   using: composite
   steps:
@@ -76,7 +80,19 @@ runs:
         rm -rf arm64 && \
         mv ${{ inputs.arm64-artifact-name }} arm64
 
+    - name: Build and push images(without latest) for amd64 and arm64
+      if: ${{ inputs.push-latest-tag == 'false' }}
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ${{ inputs.docker-file }}
+        push: true
+        platforms: ${{ inputs.platforms }}
+        tags: |
+          ${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:${{ inputs.image-tag }}
+
     - name: Build and push images for amd64 and arm64
+      if: ${{ inputs.push-latest-tag == 'true' }}
       uses: docker/build-push-action@v3
       with:
         context: .

.github/actions/build-images/action.yml

@@ -7,6 +7,10 @@ inputs:
   image-namespace:
     description: The namespace of the image registry to store the images
     required: true
+  image-name:
+    description: The name of the image to build
+    required: false
+    default: greptimedb
   image-registry-username:
     description: The username to login to the image registry
     required: true
@@ -16,32 +20,43 @@ inputs:
   version:
     description: Version of the artifact
     required: true
+  push-latest-tag:
+    description: Whether to push the latest tag
+    required: false
+    default: 'true'
+  dev-mode:
+    description: Enable dev mode, only build standard greptime
+    required: false
+    default: 'false'
 runs:
   using: composite
   steps:
     - name: Build and push standard images to dockerhub
       uses: ./.github/actions/build-greptime-images
-      with:
+      with: # The image will be used as '${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:${{ inputs.version }}'
         image-registry: ${{ inputs.image-registry }}
         image-namespace: ${{ inputs.image-namespace }}
         image-registry-username: ${{ inputs.image-registry-username }}
         image-registry-password: ${{ inputs.image-registry-password }}
-        image-name: greptimedb
+        image-name: ${{ inputs.image-name }}
         image-tag: ${{ inputs.version }}
-        docker-file: docker/ci/Dockerfile
+        docker-file: docker/ci/ubuntu/Dockerfile
         amd64-artifact-name: greptime-linux-amd64-pyo3-${{ inputs.version }}
         arm64-artifact-name: greptime-linux-arm64-pyo3-${{ inputs.version }}
         platforms: linux/amd64,linux/arm64
+        push-latest-tag: ${{ inputs.push-latest-tag }}
 
     - name: Build and push centos images to dockerhub
+      if: ${{ inputs.dev-mode == 'false' }}
       uses: ./.github/actions/build-greptime-images
       with:
         image-registry: ${{ inputs.image-registry }}
         image-namespace: ${{ inputs.image-namespace }}
         image-registry-username: ${{ inputs.image-registry-username }}
         image-registry-password: ${{ inputs.image-registry-password }}
-        image-name: greptimedb-centos
+        image-name: ${{ inputs.image-name }}-centos
         image-tag: ${{ inputs.version }}
-        docker-file: docker/ci/Dockerfile-centos
+        docker-file: docker/ci/centos/Dockerfile
         amd64-artifact-name: greptime-linux-amd64-centos-${{ inputs.version }}
         platforms: linux/amd64
+        push-latest-tag: ${{ inputs.push-latest-tag }}

.github/actions/build-linux-artifacts/action.yml

@@ -25,6 +25,22 @@ inputs:
   aws-region:
     description: AWS region
     required: true
+  dev-mode:
+    description: Enable dev mode, only build standard greptime
+    required: false
+    default: 'false'
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
+  working-dir:
+    description: Working directory to build the artifacts
+    required: false
+    default: .
 runs:
   using: composite
   steps:
@@ -33,6 +49,7 @@ runs:
       shell: bash
       # NOTE: If the BUILD_JOBS > 4, it's always OOM in EC2 instance.
       run: |
+        cd ${{ inputs.working-dir }} && \
         make run-it-in-container BUILD_JOBS=4
 
     - name: Upload sqlness logs
@@ -55,8 +72,12 @@ runs:
         aws-access-key-id: ${{ inputs.aws-access-key-id }}
         aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
         aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}
 
     - name: Build greptime without pyo3
+      if: ${{ inputs.dev-mode == 'false' }}
       uses: ./.github/actions/build-greptime-binary
       with:
         base-image: ubuntu
@@ -68,10 +89,18 @@ runs:
         aws-access-key-id: ${{ inputs.aws-access-key-id }}
         aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
         aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}
+
+    - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build.
+      shell: bash
+      run: |
+        rm -rf ./target/
 
     - name: Build greptime on centos base image
       uses: ./.github/actions/build-greptime-binary
-      if: ${{ inputs.arch == 'amd64' }} # Only build centos7 base image for amd64.
+      if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Only build centos7 base image for amd64.
       with:
         base-image: centos
         features: servers/dashboard
@@ -82,3 +111,6 @@ runs:
         aws-access-key-id: ${{ inputs.aws-access-key-id }}
         aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
         aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}

.github/actions/build-macos-artifacts/action.yml

@@ -34,6 +34,10 @@ inputs:
   aws-region:
     description: AWS region
     required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
 runs:
   using: composite
   steps:
@@ -72,7 +76,7 @@ runs:
       uses: taiki-e/install-action@nextest
 
     - name: Run integration tests
-      if: ${{ inputs.disable_run_tests == 'false' }}
+      if: ${{ inputs.disable-run-tests == 'false' }}
       shell: bash
       run: |
         make test sqlness-test
@@ -103,3 +107,4 @@ runs:
         aws-access-key-id: ${{ inputs.aws-access-key-id }}
         aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
         aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}

.github/actions/upload-artifacts/action.yml

@@ -22,10 +22,31 @@ inputs:
   aws-region:
     description: AWS region
     required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
+  upload-max-retry-times:
+    description: Max retry times for uploading artifacts to S3
+    required: false
+    default: "20"
+  upload-retry-timeout:
+    description: Timeout for uploading artifacts to S3
+    required: false
+    default: "30" # minutes
+  working-dir:
+    description: Working directory to upload the artifacts
+    required: false
+    default: .
 runs:
   using: composite
   steps:
     - name: Create artifacts directory
+      working-directory: ${{ inputs.working-dir }}
       shell: bash
       run: |
         mkdir -p ${{ inputs.artifacts-dir }} && \
@@ -37,6 +58,7 @@ runs:
     #   greptime-linux-amd64-pyo3-v0.3.0
     #   └── greptime
     - name: Compress artifacts and calculate checksum
+      working-directory: ${{ inputs.working-dir }}
       shell: bash
       run: |
         tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }} && \
@@ -48,35 +70,56 @@ runs:
       uses: actions/upload-artifact@v3
       with:
         name: ${{ inputs.artifacts-dir }}
-        path: ${{ inputs.artifacts-dir }}.tar.gz
+        path: ${{ inputs.working-dir }}/${{ inputs.artifacts-dir }}.tar.gz
 
     - name: Upload checksum
       uses: actions/upload-artifact@v3
       with:
         name: ${{ inputs.artifacts-dir }}.sha256sum
-        path: ${{ inputs.artifacts-dir }}.sha256sum
-
-    - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v2
-      with:
-        aws-access-key-id: ${{ inputs.aws-access-key-id }}
-        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
-        aws-region: ${{ inputs.aws-region }}
+        path: ${{ inputs.working-dir }}/${{ inputs.artifacts-dir }}.sha256sum
 
     - name: Upload artifacts to S3
-      shell: bash
-      # The bucket layout will be:
-      # releases/greptimedb
-      # ├── v0.1.0
-      # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
-      # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
-      # └── v0.2.0
-      #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
-      #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
-      run: |
-        aws s3 cp \
-          ${{ inputs.artifacts-dir }}.tar.gz \
-          s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.tar.gz && \
-        aws s3 cp \
-          ${{ inputs.artifacts-dir }}.sha256sum \
-          s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.sha256sum
+      if: ${{ inputs.upload-to-s3 == 'true' }}
+      uses: nick-invision/retry@v2
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
+      with:
+        max_attempts: ${{ inputs.upload-max-retry-times }}
+        timeout_minutes: ${{ inputs.upload-retry-timeout }}
+        # The bucket layout will be:
+        # releases/greptimedb
+        # ├── v0.1.0
+        # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
+        # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
+        # └── v0.2.0
+        #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
+        #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
+        command: |
+          cd ${{ inputs.working-dir }} && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.tar.gz \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.tar.gz && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.sha256sum \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.sha256sum
+
+    - name: Upload latest artifacts to S3
+      if: ${{ inputs.upload-to-s3 == 'true' && inputs.upload-latest-artifacts == 'true' }} # We'll also upload the latest artifacts to S3 in the scheduled and formal release.
+      uses: nick-invision/retry@v2
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
+      with:
+        max_attempts: ${{ inputs.upload-max-retry-times }}
+        timeout_minutes: ${{ inputs.upload-retry-timeout }}
+        command: |
+          cd ${{ inputs.working-dir }} && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.tar.gz \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/latest/${{ inputs.artifacts-dir }}.tar.gz && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.sha256sum \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/latest/${{ inputs.artifacts-dir }}.sha256sum

.github/scripts/create-version.sh

@@ -3,8 +3,9 @@
 set -e
 
 # - If it's a tag push release, the version is the tag name(${{ github.ref_name }});
-# - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nigthly-20230313;
-# - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-$(git rev-parse --short HEAD)-YYYYMMDDSS', like v0.2.0-e5b243c-2023071245;
+# - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like 'v0.2.0-nightly-20230313';
+# - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-$(git rev-parse --short HEAD)-YYYYMMDDSS', like 'v0.2.0-e5b243c-2023071245';
+# - If it's a nightly build, the version is 'nightly-YYYYMMDD-$(git rev-parse --short HEAD)', like 'nightly-20230712-e5b243c'.
 # create_version ${GIHUB_EVENT_NAME} ${NEXT_RELEASE_VERSION} ${NIGHTLY_RELEASE_PREFIX}
 function create_version() {
   # Read from envrionment variables.
@@ -23,6 +24,24 @@ function create_version() {
       exit 1
   fi
 
+  # Reuse $NEXT_RELEASE_VERSION to identify whether it's a nightly build.
+  # It will be like 'nigtly-20230808-7d0d8dc6'.
+  if [ "$NEXT_RELEASE_VERSION" = nightly ]; then
+    echo "$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")-$(git rev-parse --short HEAD)"
+    exit 0
+  fi
+
+  # Reuse $NEXT_RELEASE_VERSION to identify whether it's a dev build.
+  # It will be like 'dev-2023080819-f0e7216c'.
+  if [ "$NEXT_RELEASE_VERSION" = dev ]; then
+    if [ -z "$COMMIT_SHA" ]; then
+      echo "COMMIT_SHA is empty in dev build"
+      exit 1
+    fi
+    echo "dev-$(date "+%Y%m%d-%s")-$(echo "$COMMIT_SHA" | cut -c1-8)"
+    exit 0
+  fi
+
   # Note: Only output 'version=xxx' to stdout when everything is ok, so that it can be used in GitHub Actions Outputs.
   if [ "$GITHUB_EVENT_NAME" = push ]; then
     if [ -z "$GITHUB_REF_NAME" ]; then
@@ -31,7 +50,7 @@ function create_version() {
     fi
     echo "$GITHUB_REF_NAME"
   elif [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
-    echo "$NEXT_RELEASE_VERSION-$(git rev-parse --short HEAD)-$(date "+%Y%m%d%S")"
+    echo "$NEXT_RELEASE_VERSION-$(git rev-parse --short HEAD)-$(date "+%Y%m%d-%s")"
   elif [ "$GITHUB_EVENT_NAME" = schedule ]; then
     echo "$NEXT_RELEASE_VERSION-$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")"
   else
@@ -40,5 +59,10 @@ function create_version() {
   fi
 }
 
-# You can run as: GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigthly GITHUB_REF_NAME=v0.3.0 ./create-version.sh
+# You can run as following examples:
+#  GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly GITHUB_REF_NAME=v0.3.0 ./create-version.sh
+#  GITHUB_EVENT_NAME=workflow_dispatch NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=nightly NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=workflow_dispatch COMMIT_SHA=f0e7216c4bb6acce9b29a21ec2d683be2e3f984a NEXT_RELEASE_VERSION=dev NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
 create_version

.github/workflows/apidoc.yml

@@ -13,7 +13,7 @@ on:
 name: Build API docs
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-03
+  RUST_TOOLCHAIN: nightly-2023-08-07
 
 jobs:
   apidoc:

.github/workflows/dev-build.yml

@@ -0,0 +1,344 @@
+# Development build only build the debug version of the artifacts manually.
+name: GreptimeDB Development Build
+
+on:
+  workflow_dispatch: # Allows you to run this workflow manually.
+    inputs:
+      repository:
+        description: The public repository to build
+        required: false
+        default: GreptimeTeam/greptimedb
+      commit: # Note: We only pull the source code and use the current workflow to build the artifacts.
+        description: The commit to build
+        required: true
+      linux_amd64_runner:
+        type: choice
+        description: The runner uses to build linux-amd64 artifacts
+        default: ec2-c6i.4xlarge-amd64
+        options:
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
+          - ec2-c6i.xlarge-amd64 # 4C8G
+          - ec2-c6i.2xlarge-amd64 # 8C16G
+          - ec2-c6i.4xlarge-amd64 # 16C32G
+          - ec2-c6i.8xlarge-amd64 # 32C64G
+          - ec2-c6i.16xlarge-amd64 # 64C128G
+      linux_arm64_runner:
+        type: choice
+        description: The runner uses to build linux-arm64 artifacts
+        default: ec2-c6g.4xlarge-arm64
+        options:
+          - ec2-c6g.xlarge-arm64 # 4C8G
+          - ec2-c6g.2xlarge-arm64 # 8C16G
+          - ec2-c6g.4xlarge-arm64 # 16C32G
+          - ec2-c6g.8xlarge-arm64 # 32C64G
+          - ec2-c6g.16xlarge-arm64 # 64C128G
+      skip_test:
+        description: Do not run integration tests during the build
+        type: boolean
+        default: true
+      build_linux_amd64_artifacts:
+        type: boolean
+        description: Build linux-amd64 artifacts
+        required: false
+        default: true
+      build_linux_arm64_artifacts:
+        type: boolean
+        description: Build linux-arm64 artifacts
+        required: false
+        default: true
+      release_images:
+        type: boolean
+        description: Build and push images to DockerHub and ACR
+        required: false
+        default: true
+
+# Use env variables to control all the release process.
+env:
+  CARGO_PROFILE: nightly
+
+  # Controls whether to run tests, include unit-test, integration-test and sqlness.
+  DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}
+
+  # Always use 'dev' to indicate it's the dev build.
+  NEXT_RELEASE_VERSION: dev
+
+  NIGHTLY_RELEASE_PREFIX: nightly
+
+  # Use the different image name to avoid conflict with the release images.
+  IMAGE_NAME: greptimedb-dev
+
+  # The source code will check out in the following path: '${WORKING_DIR}/dev/greptime'.
+  CHECKOUT_GREPTIMEDB_PATH: dev/greptimedb
+
+jobs:
+  allocate-runners:
+    name: Allocate runners
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-latest
+    outputs:
+      linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
+
+      # The following EC2 resource id will be used for resource releasing.
+      linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }}
+      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}
+
+      # The 'version' use as the global tag name of the release workflow.
+      version: ${{ steps.create-version.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create version
+        id: create-version
+        run: |
+          version=$(./.github/scripts/create-version.sh) && \
+          echo $version && \
+          echo "version=$version" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+          COMMIT_SHA: ${{ inputs.commit }}
+          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
+          NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
+
+      - name: Allocate linux-amd64 runner
+        if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-amd64-runner
+        with:
+          runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+      - name: Allocate linux-arm64 runner
+        if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-arm64-runner
+        with:
+          runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+  build-linux-amd64-artifacts:
+    name: Build linux-amd64 artifacts
+    if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Checkout greptimedb
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.repository }}
+          ref: ${{ inputs.commit }}
+          path: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: amd64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: true # Only build the standard greptime binary.
+          upload-to-s3: false # No need to upload to S3.
+          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+
+  build-linux-arm64-artifacts:
+    name: Build linux-arm64 artifacts
+    if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Checkout greptimedb
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.repository }}
+          ref: ${{ inputs.commit }}
+          path: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: arm64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: true # Only build the standard greptime binary.
+          upload-to-s3: false # No need to upload to S3.
+          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+
+  release-images-to-dockerhub:
+    name: Build and push images to DockerHub
+    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-latest
+    outputs:
+      build-result: ${{ steps.set-build-result.outputs.build-result }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push images to dockerhub
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: docker.io
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-name: ${{ env.IMAGE_NAME }}
+          image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          push-latest-tag: false # Don't push the latest tag to registry.
+          dev-mode: true # Only build the standard images.
+
+      - name: Set build result
+        id: set-build-result
+        run: |
+          echo "build-result=success" >> $GITHUB_OUTPUT
+
+  release-images-to-acr:
+    name: Build and push images to ACR
+    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-latest
+    # When we push to ACR, it's easy to fail due to some unknown network issues.
+    # However, we don't want to fail the whole workflow because of this.
+    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-name: ${{ env.IMAGE_NAME }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          push-latest-tag: false # Don't push the latest tag to registry.
+          dev-mode: true # Only build the standard images.
+
+  stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-amd64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-arm64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-arm64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  notification:
+    if: ${{ always() }} # Not requiring successful dependent jobs, always run.
+    name: Send notification to Greptime team
+    needs: [
+      release-images-to-dockerhub
+    ]
+    runs-on: ubuntu-latest
+    env:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
+    steps:
+      - name: Notifiy nightly build successful result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }}
+        with:
+          payload: |
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}
+
+      - name: Notifiy nightly build failed result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.release-images-to-dockerhub.outputs.build-result != 'success' }}
+        with:
+          payload: |
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."}

.github/workflows/develop.yml

@@ -24,8 +24,12 @@ on:
 
 name: CI
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-03
+  RUST_TOOLCHAIN: nightly-2023-08-07
 
 jobs:
   typos:
@@ -51,7 +55,7 @@ jobs:
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
       - name: Run cargo check
-        run: cargo check --workspace --all-targets
+        run: cargo check --locked --workspace --all-targets
 
   toml:
     name: Toml Check
@@ -62,60 +66,21 @@ jobs:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          toolchain: stable
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
       - name: Install taplo
-        run: cargo install taplo-cli --version ^0.8 --locked
+        run: cargo +stable install taplo-cli --version ^0.8 --locked
       - name: Run taplo
-        run: taplo format --check --option "indent_string=    "
-
-  # Use coverage to run test.
-  # test:
-  #   name: Test Suite
-  #   if: github.event.pull_request.draft == false
-  #   runs-on: ubuntu-latest
-  #   timeout-minutes: 60
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - name: Cache LLVM and Clang
-  #       id: cache-llvm
-  #       uses: actions/cache@v3
-  #       with:
-  #         path: ./llvm
-  #         key: llvm
-  #     - uses: arduino/setup-protoc@v1
-  #       with:
-  #         repo-token: ${{ secrets.GITHUB_TOKEN }}
-  #     - uses: KyleMayes/install-llvm-action@v1
-  #       with:
-  #         version: "14.0"
-  #         cached: ${{ steps.cache-llvm.outputs.cache-hit }}
-  #     - uses: dtolnay/rust-toolchain@master
-  #       with:
-  #         toolchain: ${{ env.RUST_TOOLCHAIN }}
-  #     - name: Rust Cache
-  #       uses: Swatinem/rust-cache@v2
-  #     - name: Cleanup disk
-  #       uses: curoky/cleanup-disk-action@v2.0
-  #       with:
-  #         retain: 'rust,llvm'
-  #     - name: Install latest nextest release
-  #       uses: taiki-e/install-action@nextest
-  #     - name: Run tests
-  #       run: cargo nextest run
-  #       env:
-  #         CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
-  #         RUST_BACKTRACE: 1
-  #         GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-  #         GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-  #         GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-  #         UNITTEST_LOG_DIR: "__unittest_logs"
+        run: taplo format --check
 
   sqlness:
     name: Sqlness Test
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest-8-cores, windows-latest-8-cores ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v3
@@ -127,25 +92,14 @@ jobs:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-      - name: Run etcd
-        run: |
-          ETCD_VER=v3.5.7
-          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
-          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-          mkdir -p /tmp/etcd-download
-          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
-          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-
-          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
-          nohup etcd >/tmp/etcd.log 2>&1 &
       - name: Run sqlness
-        run: cargo sqlness && ls /tmp
+        run: cargo sqlness
       - name: Upload sqlness logs
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: sqlness-logs
-          path: /tmp/greptime-*.log
+          path: ${{ runner.temp }}/greptime-*.log
           retention-days: 3
 
   fmt:
@@ -234,3 +188,43 @@ jobs:
           flags: rust
           fail_ci_if_error: false
           verbose: true
+
+  test-on-windows:
+    if: github.event.pull_request.draft == false
+    runs-on: windows-latest-8-cores
+    timeout-minutes: 60
+    steps:
+      - run: git config --global core.autocrlf false
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
+      - name: Install WSL distribution
+        uses: Vampire/setup-wsl@v2
+        with:
+         distribution: Ubuntu-22.04
+      - name: Running tests
+        run: cargo nextest run -F pyo3_backend,dashboard
+        env:
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          GT_S3_REGION: ${{ secrets.S3_REGION }}
+          UNITTEST_LOG_DIR: "__unittest_logs"

.github/workflows/nightly-build.yml

@@ -0,0 +1,311 @@
+# Nightly build only do the following things:
+# 1. Run integration tests;
+# 2. Build binaries and images for linux-amd64 and linux-arm64 platform;
+name: GreptimeDB Nightly Build
+
+on:
+  schedule:
+    # Trigger at 00:00(UTC) on every day-of-week from Monday through Friday.
+    - cron: '0 0 * * 1-5'
+  workflow_dispatch: # Allows you to run this workflow manually.
+    inputs:
+      linux_amd64_runner:
+        type: choice
+        description: The runner uses to build linux-amd64 artifacts
+        default: ec2-c6i.2xlarge-amd64
+        options:
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
+          - ec2-c6i.xlarge-amd64 # 4C8G
+          - ec2-c6i.2xlarge-amd64 # 8C16G
+          - ec2-c6i.4xlarge-amd64 # 16C32G
+          - ec2-c6i.8xlarge-amd64 # 32C64G
+          - ec2-c6i.16xlarge-amd64 # 64C128G
+      linux_arm64_runner:
+        type: choice
+        description: The runner uses to build linux-arm64 artifacts
+        default: ec2-c6g.2xlarge-arm64
+        options:
+          - ec2-c6g.xlarge-arm64 # 4C8G
+          - ec2-c6g.2xlarge-arm64 # 8C16G
+          - ec2-c6g.4xlarge-arm64 # 16C32G
+          - ec2-c6g.8xlarge-arm64 # 32C64G
+          - ec2-c6g.16xlarge-arm64 # 64C128G
+      skip_test:
+        description: Do not run integration tests during the build
+        type: boolean
+        default: true
+      build_linux_amd64_artifacts:
+        type: boolean
+        description: Build linux-amd64 artifacts
+        required: false
+        default: false
+      build_linux_arm64_artifacts:
+        type: boolean
+        description: Build linux-arm64 artifacts
+        required: false
+        default: false
+      release_images:
+        type: boolean
+        description: Build and push images to DockerHub and ACR
+        required: false
+        default: false
+
+# Use env variables to control all the release process.
+env:
+  CARGO_PROFILE: nightly
+
+  # Controls whether to run tests, include unit-test, integration-test and sqlness.
+  DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}
+
+  # Always use 'nightly' to indicate it's the nightly build.
+  NEXT_RELEASE_VERSION: nightly
+
+  NIGHTLY_RELEASE_PREFIX: nightly
+
+jobs:
+  allocate-runners:
+    name: Allocate runners
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-latest
+    outputs:
+      linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
+
+      # The following EC2 resource id will be used for resource releasing.
+      linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }}
+      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}
+
+      # The 'version' use as the global tag name of the release workflow.
+      version: ${{ steps.create-version.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create version
+        id: create-version
+        run: |
+          version=$(./.github/scripts/create-version.sh) && \
+          echo $version && \
+          echo "version=$version" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
+          NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
+
+      - name: Allocate linux-amd64 runner
+        if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-amd64-runner
+        with:
+          runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+      - name: Allocate linux-arm64 runner
+        if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-arm64-runner
+        with:
+          runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+  build-linux-amd64-artifacts:
+    name: Build linux-amd64 artifacts
+    if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: amd64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-latest-artifacts: false
+
+  build-linux-arm64-artifacts:
+    name: Build linux-arm64 artifacts
+    if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: arm64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-latest-artifacts: false
+
+  release-images-to-dockerhub:
+    name: Build and push images to DockerHub
+    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-latest
+    outputs:
+      nightly-build-result: ${{ steps.set-nightly-build-result.outputs.nightly-build-result }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push images to dockerhub
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: docker.io
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          push-latest-tag: false # Don't push the latest tag to registry.
+
+      - name: Set nightly build result
+        id: set-nightly-build-result
+        run: |
+          echo "nightly-build-result=success" >> $GITHUB_OUTPUT
+
+  release-images-to-acr:
+    name: Build and push images to ACR
+    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-latest
+    # When we push to ACR, it's easy to fail due to some unknown network issues.
+    # However, we don't want to fail the whole workflow because of this.
+    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          push-latest-tag: false # Don't push the latest tag to registry.
+
+  stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-amd64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-arm64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-arm64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  notification:
+    if: ${{ always() }} # Not requiring successful dependent jobs, always run.
+    name: Send notification to Greptime team
+    needs: [
+      release-images-to-dockerhub
+    ]
+    runs-on: ubuntu-latest
+    env:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
+    steps:
+      - name: Notifiy nightly build successful result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result == 'success' }}
+        with:
+          payload: |
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}
+
+      - name: Notifiy nightly build failed result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result != 'success' }}
+        with:
+          payload: |
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."}

.github/workflows/release.yml

@@ -82,7 +82,7 @@ on:
 # Use env variables to control all the release process.
 env:
   # The arguments of building greptime.
-  RUST_TOOLCHAIN: nightly-2023-05-03
+  RUST_TOOLCHAIN: nightly-2023-08-07
   CARGO_PROFILE: nightly
 
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
@@ -181,6 +181,7 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
@@ -204,6 +205,7 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
 
   build-macos-artifacts:
     name: Build macOS artifacts
@@ -250,6 +252,7 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}
 
   release-images-to-dockerhub:
     name: Build and push images to DockerHub
@@ -336,7 +339,7 @@ jobs:
         uses: ./.github/actions/build-dev-builder-image
         with:
           dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
-          dockerhub-image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
           acr-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
           acr-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
           acr-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}

Cargo.toml

@@ -2,22 +2,24 @@
 members = [
     "benchmarks",
     "src/api",
+    "src/auth",
     "src/catalog",
     "src/client",
     "src/cmd",
     "src/common/base",
     "src/common/catalog",
+    "src/common/config",
     "src/common/datasource",
     "src/common/error",
     "src/common/function",
-    "src/common/function-macro",
+    "src/common/macro",
+    "src/common/greptimedb-telemetry",
     "src/common/grpc",
     "src/common/grpc-expr",
     "src/common/mem-prof",
     "src/common/meta",
     "src/common/procedure",
     "src/common/procedure-test",
-    "src/common/pprof",
     "src/common/query",
     "src/common/recordbatch",
     "src/common/runtime",
@@ -25,16 +27,17 @@ members = [
     "src/common/telemetry",
     "src/common/test-util",
     "src/common/time",
+    "src/common/version",
     "src/datanode",
     "src/datatypes",
-    "src/file-table-engine",
+    "src/file-engine",
     "src/frontend",
     "src/log-store",
     "src/meta-client",
     "src/meta-srv",
-    "src/mito",
     "src/mito2",
     "src/object-store",
+    "src/operator",
     "src/partition",
     "src/promql",
     "src/query",
@@ -45,17 +48,18 @@ members = [
     "src/storage",
     "src/store-api",
     "src/table",
-    "src/table-procedure",
     "tests-integration",
     "tests/runner",
 ]
+resolver = "2"
 
 [workspace.package]
-version = "0.3.2"
+version = "0.4.0-nightly"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
+aquamarine = "0.3"
 arrow = { version = "43.0" }
 etcd-client = "0.11"
 arrow-array = "43.0"
@@ -64,17 +68,18 @@ arrow-schema = { version = "43.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "2ceb7f927c40787773fdc466d6a4b79f3a6c0001" }
+datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
 derive_builder = "0.12"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "eeae2d0dfa8ee320a7b9e987b4631a6c1c732ebd" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "115c1080773be8a819e50b257fece9f839a0c836" }
+humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
 once_cell = "1.18"
@@ -84,10 +89,19 @@ paste = "1.0"
 prost = "0.11"
 rand = "0.8"
 regex = "1.8"
+reqwest = { version = "0.11", default-features = false, features = [
+    "json",
+    "rustls-tls-native-roots",
+    "stream",
+] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+smallvec = "1"
 snafu = { version = "0.7", features = ["backtraces"] }
-sqlparser = "0.35"
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4", features = [
+    "visitor",
+] }
+strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
 tokio = { version = "1.28", features = ["full"] }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
@@ -96,6 +110,58 @@ tonic = { version = "0.9", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
 metrics = "0.20"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
+## workspaces members
+api = { path = "src/api" }
+auth = { path = "src/auth" }
+catalog = { path = "src/catalog" }
+client = { path = "src/client" }
+cmd = { path = "src/cmd" }
+common-base = { path = "src/common/base" }
+common-catalog = { path = "src/common/catalog" }
+common-config = { path = "src/common/config" }
+common-datasource = { path = "src/common/datasource" }
+common-error = { path = "src/common/error" }
+common-function = { path = "src/common/function" }
+common-macro = { path = "src/common/macro" }
+common-greptimedb-telemetry = { path = "src/common/greptimedb-telemetry" }
+common-grpc = { path = "src/common/grpc" }
+common-grpc-expr = { path = "src/common/grpc-expr" }
+common-mem-prof = { path = "src/common/mem-prof" }
+common-meta = { path = "src/common/meta" }
+common-procedure = { path = "src/common/procedure" }
+common-procedure-test = { path = "src/common/procedure-test" }
+common-pprof = { path = "src/common/pprof" }
+common-query = { path = "src/common/query" }
+common-recordbatch = { path = "src/common/recordbatch" }
+common-runtime = { path = "src/common/runtime" }
+substrait = { path = "src/common/substrait" }
+common-telemetry = { path = "src/common/telemetry" }
+common-test-util = { path = "src/common/test-util" }
+common-time = { path = "src/common/time" }
+common-version = { path = "src/common/version" }
+datanode = { path = "src/datanode" }
+datatypes = { path = "src/datatypes" }
+file-engine = { path = "src/file-engine" }
+frontend = { path = "src/frontend" }
+log-store = { path = "src/log-store" }
+meta-client = { path = "src/meta-client" }
+meta-srv = { path = "src/meta-srv" }
+mito = { path = "src/mito" }
+mito2 = { path = "src/mito2" }
+operator = { path = "src/operator" }
+object-store = { path = "src/object-store" }
+partition = { path = "src/partition" }
+promql = { path = "src/promql" }
+query = { path = "src/query" }
+raft-engine = { git = "https://github.com/tikv/raft-engine.git", rev = "22dfb426cd994602b57725ef080287d3e53db479" }
+script = { path = "src/script" }
+servers = { path = "src/servers" }
+session = { path = "src/session" }
+sql = { path = "src/sql" }
+storage = { path = "src/storage" }
+store-api = { path = "src/store-api" }
+table = { path = "src/table" }
+table-procedure = { path = "src/table-procedure" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"

Makefile

@@ -12,6 +12,8 @@ BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu
 RUST_TOOLCHAIN ?= $(shell cat rust-toolchain.toml | grep channel | cut -d'"' -f2)
 CARGO_REGISTRY_CACHE ?= ${HOME}/.cargo/registry
+ARCH := $(shell uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
+OUTPUT_DIR := $(shell if [ "$(RELEASE)" = "true" ]; then echo "release"; elif [ ! -z "$(CARGO_PROFILE)" ]; then echo "$(CARGO_PROFILE)" ; else echo "debug"; fi)
 
 # The arguments for running integration tests.
 ETCD_VERSION ?= v3.5.9
@@ -43,6 +45,10 @@ ifneq ($(strip $(TARGET)),)
 	CARGO_BUILD_OPTS += --target ${TARGET}
 endif
 
+ifneq ($(strip $(RELEASE)),)
+	CARGO_BUILD_OPTS += --release
+endif
+
 ifeq ($(BUILDX_MULTI_PLATFORM_BUILD), true)
 	BUILDX_MULTI_PLATFORM_BUILD_OPTS := --platform linux/amd64,linux/arm64 --push
 else
@@ -52,26 +58,20 @@ endif
 ##@ Build
 
 .PHONY: build
-build: ## Build debug version greptime. If USE_DEV_BUILDER is true, the binary will be built in dev-builder.
-ifeq ($(USE_DEV_BUILDER), true)
-	docker run --network=host \
-	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
-	make build CARGO_PROFILE=${CARGO_PROFILE} FEATURES=${FEATURES} TARGET_DIR=${TARGET_DIR}
-else
+build: ## Build debug version greptime.
 	cargo build ${CARGO_BUILD_OPTS}
-endif
 
-.PHONY: release
-release:  ## Build release version greptime. If USE_DEV_BUILDER is true, the binary will be built in dev-builder.
-ifeq ($(USE_DEV_BUILDER), true)
+.POHNY: build-by-dev-builder
+build-by-dev-builder: ## Build greptime by dev-builder.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
-	make release CARGO_PROFILE=${CARGO_PROFILE} FEATURES=${FEATURES} TARGET_DIR=${TARGET_DIR}
-else
-	cargo build --release ${CARGO_BUILD_OPTS}
-endif
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
+	make build \
+	CARGO_PROFILE=${CARGO_PROFILE} \
+	FEATURES=${FEATURES} \
+	TARGET_DIR=${TARGET_DIR} \
+	TARGET=${TARGET} \
+	RELEASE=${RELEASE}
 
 .PHONY: clean
 clean: ## Clean the project.
@@ -83,37 +83,34 @@ fmt: ## Format all the Rust code.
 
 .PHONY: fmt-toml
 fmt-toml: ## Format all TOML files.
-	taplo format --option "indent_string=    "
+	taplo format
 
 .PHONY: check-toml
 check-toml: ## Check all TOML files.
-	taplo format --check --option "indent_string=    "
+	taplo format --check
 
 .PHONY: docker-image
-docker-image: multi-platform-buildx ## Build docker image.
+docker-image: build-by-dev-builder ## Build docker image.
+	mkdir -p ${ARCH} && \
+	cp ./target/${OUTPUT_DIR}/greptime ${ARCH}/greptime && \
+	docker build -f docker/ci/${BASE_IMAGE}/Dockerfile -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb:${IMAGE_TAG} . && \
+	rm -r ${ARCH}
+
+.PHONY: docker-image-buildx
+docker-image-buildx: multi-platform-buildx ## Build docker image by buildx.
 	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
-	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" --build-arg="FEATURES=${FEATURES}" \
-	  -f docker/${BASE_IMAGE}/Dockerfile \
+	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" \
+	  --build-arg="FEATURES=${FEATURES}" \
+	  --build-arg="OUTPUT_DIR=${OUTPUT_DIR}" \
+	  -f docker/buildx/${BASE_IMAGE}/Dockerfile \
 	  -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
 
-.PHONY: build-greptime-by-buildx
-build-greptime-by-buildx: multi-platform-buildx ## Build greptime binary by docker buildx. The binary will be copied to the current directory.
-	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
-	  --target=builder \
-	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" --build-arg="FEATURES=${FEATURES}" \
-	  -f docker/${BASE_IMAGE}/Dockerfile \
-	  -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb-builder:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
-
-	docker run --rm -v ${PWD}:/data \
-      --entrypoint cp ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb-builder:${IMAGE_TAG} \
-      /out/target/${CARGO_PROFILE}/greptime /data/greptime
-
 .PHONY: dev-builder
 dev-builder: multi-platform-buildx ## Build dev-builder image.
 	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
 	--build-arg="RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" \
-	-f docker/dev-builder/Dockerfile \
-	-t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
+	-f docker/dev-builder/${BASE_IMAGE}/Dockerfile \
+	-t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
 
 .PHONY: multi-platform-buildx
 multi-platform-buildx: ## Create buildx multi-platform builder.
@@ -137,7 +134,7 @@ check: ## Cargo check all the targets.
 
 .PHONY: clippy
 clippy: ## Check clippy rules.
-	cargo clippy --workspace --all-targets -- -D warnings
+	cargo clippy --workspace --all-targets -F pyo3_backend -- -D warnings
 
 .PHONY: fmt-check
 fmt-check: ## Check code format.
@@ -155,7 +152,7 @@ stop-etcd: ## Stop single node etcd for testing purpose.
 run-it-in-container: start-etcd ## Run integration tests in dev-builder.
 	docker run --network=host \
 	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v /tmp:/tmp \
-	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
 	make test sqlness-test BUILD_JOBS=${BUILD_JOBS}
 
 ##@ General

benchmarks/Cargo.toml

@@ -7,7 +7,7 @@ license.workspace = true
 [dependencies]
 arrow.workspace = true
 clap = { version = "4.0", features = ["derive"] }
-client = { path = "../src/client" }
+client = { workspace = true }
 indicatif = "0.17.1"
 itertools.workspace = true
 parquet.workspace = true

benchmarks/src/bin/nyc-taxi.rs

@@ -27,7 +27,7 @@ use arrow::record_batch::RecordBatch;
 use clap::Parser;
 use client::api::v1::column::Values;
 use client::api::v1::{
-    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests,
+    Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests, SemanticType,
 };
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
@@ -105,7 +105,6 @@ async fn write_data(
         let (columns, row_count) = convert_record_batch(record_batch);
         let request = InsertRequest {
             table_name: TABLE_NAME.to_string(),
-            region_number: 0,
             columns,
             row_count,
         };
@@ -189,7 +188,7 @@ fn build_values(column: &ArrayRef) -> (Values, ColumnDataType) {
             let values = array.values();
             (
                 Values {
-                    ts_microsecond_values: values.to_vec(),
+                    timestamp_microsecond_values: values.to_vec(),
                     ..Default::default()
                 },
                 ColumnDataType::TimestampMicrosecond,
@@ -253,124 +252,161 @@ fn create_table_expr() -> CreateTableExpr {
         column_defs: vec![
             ColumnDef {
                 name: "VendorID".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                data_type: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Tag as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "tpep_pickup_datetime".to_string(),
-                datatype: ColumnDataType::TimestampMicrosecond as i32,
+                data_type: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Timestamp as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "tpep_dropoff_datetime".to_string(),
-                datatype: ColumnDataType::TimestampMicrosecond as i32,
+                data_type: ColumnDataType::TimestampMicrosecond as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "passenger_count".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "trip_distance".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "RatecodeID".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "store_and_fwd_flag".to_string(),
-                datatype: ColumnDataType::String as i32,
+                data_type: ColumnDataType::String as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "PULocationID".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                data_type: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "DOLocationID".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                data_type: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "payment_type".to_string(),
-                datatype: ColumnDataType::Int64 as i32,
+                data_type: ColumnDataType::Int64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "fare_amount".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "extra".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "mta_tax".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "tip_amount".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "tolls_amount".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "improvement_surcharge".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "total_amount".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "congestion_surcharge".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "airport_fee".to_string(),
-                datatype: ColumnDataType::Float64 as i32,
+                data_type: ColumnDataType::Float64 as i32,
                 is_nullable: true,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
         ],
         time_index: "tpep_pickup_datetime".to_string(),
         primary_keys: vec!["VendorID".to_string()],
         create_if_not_exists: false,
         table_options: Default::default(),
-        region_numbers: vec![0],
         table_id: None,
         engine: "mito".to_string(),
     }

config/datanode.example.toml

@@ -1,7 +1,5 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
-# Whether to use in-memory catalog, see `standalone.example.toml`.
-enable_memory_catalog = false
 # The datanode identifier, should be unique.
 node_id = 42
 # gRPC server address, "127.0.0.1:3001" by default.
@@ -10,13 +8,16 @@ rpc_addr = "127.0.0.1:3001"
 rpc_hostname = "127.0.0.1"
 # The number of gRPC server worker threads, 8 by default.
 rpc_runtime_size = 8
+# Start services after regions have obtained leases.
+# It will block the datanode start if it can't receive leases in the heartbeat from metasrv.
+require_lease_before_startup = false
 
 [heartbeat]
 # Interval for sending heartbeat messages to the Metasrv in milliseconds, 5000 by default.
 interval_millis = 5000
 
 # Metasrv client options.
-[meta_client_options]
+[meta_client]
 # Metasrv address list.
 metasrv_addrs = ["127.0.0.1:3002"]
 # Operation timeout in milliseconds, 3000 by default.
@@ -38,8 +39,9 @@ sync_write = false
 
 # Storage options, see `standalone.example.toml`.
 [storage]
-type = "File"
+# The working home directory.
 data_home = "/tmp/greptimedb/"
+type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
 
@@ -56,8 +58,6 @@ max_purge_tasks = 32
 checkpoint_margin = 10
 # Region manifest logs and checkpoints gc execution duration
 gc_duration = '10m'
-# Whether to try creating a manifest checkpoint on region opening
-checkpoint_on_startup = false
 
 # Storage flush options
 [storage.flush]
@@ -72,10 +72,27 @@ auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
 
-# Procedure storage options, see `standalone.example.toml`.
-[procedure]
-max_retry_times = 3
-retry_delay = "500ms"
+# Mito engine options
+[[region_engine]]
+[region_engine.mito]
+# Number of region workers
+num_workers = 8
+# Request channel size of each worker
+worker_channel_size = 128
+# Max batch size for a worker to handle requests
+worker_request_batch_size = 64
+# Number of meta action updated to trigger a new checkpoint for the manifest
+manifest_checkpoint_distance = 10
+# Manifest compression type
+manifest_compress_type = "Uncompressed"
+# Max number of running background jobs
+max_background_jobs = 4
+# Interval to auto flush a region if it has not flushed yet.
+auto_flush_interval = "1h"
+# Global write buffer size for all regions.
+global_write_buffer_size = "1GB"
+# Global write buffer size threshold to reject write requests (default 2G).
+global_write_buffer_reject_size = "2GB"
 
 # Log options
 # [logging]

config/frontend.example.toml

@@ -8,57 +8,56 @@ interval_millis = 5000
 retry_interval_millis = 5000
 
 # HTTP server options, see `standalone.example.toml`.
-[http_options]
+[http]
 addr = "127.0.0.1:4000"
 timeout = "30s"
 body_limit = "64MB"
 
 # gRPC server options, see `standalone.example.toml`.
-[grpc_options]
+[grpc]
 addr = "127.0.0.1:4001"
 runtime_size = 8
 
 # MySQL server options, see `standalone.example.toml`.
-[mysql_options]
+[mysql]
+enable = true
 addr = "127.0.0.1:4002"
 runtime_size = 2
 
 # MySQL server TLS options, see `standalone.example.toml`.
-[mysql_options.tls]
+[mysql.tls]
 mode = "disable"
 cert_path = ""
 key_path = ""
 
 # PostgresSQL server options, see `standalone.example.toml`.
-[postgres_options]
+[postgres]
+enable = true
 addr = "127.0.0.1:4003"
 runtime_size = 2
 
 # PostgresSQL server TLS options, see `standalone.example.toml`.
-[postgres_options.tls]
+[postgres.tls]
 mode = "disable"
 cert_path = ""
 key_path = ""
 
 # OpenTSDB protocol options, see `standalone.example.toml`.
-[opentsdb_options]
+[opentsdb]
+enable = true
 addr = "127.0.0.1:4242"
 runtime_size = 2
 
 # InfluxDB protocol options, see `standalone.example.toml`.
-[influxdb_options]
+[influxdb]
 enable = true
 
 # Prometheus remote storage options, see `standalone.example.toml`.
-[prom_store_options]
+[prom_store]
 enable = true
 
-# Prometheus protocol options, see `standalone.example.toml`.
-[prometheus_options]
-addr = "127.0.0.1:4004"
-
 # Metasrv client options, see `datanode.example.toml`.
-[meta_client_options]
+[meta_client]
 metasrv_addrs = ["127.0.0.1:3002"]
 timeout_millis = 3000
 # DDL timeouts options.

config/metasrv.example.toml

@@ -1,11 +1,11 @@
+# The working home directory.
+data_home = "/tmp/metasrv/"
 # The bind address of metasrv, "127.0.0.1:3002" by default.
 bind_addr = "127.0.0.1:3002"
 # The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost.
 server_addr = "127.0.0.1:3002"
 # Etcd server address, "127.0.0.1:2379" by default.
 store_addr = "127.0.0.1:2379"
-# Datanode lease in seconds, 15 seconds by default.
-datanode_lease_secs = 15
 # Datanode selector type.
 # - "LeaseBased" (default value).
 # - "LoadBased"
@@ -13,6 +13,8 @@ datanode_lease_secs = 15
 selector = "LeaseBased"
 # Store data in memory, false by default.
 use_memory_store = false
+# Whether to enable greptimedb telemetry, true by default.
+enable_telemetry = true
 
 # Log options, see `standalone.example.toml`
 # [logging]
@@ -22,7 +24,7 @@ use_memory_store = false
 # Procedure storage options.
 [procedure]
 # Procedure max retry time.
-max_retry_times = 3
+max_retry_times = 12
 # Initial retry delay of procedures, increases exponentially
 retry_delay = "500ms"
 

config/standalone.example.toml

@@ -1,10 +1,10 @@
 # Node running mode, "standalone" or "distributed".
 mode = "standalone"
-# Whether to use in-memory catalog, `false` by default.
-enable_memory_catalog = false
+# Whether to enable greptimedb telemetry, true by default.
+enable_telemetry = true
 
 # HTTP server options.
-[http_options]
+[http]
 # Server address, "127.0.0.1:4000" by default.
 addr = "127.0.0.1:4000"
 # HTTP request timeout, 30s by default.
@@ -14,21 +14,23 @@ timeout = "30s"
 body_limit = "64MB"
 
 # gRPC server options.
-[grpc_options]
+[grpc]
 # Server address, "127.0.0.1:4001" by default.
 addr = "127.0.0.1:4001"
 # The number of server worker threads, 8 by default.
 runtime_size = 8
 
 # MySQL server options.
-[mysql_options]
+[mysql]
+# Whether to enable
+enable = true
 # Server address, "127.0.0.1:4002" by default.
 addr = "127.0.0.1:4002"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # MySQL server TLS options.
-[mysql_options.tls]
+[mysql.tls]
 # TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
 # - "disable" (default value)
 # - "prefer"
@@ -42,14 +44,16 @@ cert_path = ""
 key_path = ""
 
 # PostgresSQL server options.
-[postgres_options]
+[postgres]
+# Whether to enable
+enable = true
 # Server address, "127.0.0.1:4003" by default.
 addr = "127.0.0.1:4003"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # PostgresSQL server TLS options, see `[mysql_options.tls]` section.
-[postgres_options.tls]
+[postgres.tls]
 # TLS mode.
 mode = "disable"
 # certificate file path.
@@ -58,31 +62,26 @@ cert_path = ""
 key_path = ""
 
 # OpenTSDB protocol options.
-[opentsdb_options]
+[opentsdb]
+# Whether to enable
+enable = true
 # OpenTSDB telnet API server address, "127.0.0.1:4242" by default.
 addr = "127.0.0.1:4242"
 # The number of server worker threads, 2 by default.
 runtime_size = 2
 
 # InfluxDB protocol options.
-[influxdb_options]
+[influxdb]
 # Whether to enable InfluxDB protocol in HTTP API, true by default.
 enable = true
 
 # Prometheus remote storage options
-[prom_store_options]
+[prom_store]
 # Whether to enable Prometheus remote write and read in HTTP API, true by default.
 enable = true
 
-# Prometheus protocol options
-[prometheus_options]
-# Prometheus API server address, "127.0.0.1:4004" by default.
-addr = "127.0.0.1:4004"
-
 # WAL options.
 [wal]
-# WAL data directory
-# dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
 file_size = "256MB"
 # WAL purge threshold.
@@ -94,12 +93,26 @@ read_batch_size = 128
 # Whether to sync log file after every write.
 sync_write = false
 
+# Kv options.
+[kv_store]
+# Kv file size in bytes.
+file_size = "256MB"
+# Kv purge threshold.
+purge_threshold = "4GB"
+
+# Procedure storage options.
+[procedure]
+# Procedure max retry time.
+max_retry_times = 3
+# Initial retry delay of procedures, increases exponentially
+retry_delay = "500ms"
+
 # Storage options.
 [storage]
+# The working home directory.
+data_home = "/tmp/greptimedb/"
 # Storage type.
 type = "File"
-# Data directory, "/tmp/greptimedb/data" by default.
-data_home = "/tmp/greptimedb/"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
 
@@ -119,8 +132,6 @@ max_purge_tasks = 32
 checkpoint_margin = 10
 # Region manifest logs and checkpoints gc execution duration
 gc_duration = '10m'
-# Whether to try creating a manifest checkpoint on region opening
-checkpoint_on_startup = false
 
 # Storage flush options
 [storage.flush]
@@ -135,13 +146,6 @@ auto_flush_interval = "1h"
 # Global write buffer size for all regions.
 global_write_buffer_size = "1GB"
 
-# Procedure storage options.
-[procedure]
-# Procedure max retry time.
-max_retry_times = 3
-# Initial retry delay of procedures, increases exponentially
-retry_delay = "500ms"
-
 # Log options
 # [logging]
 # Specify logs directory.

docker/buildx/centos/Dockerfile

@@ -2,6 +2,7 @@ FROM centos:7 as builder
 
 ARG CARGO_PROFILE
 ARG FEATURES
+ARG OUTPUT_DIR
 
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
@@ -13,7 +14,8 @@ RUN yum install -y epel-release  \
     openssl-devel  \
     centos-release-scl  \
     rh-python38  \
-    rh-python38-python-devel
+    rh-python38-python-devel \
+    which
 
 # Install protoc
 RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
@@ -35,17 +37,18 @@ RUN --mount=target=.,rw \
 # Export the binary to the clean image.
 FROM centos:7 as base
 
-ARG CARGO_PROFILE
+ARG OUTPUT_DIR
 
 RUN yum install -y epel-release \
     openssl \
     openssl-devel  \
     centos-release-scl  \
     rh-python38  \
-    rh-python38-python-devel
+    rh-python38-python-devel \
+    which
 
 WORKDIR /greptime
-COPY --from=builder /out/target/${CARGO_PROFILE}/greptime /greptime/bin/
+COPY --from=builder /out/target/${OUTPUT_DIR}/greptime /greptime/bin/
 ENV PATH /greptime/bin/:$PATH
 
 ENTRYPOINT ["greptime"]

docker/buildx/ubuntu/Dockerfile

@@ -2,6 +2,7 @@ FROM ubuntu:22.04 as builder
 
 ARG CARGO_PROFILE
 ARG FEATURES
+ARG OUTPUT_DIR
 
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
@@ -25,7 +26,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-mo
 ENV PATH /root/.cargo/bin/:$PATH
 
 # Build the project in release mode.
-RUN --mount=target=.,rw \
+RUN --mount=target=. \
     --mount=type=cache,target=/root/.cargo/registry \
     make build \
     CARGO_PROFILE=${CARGO_PROFILE} \
@@ -36,7 +37,7 @@ RUN --mount=target=.,rw \
 # TODO(zyy17): Maybe should use the more secure container image.
 FROM ubuntu:22.04 as base
 
-ARG CARGO_PROFILE
+ARG OUTPUT_DIR
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get \
     -y install ca-certificates \
@@ -50,7 +51,7 @@ COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt
 RUN python3 -m pip install -r /etc/greptime/requirements.txt
 
 WORKDIR /greptime
-COPY --from=builder /out/target/${CARGO_PROFILE}/greptime /greptime/bin/
+COPY --from=builder /out/target/${OUTPUT_DIR}/greptime /greptime/bin/
 ENV PATH /greptime/bin/:$PATH
 
 ENTRYPOINT ["greptime"]

docker/dev-builder/centos/Dockerfile

@@ -0,0 +1,29 @@
+FROM centos:7 as builder
+
+ENV LANG en_US.utf8
+
+# Install dependencies
+RUN ulimit -n 1024000 && yum groupinstall -y 'Development Tools'
+RUN yum install -y epel-release  \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel \
+    which
+
+# Install protoc
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
+RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
+
+# Install Rust
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH
+
+# Install Rust toolchains.
+ARG RUST_TOOLCHAIN
+RUN rustup toolchain install ${RUST_TOOLCHAIN}
+
+# Install nextest.
+RUN cargo install cargo-nextest --locked

docs/rfcs/2023-08-04-table-trait-refactor.md

@@ -0,0 +1,175 @@
+---
+Feature Name: table-trait-refactor
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/2065
+Date: 2023-08-04
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+Refactor Table Trait
+--------------------
+
+# Summary
+Refactor `Table` trait to adapt the new region server architecture and make code more straightforward.
+
+# Motivation
+The `Table` is designed in the background of both frontend and datanode keeping the same concepts. And all the operations are served by a `Table`. However, in our practice, we found that not all the operations are suitable to be served by a `Table`. For example, the `Table` doesn't hold actual physical data itself, thus operations like write or alter are simply a proxy over underlying regions. And in the recent refactor to datanode ([rfc table-engine-refactor](./2023-07-06-table-engine-refactor.md)), we are changing datanode to region server that is only aware of `Region` things. This also calls for a refactor to the `Table` trait.
+
+# Details
+
+## Definitions
+
+The current `Table` trait contains the following methods:
+```rust
+pub trait Table {
+    /// Get a reference to the schema for this table
+    fn schema(&self) -> SchemaRef;
+
+    /// Get a reference to the table info.
+    fn table_info(&self) -> TableInfoRef;
+
+    /// Get the type of this table for metadata/catalog purposes.
+    fn table_type(&self) -> TableType;
+
+    /// Insert values into table.
+    ///
+    /// Returns number of inserted rows.
+    async fn insert(&self, _request: InsertRequest) -> Result<usize>;
+
+    /// Generate a record batch stream for querying.
+    async fn scan_to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream>;
+
+    /// Tests whether the table provider can make use of any or all filter expressions
+    /// to optimise data retrieval.
+    fn supports_filters_pushdown(&self, filters: &[&Expr]) -> Result<Vec<FilterPushDownType>>;
+
+    /// Alter table.
+    async fn alter(&self, _context: AlterContext, _request: &AlterTableRequest) -> Result<()>;
+
+    /// Delete rows in the table.
+    ///
+    /// Returns number of deleted rows.
+    async fn delete(&self, _request: DeleteRequest) -> Result<usize>;
+
+    /// Flush table.
+    ///
+    /// Options:
+    /// - region_number: specify region to flush.
+    /// - wait: Whether to wait until flush is done.
+    async fn flush(&self, region_number: Option<RegionNumber>, wait: Option<bool>) -> Result<()>;
+
+    /// Close the table.
+    async fn close(&self, _regions: &[RegionNumber]) -> Result<()>;
+
+    /// Get region stats in this table.
+    fn region_stats(&self) -> Result<Vec<RegionStat>>;
+
+    /// Return true if contains the region
+    fn contains_region(&self, _region: RegionNumber) -> Result<bool>;
+
+    /// Get statistics for this table, if available
+    fn statistics(&self) -> Option<TableStatistics>;
+
+    async fn compact(&self, region_number: Option<RegionNumber>, wait: Option<bool>) -> Result<()>;
+}
+```
+
+We can divide those methods into three categories from the perspective of functionality:
+
+|     Retrieve Metadata      | Manipulate Data |    Read Data     |
+| :------------------------: | :-------------: | :--------------: |
+|          `schema`          |    `insert`     | `scan_to_stream` |
+|        `table_info`        |     `alter`     |                  |
+|        `table_type`        |    `delete`     |                  |
+| `supports_filter_pushdown` |     `flush`     |                  |
+|       `region_stats`       |     `close`     |                  |
+|     `contains_region`      |    `compact`    |                  |
+|        `statistics`        |                 |                  |
+
+And considering most of the access to metadata happens in frontend, like route or query; and all the persisted data are stored in regions; while only the query engine needs to read data. We can divide the `Table` trait into three concepts:
+
+- struct `Table` provides metadata:
+    ```rust
+    impl Table {
+        /// Get a reference to the schema for this table
+        fn schema(&self) -> SchemaRef;
+
+        /// Get a reference to the table info.
+        fn table_info(&self) -> TableInfoRef;
+
+        /// Get the type of this table for metadata/catalog purposes.
+        fn table_type(&self) -> TableType;
+
+        /// Get statistics for this table, if available
+        fn statistics(&self) -> Option<TableStatistics>;
+
+        fn to_data_source(&self) -> DataSourceRef;
+    }
+    ```
+- Requests to region server
+  - `InsertRequest`
+  - `AlterRequest`
+  - `DeleteRequest`
+  - `FlushRequest`
+  - `CompactRequest`
+  - `CloseRequest`
+
+- trait `DataSource` provides data (`RecordBatch`)
+    ```rust
+    trait DataSource {
+        fn get_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream>;
+    }
+    ```
+
+## Use `Table`
+
+`Table` will only be used in frontend. It's constructed from the `OpenTableRequest` or `CreateTableRequest`.
+
+`Table` also provides a method `to_data_source` to generate a `DataSource` from itself. But this method is only for non-`TableType::Base` tables (i.e., `TableType::View` and `TableType::Temporary`) because `TableType::Base` table doesn't hold actual data itself. Its `DataSource` should be constructed from the `Region` directly (in other words, it's a remote query).
+
+And it requires some extra information to construct a `DataSource`, named `TableSourceProvider`:
+
+```rust
+type TableFactory = Arc<dyn Fn() -> DataSourceRef>;
+
+pub enum TableSourceProvider {
+    Base,
+    View(LogicalPlan),
+    Temporary(TableFactory),
+}
+```
+
+## Use `DataSource`
+
+`DataSource` will be adapted to the `TableProvider` from DataFusion that can be `scan()`ed in a `TableScan` plan.
+
+In frontend this is done in the planning phase. And datanode will have one implementation for `Region` to generate record batch stream.
+
+## Interact with RegionServer
+
+Previously, persisted state change operations were through the old `Table` trait, like said before. Now they will come from the action source, like the procedure or protocol handler directly to the region server. E.g., on alter table, the corresponding procedure will generate its `AlterRequest` and send it to regions. Or write request will be split in frontend handler, and sent to regions. `Table` only provides necessary metadata like route information if needed, but not the necessary part anymore.
+
+## Implement temporary table
+
+Temporary table is a special table that doesn't revolves to any persistent physical region. Examples are:
+- the `Numbers` table for testing, which produces a record batch that contains 0-100 integers.
+- tables in information schema. It is an interface for querying catalog's metadata. The contents are generated on the fly with information from `CatalogManager`. The `CatalogManager` can be held in `TableFactory`.
+- Function table that produces data generated by a formula or a function. Like something that always `sin(current_timestamp())`.
+
+## Relationship among those components
+
+Here is a diagram to show the relationship among those components, and how they interact with each other.
+
+```mermaid
+erDiagram
+    CatalogManager ||--|{ Table : manages
+    Table ||--|{ DataStream : generates
+    Table ||--|{ Region : routes
+    Region ||--|{ DataStream : implements
+    DataStream }|..|| QueryEngine : adapts-to
+    Procedure ||--|{ Region : requests
+    Protocol ||--|{ Region : writes
+    Protocol ||--|{ QueryEngine : queries
+```
+
+# Drawback
+This is a breaking change.

docs/rfcs/2023-08-13-metadata-txn.md

@@ -0,0 +1,90 @@
+---
+Feature Name: Update Metadata in single transaction
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/1715
+Date: 2023-08-13
+Author: "Feng Yangsen <fengys1996@gmail.com>, Xu Wenkang <wenymedia@gmail.com>"
+---
+
+# Summary
+Update Metadata in single transaction.
+
+# Motivation
+Currently, multiple transactions are involved during the procedure. This implementation is inefficient, and it's hard to make data consistent. Therefore, We can update multiple metadata in a single transaction.
+
+# Details 
+Now we have the following table metadata keys:
+
+**TableInfo** 
+```rust
+// __table_info/{table_id}
+pub struct TableInfoKey {
+    table_id: TableId,
+}
+
+pub struct TableInfoValue {
+    pub table_info: RawTableInfo,
+    version: u64,
+}
+```
+
+**TableRoute** 
+```rust
+// __table_route/{table_id}
+pub struct NextTableRouteKey {
+    table_id: TableId,
+}
+
+pub struct TableRoute {
+    pub region_routes: Vec<RegionRoute>,
+}
+```
+**DatanodeTable**
+```rust
+// __table_route/{datanode_id}/{table_id}
+pub struct DatanodeTableKey {
+    datanode_id: DatanodeId,
+    table_id: TableId,
+}
+
+pub struct DatanodeTableValue {
+    pub table_id: TableId,
+    pub regions: Vec<RegionNumber>,
+    version: u64,
+}
+```
+
+**TableNameKey**
+```rust
+// __table_name/{CatalogName}/{SchemaName}/{TableName}
+pub struct TableNameKey<'a> {
+    pub catalog: &'a str,
+    pub schema: &'a str,
+    pub table: &'a str,
+}
+
+pub struct TableNameValue {
+    table_id: TableId,
+}
+```
+
+These table metadata only updates in the following operations.
+
+## Region Failover
+It needs to update `TableRoute` key and `DatanodeTable` keys. If the `TableRoute` equals the Snapshot of `TableRoute` submitting the Failover task, then we can safely update these keys.
+
+After submitting Failover tasks to acquire locks for execution, the `TableRoute` may be updated by another task. After acquiring the lock, we can get the latest `TableRoute` again and then execute it if needed.
+
+## Create Table DDL
+Creates all of the above keys. `TableRoute`, `TableInfo`, should be empty.
+
+The **TableNameKey**'s lock will be held by the procedure framework.
+## Drop Table DDL
+
+`TableInfoKey` and `NextTableRouteKey` will be added with  `__removed-` prefix, and the other above keys will be deleted.  The transaction will not compare any keys.
+## Alter Table DDL
+
+1. Rename table, updates `TableInfo` and `TableName`. Compares `TableInfo`, and the new `TableNameKey` should be empty, and TableInfo should equal the Snapshot when submitting DDL.
+
+The old and new **TableNameKey**'s lock will be held by the procedure framework.
+
+2. Alter table, updates `TableInfo`. `TableInfo` should equal the Snapshot when submitting DDL.

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-05-03"
+channel = "nightly-2023-08-07"

scripts/fetch-dashboard-assets.sh

@@ -2,14 +2,14 @@
 
 # This script is used to download built dashboard assets from the "GreptimeTeam/dashboard" repository.
 
-set -e
+set -e -x
 
 declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
 declare -r ROOT_DIR=$(dirname ${SCRIPT_DIR})
 declare -r STATIC_DIR="$ROOT_DIR/src/servers/dashboard"
 OUT_DIR="${1:-$SCRIPT_DIR}"
 
-RELEASE_VERSION="$(cat $STATIC_DIR/VERSION)"
+RELEASE_VERSION="$(cat $STATIC_DIR/VERSION | tr -d '\t\r\n ')"
 
 echo "Downloading assets to dir: $OUT_DIR"
 cd $OUT_DIR

scripts/install.sh

@@ -61,7 +61,16 @@ if [ -n "${OS_TYPE}" ] && [ -n "${ARCH_TYPE}" ]; then
     fi
 
     echo "Downloading ${BIN}, OS: ${OS_TYPE}, Arch: ${ARCH_TYPE}, Version: ${VERSION}"
+    PACKAGE_NAME="${BIN}-${OS_TYPE}-${ARCH_TYPE}-${VERSION}.tar.gz"
 
-    wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
-    tar xvf ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && rm ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && echo "Run './${BIN} --help' to get started"
+    if [ -n "${PACKAGE_NAME}" ]; then
+      wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${PACKAGE_NAME}"
+
+      # Extract the binary and clean the rest.
+      tar xvf "${PACKAGE_NAME}" && \
+      mv "${PACKAGE_NAME%.tar.gz}/${BIN}" "${PWD}" && \
+      rm -r "${PACKAGE_NAME}" && \
+      rm -r "${PACKAGE_NAME%.tar.gz}" && \
+      echo "Run './${BIN} --help' to get started"
+    fi
 fi

src/api/Cargo.toml

@@ -5,10 +5,10 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-common-base = { path = "../common/base" }
-common-error = { path = "../common/error" }
-common-time = { path = "../common/time" }
-datatypes = { path = "../datatypes" }
+common-base = { workspace = true }
+common-error = { workspace = true }
+common-time = { workspace = true }
+datatypes = { workspace = true }
 greptime-proto.workspace = true
 prost.workspace = true
 snafu = { version = "0.7", features = ["backtraces"] }
@@ -16,3 +16,6 @@ tonic.workspace = true
 
 [build-dependencies]
 tonic-build = "0.9"
+
+[dev-dependencies]
+paste = "1.0"

src/api/src/error.rs

@@ -34,22 +34,14 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display(
-        "Failed to convert column default constraint, column: {}, source: {}",
-        column,
-        source
-    ))]
+    #[snafu(display("Failed to convert column default constraint, column: {}", column))]
     ConvertColumnDefaultConstraint {
         column: String,
         location: Location,
         source: datatypes::error::Error,
     },
 
-    #[snafu(display(
-        "Invalid column default constraint, column: {}, source: {}",
-        column,
-        source
-    ))]
+    #[snafu(display("Invalid column default constraint, column: {}", column))]
     InvalidColumnDefaultConstraint {
         column: String,
         location: Location,

src/api/src/v1/column_def.rs

@@ -12,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
+use std::collections::HashMap;
+
+use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, COMMENT_KEY};
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
@@ -20,7 +22,7 @@ use crate::helper::ColumnDataTypeWrapper;
 use crate::v1::ColumnDef;
 
 pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
-    let data_type = ColumnDataTypeWrapper::try_new(column_def.datatype)?;
+    let data_type = ColumnDataTypeWrapper::try_new(column_def.data_type)?;
 
     let constraint = if column_def.default_constraint.is_empty() {
         None
@@ -34,9 +36,17 @@ pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
         )
     };
 
-    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
-        .with_default_constraint(constraint)
-        .context(error::InvalidColumnDefaultConstraintSnafu {
-            column: &column_def.name,
-        })
+    let mut metadata = HashMap::new();
+    if !column_def.comment.is_empty() {
+        metadata.insert(COMMENT_KEY.to_string(), column_def.comment.clone());
+    }
+
+    Ok(
+        ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
+            .with_default_constraint(constraint)
+            .context(error::InvalidColumnDefaultConstraintSnafu {
+                column: &column_def.name,
+            })?
+            .with_metadata(metadata),
+    )
 }

src/auth/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "auth"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+default = []
+testing = []
+
+[dependencies]
+api.workspace = true
+async-trait.workspace = true
+common-error.workspace = true
+digest = "0.10"
+hex = { version = "0.4" }
+secrecy = { version = "0.8", features = ["serde", "alloc"] }
+sha1 = "0.10"
+snafu.workspace = true
+sql.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+common-test-util.workspace = true

src/auth/src/common.rs

@@ -0,0 +1,147 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use digest::Digest;
+use secrecy::SecretString;
+use sha1::Sha1;
+use snafu::{ensure, OptionExt};
+
+use crate::error::{IllegalParamSnafu, InvalidConfigSnafu, Result, UserPasswordMismatchSnafu};
+use crate::user_info::DefaultUserInfo;
+use crate::user_provider::static_user_provider::{StaticUserProvider, STATIC_USER_PROVIDER};
+use crate::{UserInfoRef, UserProviderRef};
+
+pub(crate) const DEFAULT_USERNAME: &str = "greptime";
+
+/// construct a [`UserInfo`](crate::user_info::UserInfo) impl with name
+/// use default username `greptime` if None is provided
+pub fn userinfo_by_name(username: Option<String>) -> UserInfoRef {
+    DefaultUserInfo::with_name(username.unwrap_or_else(|| DEFAULT_USERNAME.to_string()))
+}
+
+pub fn user_provider_from_option(opt: &String) -> Result<UserProviderRef> {
+    let (name, content) = opt.split_once(':').context(InvalidConfigSnafu {
+        value: opt.to_string(),
+        msg: "UserProviderOption must be in format `<option>:<value>`",
+    })?;
+    match name {
+        STATIC_USER_PROVIDER => {
+            let provider =
+                StaticUserProvider::try_from(content).map(|p| Arc::new(p) as UserProviderRef)?;
+            Ok(provider)
+        }
+        _ => InvalidConfigSnafu {
+            value: name.to_string(),
+            msg: "Invalid UserProviderOption",
+        }
+        .fail(),
+    }
+}
+
+type Username<'a> = &'a str;
+type HostOrIp<'a> = &'a str;
+
+#[derive(Debug, Clone)]
+pub enum Identity<'a> {
+    UserId(Username<'a>, Option<HostOrIp<'a>>),
+}
+
+pub type HashedPassword<'a> = &'a [u8];
+pub type Salt<'a> = &'a [u8];
+
+/// Authentication information sent by the client.
+pub enum Password<'a> {
+    PlainText(SecretString),
+    MysqlNativePassword(HashedPassword<'a>, Salt<'a>),
+    PgMD5(HashedPassword<'a>, Salt<'a>),
+}
+
+pub fn auth_mysql(
+    auth_data: HashedPassword,
+    salt: Salt,
+    username: &str,
+    save_pwd: &[u8],
+) -> Result<()> {
+    ensure!(
+        auth_data.len() == 20,
+        IllegalParamSnafu {
+            msg: "Illegal mysql password length"
+        }
+    );
+    // ref: https://github.com/mysql/mysql-server/blob/a246bad76b9271cb4333634e954040a970222e0a/sql/auth/password.cc#L62
+    let hash_stage_2 = double_sha1(save_pwd);
+    let tmp = sha1_two(salt, &hash_stage_2);
+    // xor auth_data and tmp
+    let mut xor_result = [0u8; 20];
+    for i in 0..20 {
+        xor_result[i] = auth_data[i] ^ tmp[i];
+    }
+    let candidate_stage_2 = sha1_one(&xor_result);
+    if candidate_stage_2 == hash_stage_2 {
+        Ok(())
+    } else {
+        UserPasswordMismatchSnafu {
+            username: username.to_string(),
+        }
+        .fail()
+    }
+}
+
+fn sha1_two(input_1: &[u8], input_2: &[u8]) -> Vec<u8> {
+    let mut hasher = Sha1::new();
+    hasher.update(input_1);
+    hasher.update(input_2);
+    hasher.finalize().to_vec()
+}
+
+fn sha1_one(data: &[u8]) -> Vec<u8> {
+    let mut hasher = Sha1::new();
+    hasher.update(data);
+    hasher.finalize().to_vec()
+}
+
+fn double_sha1(data: &[u8]) -> Vec<u8> {
+    sha1_one(&sha1_one(data))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_sha() {
+        let sha_1_answer: Vec<u8> = vec![
+            124, 74, 141, 9, 202, 55, 98, 175, 97, 229, 149, 32, 148, 61, 194, 100, 148, 248, 148,
+            27,
+        ];
+        let sha_1 = sha1_one("123456".as_bytes());
+        assert_eq!(sha_1, sha_1_answer);
+
+        let double_sha1_answer: Vec<u8> = vec![
+            107, 180, 131, 126, 183, 67, 41, 16, 94, 228, 86, 141, 218, 125, 198, 126, 210, 202,
+            42, 217,
+        ];
+        let double_sha1 = double_sha1("123456".as_bytes());
+        assert_eq!(double_sha1, double_sha1_answer);
+
+        let sha1_2_answer: Vec<u8> = vec![
+            132, 115, 215, 211, 99, 186, 164, 206, 168, 152, 217, 192, 117, 47, 240, 252, 142, 244,
+            37, 204,
+        ];
+        let sha1_2 = sha1_two("123456".as_bytes(), "654321".as_bytes());
+        assert_eq!(sha1_2, sha1_2_answer);
+    }
+}

src/auth/src/error.rs

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,83 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
-
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use secrecy::SecretString;
-use session::context::UserInfo;
-use snafu::{Location, OptionExt, Snafu};
-
-use crate::auth::user_provider::StaticUserProvider;
-
-pub mod user_provider;
-
-#[async_trait::async_trait]
-pub trait UserProvider: Send + Sync {
-    fn name(&self) -> &str;
-
-    /// [`authenticate`] checks whether a user is valid and allowed to access the database.
-    async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfo>;
-
-    /// [`authorize`] checks whether a connection request
-    /// from a certain user to a certain catalog/schema is legal.
-    /// This method should be called after [`authenticate`].
-    async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfo) -> Result<()>;
-
-    /// [`auth`] is a combination of [`authenticate`] and [`authorize`].
-    /// In most cases it's preferred for both convenience and performance.
-    async fn auth(
-        &self,
-        id: Identity<'_>,
-        password: Password<'_>,
-        catalog: &str,
-        schema: &str,
-    ) -> Result<UserInfo> {
-        let user_info = self.authenticate(id, password).await?;
-        self.authorize(catalog, schema, &user_info).await?;
-        Ok(user_info)
-    }
-}
-
-pub type UserProviderRef = Arc<dyn UserProvider>;
-
-type Username<'a> = &'a str;
-type HostOrIp<'a> = &'a str;
-
-#[derive(Debug, Clone)]
-pub enum Identity<'a> {
-    UserId(Username<'a>, Option<HostOrIp<'a>>),
-}
-
-pub type HashedPassword<'a> = &'a [u8];
-pub type Salt<'a> = &'a [u8];
-
-/// Authentication information sent by the client.
-pub enum Password<'a> {
-    PlainText(SecretString),
-    MysqlNativePassword(HashedPassword<'a>, Salt<'a>),
-    PgMD5(HashedPassword<'a>, Salt<'a>),
-}
-
-pub fn user_provider_from_option(opt: &String) -> Result<UserProviderRef> {
-    let (name, content) = opt.split_once(':').context(InvalidConfigSnafu {
-        value: opt.to_string(),
-        msg: "UserProviderOption must be in format `<option>:<value>`",
-    })?;
-    match name {
-        user_provider::STATIC_USER_PROVIDER => {
-            let provider =
-                StaticUserProvider::try_from(content).map(|p| Arc::new(p) as UserProviderRef)?;
-            Ok(provider)
-        }
-        _ => InvalidConfigSnafu {
-            value: name.to_string(),
-            msg: "Invalid UserProviderOption",
-        }
-        .fail(),
-    }
-}
+use snafu::{Location, Snafu};
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
@@ -102,13 +28,13 @@ pub enum Error {
     #[snafu(display("Internal state error: {}", msg))]
     InternalState { msg: String },
 
-    #[snafu(display("IO error, source: {}", source))]
+    #[snafu(display("IO error"))]
     Io {
         source: std::io::Error,
         location: Location,
     },
 
-    #[snafu(display("Auth failed, source: {}", source))]
+    #[snafu(display("Auth failed"))]
     AuthBackend {
         location: Location,
         source: BoxedError,
@@ -134,6 +60,9 @@ pub enum Error {
         schema: String,
         username: String,
     },
+
+    #[snafu(display("User is not authorized to perform this action"))]
+    PermissionDenied { location: Location },
 }
 
 impl ErrorExt for Error {
@@ -149,6 +78,7 @@ impl ErrorExt for Error {
             Error::UnsupportedPasswordType { .. } => StatusCode::UnsupportedPasswordType,
             Error::UserPasswordMismatch { .. } => StatusCode::UserPasswordMismatch,
             Error::AccessDenied { .. } => StatusCode::AccessDenied,
+            Error::PermissionDenied { .. } => StatusCode::PermissionDenied,
         }
     }
 

src/auth/src/lib.rs

@@ -0,0 +1,34 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod common;
+pub mod error;
+mod permission;
+mod user_info;
+mod user_provider;
+
+#[cfg(feature = "testing")]
+pub mod tests;
+
+pub use common::{
+    auth_mysql, user_provider_from_option, userinfo_by_name, HashedPassword, Identity, Password,
+};
+pub use permission::{PermissionChecker, PermissionReq, PermissionResp};
+pub use user_info::UserInfo;
+pub use user_provider::UserProvider;
+
+/// pub type alias
+pub type UserInfoRef = std::sync::Arc<dyn UserInfo>;
+pub type UserProviderRef = std::sync::Arc<dyn UserProvider>;
+pub type PermissionCheckerRef = std::sync::Arc<dyn PermissionChecker>;

src/auth/src/permission.rs

@@ -0,0 +1,64 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+
+use api::v1::greptime_request::Request;
+use sql::statements::statement::Statement;
+
+use crate::error::{PermissionDeniedSnafu, Result};
+use crate::{PermissionCheckerRef, UserInfoRef};
+
+#[derive(Debug, Clone)]
+pub enum PermissionReq<'a> {
+    GrpcRequest(&'a Request),
+    SqlStatement(&'a Statement),
+    PromQuery,
+    Opentsdb,
+    LineProtocol,
+    PromStoreWrite,
+    PromStoreRead,
+    Otlp,
+}
+
+#[derive(Debug)]
+pub enum PermissionResp {
+    Allow,
+    Reject,
+}
+
+pub trait PermissionChecker: Send + Sync {
+    fn check_permission(
+        &self,
+        user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> Result<PermissionResp>;
+}
+
+impl PermissionChecker for Option<&PermissionCheckerRef> {
+    fn check_permission(
+        &self,
+        user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> Result<PermissionResp> {
+        match self {
+            Some(checker) => match checker.check_permission(user_info, req) {
+                Ok(PermissionResp::Reject) => PermissionDeniedSnafu.fail(),
+                Ok(PermissionResp::Allow) => Ok(PermissionResp::Allow),
+                Err(e) => Err(e),
+            },
+            None => Ok(PermissionResp::Allow),
+        }
+    }
+}

src/auth/src/tests.rs

@@ -11,14 +11,14 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 use secrecy::ExposeSecret;
-use servers::auth::user_provider::auth_mysql;
-use servers::auth::{
-    AccessDeniedSnafu, Identity, Password, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu,
-    UserPasswordMismatchSnafu, UserProvider,
+
+use crate::error::{
+    AccessDeniedSnafu, Result, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu,
+    UserPasswordMismatchSnafu,
 };
-use session::context::UserInfo;
+use crate::user_info::DefaultUserInfo;
+use crate::{auth_mysql, Identity, Password, UserInfoRef, UserProvider};
 
 pub struct DatabaseAuthInfo<'a> {
     pub catalog: &'a str,
@@ -56,17 +56,13 @@ impl UserProvider for MockUserProvider {
         "mock_user_provider"
     }
 
-    async fn authenticate(
-        &self,
-        id: Identity<'_>,
-        password: Password<'_>,
-    ) -> servers::auth::Result<UserInfo> {
+    async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfoRef> {
         match id {
             Identity::UserId(username, _host) => match password {
                 Password::PlainText(password) => {
                     if username == "greptime" {
                         if password.expose_secret() == "greptime" {
-                            Ok(UserInfo::new("greptime"))
+                            Ok(DefaultUserInfo::with_name("greptime"))
                         } else {
                             UserPasswordMismatchSnafu {
                                 username: username.to_string(),
@@ -82,7 +78,7 @@ impl UserProvider for MockUserProvider {
                 }
                 Password::MysqlNativePassword(auth_data, salt) => {
                     auth_mysql(auth_data, salt, username, "greptime".as_bytes())
-                        .map(|_| UserInfo::new(username))
+                        .map(|_| DefaultUserInfo::with_name(username))
                 }
                 _ => UnsupportedPasswordTypeSnafu {
                     password_type: "mysql_native_password",
@@ -92,12 +88,7 @@ impl UserProvider for MockUserProvider {
         }
     }
 
-    async fn authorize(
-        &self,
-        catalog: &str,
-        schema: &str,
-        user_info: &UserInfo,
-    ) -> servers::auth::Result<()> {
+    async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfoRef) -> Result<()> {
         if catalog == self.catalog && schema == self.schema && user_info.username() == self.username
         {
             Ok(())
@@ -114,6 +105,8 @@ impl UserProvider for MockUserProvider {
 
 #[tokio::test]
 async fn test_auth_by_plain_text() {
+    use crate::error;
+
     let user_provider = MockUserProvider::default();
     assert_eq!("mock_user_provider", user_provider.name());
 
@@ -137,7 +130,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UnsupportedPasswordType { .. }
+        error::Error::UnsupportedPasswordType { .. }
     ));
 
     // auth failed, err: user not exist.
@@ -150,7 +143,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UserNotFound { .. }
+        error::Error::UserNotFound { .. }
     ));
 
     // auth failed, err: wrong password
@@ -163,7 +156,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UserPasswordMismatch { .. }
+        error::Error::UserPasswordMismatch { .. }
     ))
 }
 
@@ -176,8 +169,8 @@ async fn test_schema_validate() {
         username: "test_user",
     });
 
-    let right_user = UserInfo::new("test_user");
-    let wrong_user = UserInfo::default();
+    let right_user = DefaultUserInfo::with_name("test_user");
+    let wrong_user = DefaultUserInfo::with_name("greptime");
 
     // check catalog
     let re = validator

src/auth/src/user_info.rs

@@ -0,0 +1,47 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use crate::UserInfoRef;
+
+pub trait UserInfo: Debug + Sync + Send {
+    fn as_any(&self) -> &dyn Any;
+    fn username(&self) -> &str;
+}
+
+#[derive(Debug)]
+pub(crate) struct DefaultUserInfo {
+    username: String,
+}
+
+impl DefaultUserInfo {
+    pub(crate) fn with_name(username: impl Into<String>) -> UserInfoRef {
+        Arc::new(Self {
+            username: username.into(),
+        })
+    }
+}
+
+impl UserInfo for DefaultUserInfo {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn username(&self) -> &str {
+        self.username.as_str()
+    }
+}

src/auth/src/user_provider.rs

@@ -0,0 +1,46 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) mod static_user_provider;
+
+use crate::common::{Identity, Password};
+use crate::error::Result;
+use crate::UserInfoRef;
+
+#[async_trait::async_trait]
+pub trait UserProvider: Send + Sync {
+    fn name(&self) -> &str;
+
+    /// Checks whether a user is valid and allowed to access the database.
+    async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfoRef>;
+
+    /// Checks whether a connection request
+    /// from a certain user to a certain catalog/schema is legal.
+    /// This method should be called after [authenticate()](UserProvider::authenticate()).
+    async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfoRef) -> Result<()>;
+
+    /// Combination of [authenticate()](UserProvider::authenticate()) and [authorize()](UserProvider::authorize()).
+    /// In most cases it's preferred for both convenience and performance.
+    async fn auth(
+        &self,
+        id: Identity<'_>,
+        password: Password<'_>,
+        catalog: &str,
+        schema: &str,
+    ) -> Result<UserInfoRef> {
+        let user_info = self.authenticate(id, password).await?;
+        self.authorize(catalog, schema, &user_info).await?;
+        Ok(user_info)
+    }
+}

src/auth/src/user_provider/static_user_provider.rs

@@ -19,20 +19,17 @@ use std::io::BufRead;
 use std::path::Path;
 
 use async_trait::async_trait;
-use digest;
-use digest::Digest;
 use secrecy::ExposeSecret;
-use session::context::UserInfo;
-use sha1::Sha1;
 use snafu::{ensure, OptionExt, ResultExt};
 
-use crate::auth::{
-    Error, HashedPassword, Identity, IllegalParamSnafu, InvalidConfigSnafu, IoSnafu, Password,
-    Result, Salt, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu, UserPasswordMismatchSnafu,
-    UserProvider,
+use crate::error::{
+    Error, IllegalParamSnafu, InvalidConfigSnafu, IoSnafu, Result, UnsupportedPasswordTypeSnafu,
+    UserNotFoundSnafu, UserPasswordMismatchSnafu,
 };
+use crate::user_info::DefaultUserInfo;
+use crate::{auth_mysql, Identity, Password, UserInfoRef, UserProvider};
 
-pub const STATIC_USER_PROVIDER: &str = "static_user_provider";
+pub(crate) const STATIC_USER_PROVIDER: &str = "static_user_provider";
 
 impl TryFrom<&str> for StaticUserProvider {
     type Error = Error;
@@ -91,7 +88,7 @@ impl TryFrom<&str> for StaticUserProvider {
     }
 }
 
-pub struct StaticUserProvider {
+pub(crate) struct StaticUserProvider {
     users: HashMap<String, Vec<u8>>,
 }
 
@@ -105,7 +102,7 @@ impl UserProvider for StaticUserProvider {
         &self,
         input_id: Identity<'_>,
         input_pwd: Password<'_>,
-    ) -> Result<UserInfo> {
+    ) -> Result<UserInfoRef> {
         match input_id {
             Identity::UserId(username, _) => {
                 ensure!(
@@ -127,7 +124,7 @@ impl UserProvider for StaticUserProvider {
                             }
                         );
                         return if save_pwd == pwd.expose_secret().as_bytes() {
-                            Ok(UserInfo::new(username))
+                            Ok(DefaultUserInfo::with_name(username))
                         } else {
                             UserPasswordMismatchSnafu {
                                 username: username.to_string(),
@@ -136,14 +133,8 @@ impl UserProvider for StaticUserProvider {
                         };
                     }
                     Password::MysqlNativePassword(auth_data, salt) => {
-                        ensure!(
-                            auth_data.len() == 20,
-                            IllegalParamSnafu {
-                                msg: "Illegal MySQL native password format, length != 20"
-                            }
-                        );
                         auth_mysql(auth_data, salt, username, save_pwd)
-                            .map(|_| UserInfo::new(username))
+                            .map(|_| DefaultUserInfo::with_name(username))
                     }
                     Password::PgMD5(_, _) => UnsupportedPasswordTypeSnafu {
                         password_type: "pg_md5",
@@ -154,88 +145,28 @@ impl UserProvider for StaticUserProvider {
         }
     }
 
-    async fn authorize(&self, _catalog: &str, _schema: &str, _user_info: &UserInfo) -> Result<()> {
+    async fn authorize(
+        &self,
+        _catalog: &str,
+        _schema: &str,
+        _user_info: &UserInfoRef,
+    ) -> Result<()> {
         // default allow all
         Ok(())
     }
 }
 
-pub fn auth_mysql(
-    auth_data: HashedPassword,
-    salt: Salt,
-    username: &str,
-    save_pwd: &[u8],
-) -> Result<()> {
-    // ref: https://github.com/mysql/mysql-server/blob/a246bad76b9271cb4333634e954040a970222e0a/sql/auth/password.cc#L62
-    let hash_stage_2 = double_sha1(save_pwd);
-    let tmp = sha1_two(salt, &hash_stage_2);
-    // xor auth_data and tmp
-    let mut xor_result = [0u8; 20];
-    for i in 0..20 {
-        xor_result[i] = auth_data[i] ^ tmp[i];
-    }
-    let candidate_stage_2 = sha1_one(&xor_result);
-    if candidate_stage_2 == hash_stage_2 {
-        Ok(())
-    } else {
-        UserPasswordMismatchSnafu {
-            username: username.to_string(),
-        }
-        .fail()
-    }
-}
-
-fn sha1_two(input_1: &[u8], input_2: &[u8]) -> Vec<u8> {
-    let mut hasher = Sha1::new();
-    hasher.update(input_1);
-    hasher.update(input_2);
-    hasher.finalize().to_vec()
-}
-
-fn sha1_one(data: &[u8]) -> Vec<u8> {
-    let mut hasher = Sha1::new();
-    hasher.update(data);
-    hasher.finalize().to_vec()
-}
-
-fn double_sha1(data: &[u8]) -> Vec<u8> {
-    sha1_one(&sha1_one(data))
-}
-
 #[cfg(test)]
 pub mod test {
     use std::fs::File;
     use std::io::{LineWriter, Write};
 
     use common_test_util::temp_dir::create_temp_dir;
-    use session::context::UserInfo;
 
-    use crate::auth::user_provider::{double_sha1, sha1_one, sha1_two, StaticUserProvider};
-    use crate::auth::{Identity, Password, UserProvider};
-
-    #[test]
-    fn test_sha() {
-        let sha_1_answer: Vec<u8> = vec![
-            124, 74, 141, 9, 202, 55, 98, 175, 97, 229, 149, 32, 148, 61, 194, 100, 148, 248, 148,
-            27,
-        ];
-        let sha_1 = sha1_one("123456".as_bytes());
-        assert_eq!(sha_1, sha_1_answer);
-
-        let double_sha1_answer: Vec<u8> = vec![
-            107, 180, 131, 126, 183, 67, 41, 16, 94, 228, 86, 141, 218, 125, 198, 126, 210, 202,
-            42, 217,
-        ];
-        let double_sha1 = double_sha1("123456".as_bytes());
-        assert_eq!(double_sha1, double_sha1_answer);
-
-        let sha1_2_answer: Vec<u8> = vec![
-            132, 115, 215, 211, 99, 186, 164, 206, 168, 152, 217, 192, 117, 47, 240, 252, 142, 244,
-            37, 204,
-        ];
-        let sha1_2 = sha1_two("123456".as_bytes(), "654321".as_bytes());
-        assert_eq!(sha1_2, sha1_2_answer);
-    }
+    use crate::user_info::DefaultUserInfo;
+    use crate::user_provider::static_user_provider::StaticUserProvider;
+    use crate::user_provider::{Identity, Password};
+    use crate::UserProvider;
 
     async fn test_authenticate(provider: &dyn UserProvider, username: &str, password: &str) {
         let re = provider
@@ -249,9 +180,10 @@ pub mod test {
 
     #[tokio::test]
     async fn test_authorize() {
+        let user_info = DefaultUserInfo::with_name("root");
         let provider = StaticUserProvider::try_from("cmd:root=123456,admin=654321").unwrap();
         provider
-            .authorize("catalog", "schema", &UserInfo::new("root"))
+            .authorize("catalog", "schema", &user_info)
             .await
             .unwrap();
     }

src/auth/tests/mod.rs

@@ -0,0 +1,61 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(assert_matches)]
+use std::assert_matches::assert_matches;
+use std::sync::Arc;
+
+use api::v1::greptime_request::Request;
+use auth::error::Error::InternalState;
+use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq, PermissionResp, UserInfoRef};
+use sql::statements::show::{ShowDatabases, ShowKind};
+use sql::statements::statement::Statement;
+
+struct DummyPermissionChecker;
+
+impl PermissionChecker for DummyPermissionChecker {
+    fn check_permission(
+        &self,
+        _user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> auth::error::Result<PermissionResp> {
+        match req {
+            PermissionReq::GrpcRequest(_) => Ok(PermissionResp::Allow),
+            PermissionReq::SqlStatement(_) => Ok(PermissionResp::Reject),
+            _ => Err(InternalState {
+                msg: "testing".to_string(),
+            }),
+        }
+    }
+}
+
+#[test]
+fn test_permission_checker() {
+    let checker: PermissionCheckerRef = Arc::new(DummyPermissionChecker);
+
+    let grpc_result = checker.check_permission(
+        None,
+        PermissionReq::GrpcRequest(&Request::Query(Default::default())),
+    );
+    assert_matches!(grpc_result, Ok(PermissionResp::Allow));
+
+    let sql_result = checker.check_permission(
+        None,
+        PermissionReq::SqlStatement(&Statement::ShowDatabases(ShowDatabases::new(ShowKind::All))),
+    );
+    assert_matches!(sql_result, Ok(PermissionResp::Reject));
+
+    let err_result = checker.check_permission(None, PermissionReq::Opentsdb);
+    assert_matches!(err_result, Err(InternalState { msg }) if msg == "testing");
+}

src/catalog/Cargo.toml

@@ -8,45 +8,45 @@ license.workspace = true
 testing = []
 
 [dependencies]
-api = { path = "../api" }
+api = { workspace = true }
 arc-swap = "1.0"
 arrow-schema.workspace = true
 async-stream.workspace = true
 async-trait = "0.1"
-common-catalog = { path = "../common/catalog" }
-common-error = { path = "../common/error" }
-common-grpc = { path = "../common/grpc" }
-common-meta = { path = "../common/meta" }
-common-query = { path = "../common/query" }
-common-recordbatch = { path = "../common/recordbatch" }
-common-runtime = { path = "../common/runtime" }
-common-telemetry = { path = "../common/telemetry" }
-common-time = { path = "../common/time" }
+common-catalog = { workspace = true }
+common-error = { workspace = true }
+common-grpc = { workspace = true }
+common-meta = { workspace = true }
+common-query = { workspace = true }
+common-recordbatch = { workspace = true }
+common-runtime = { workspace = true }
+common-telemetry = { workspace = true }
+common-time = { workspace = true }
 dashmap = "5.4"
 datafusion.workspace = true
-datatypes = { path = "../datatypes" }
+datatypes = { workspace = true }
 futures = "0.3"
 futures-util.workspace = true
 lazy_static.workspace = true
-meta-client = { path = "../meta-client" }
+meta-client = { workspace = true }
 metrics.workspace = true
 moka = { version = "0.11", features = ["future"] }
 parking_lot = "0.12"
+partition.workspace = true
 regex.workspace = true
-serde = "1.0"
+serde.workspace = true
 serde_json = "1.0"
-session = { path = "../session" }
+session = { workspace = true }
 snafu = { version = "0.7", features = ["backtraces"] }
-store-api = { path = "../store-api" }
-table = { path = "../table" }
+store-api = { workspace = true }
+table = { workspace = true }
 tokio.workspace = true
 
 [dev-dependencies]
-catalog = { path = ".", features = ["testing"] }
-common-test-util = { path = "../common/test-util" }
+catalog = { workspace = true, features = ["testing"] }
 chrono.workspace = true
-log-store = { path = "../log-store" }
-mito = { path = "../mito", features = ["test"] }
-object-store = { path = "../object-store" }
-storage = { path = "../storage" }
+common-test-util = { workspace = true }
+log-store = { workspace = true }
+object-store = { workspace = true }
+storage = { workspace = true }
 tokio.workspace = true

src/catalog/src/error.rs

@@ -20,38 +20,43 @@ use common_error::status_code::StatusCode;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
 use snafu::{Location, Snafu};
+use table::metadata::TableId;
 use tokio::task::JoinError;
 
-use crate::DeregisterTableRequest;
-
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
-    #[snafu(display(
-        "Failed to re-compile script due to internal error, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to list catalogs"))]
+    ListCatalogs {
+        location: Location,
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to list {}'s schemas", catalog))]
+    ListSchemas {
+        location: Location,
+        catalog: String,
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to re-compile script due to internal error"))]
     CompileScriptInternal {
         location: Location,
         source: BoxedError,
     },
-    #[snafu(display("Failed to open system catalog table, source: {}", source))]
+    #[snafu(display("Failed to open system catalog table"))]
     OpenSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to create system catalog table, source: {}", source))]
+    #[snafu(display("Failed to create system catalog table"))]
     CreateSystemCatalog {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display(
-        "Failed to create table, table info: {}, source: {}",
-        table_info,
-        source
-    ))]
+    #[snafu(display("Failed to create table, table info: {}", table_info))]
     CreateTable {
         table_info: String,
         location: Location,
@@ -85,13 +90,13 @@ pub enum Error {
     #[snafu(display("Catalog value is not present"))]
     EmptyValue { location: Location },
 
-    #[snafu(display("Failed to deserialize value, source: {}", source))]
+    #[snafu(display("Failed to deserialize value"))]
     ValueDeserialize {
         source: serde_json::error::Error,
         location: Location,
     },
 
-    #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
+    #[snafu(display("Table engine not found: {}", engine_name))]
     TableEngineNotFound {
         engine_name: String,
         location: Location,
@@ -129,14 +134,14 @@ pub enum Error {
     #[snafu(display("Operation {} not supported", op))]
     NotSupported { op: String, location: Location },
 
-    #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
+    #[snafu(display("Failed to open table {table_id}"))]
     OpenTable {
-        table_info: String,
+        table_id: TableId,
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("Failed to open table in parallel, source: {}", source))]
+    #[snafu(display("Failed to open table in parallel"))]
     ParallelOpenTable { source: JoinError },
 
     #[snafu(display("Table not found while opening table, table info: {}", table_info))]
@@ -151,72 +156,52 @@ pub enum Error {
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to create recordbatch, source: {}", source))]
+    #[snafu(display("Failed to create recordbatch"))]
     CreateRecordBatch {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display(
-        "Failed to insert table creation record to system catalog, source: {}",
-        source
-    ))]
+    #[snafu(display("Failed to insert table creation record to system catalog"))]
     InsertCatalogRecord {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display(
-        "Failed to deregister table, request: {:?}, source: {}",
-        request,
-        source
-    ))]
-    DeregisterTable {
-        request: DeregisterTableRequest,
-        location: Location,
-        source: table::error::Error,
-    },
-
-    #[snafu(display("Illegal catalog manager state: {}", msg))]
-    IllegalManagerState { location: Location, msg: String },
-
-    #[snafu(display("Failed to scan system catalog table, source: {}", source))]
+    #[snafu(display("Failed to scan system catalog table"))]
     SystemCatalogTableScan {
         location: Location,
         source: table::error::Error,
     },
 
-    #[snafu(display("{source}"))]
+    #[snafu(display(""))]
     Internal {
         location: Location,
         source: BoxedError,
     },
 
-    #[snafu(display(
-        "Failed to upgrade weak catalog manager reference. location: {}",
-        location
-    ))]
+    #[snafu(display("Failed to upgrade weak catalog manager reference"))]
     UpgradeWeakCatalogManagerRef { location: Location },
 
-    #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
+    #[snafu(display("Failed to execute system catalog table scan"))]
     SystemCatalogTableScanExec {
         location: Location,
         source: common_query::error::Error,
     },
 
-    #[snafu(display("Cannot parse catalog value, source: {}", source))]
+    #[snafu(display("Cannot parse catalog value"))]
     InvalidCatalogValue {
         location: Location,
         source: common_catalog::error::Error,
     },
 
-    #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
+    #[snafu(display("Failed to perform metasrv operation"))]
     MetaSrv {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Invalid table info in catalog, source: {}", source))]
+    #[snafu(display("Invalid table info in catalog"))]
     InvalidTableInfoInCatalog {
         location: Location,
         source: datatypes::error::Error,
@@ -225,17 +210,14 @@ pub enum Error {
     #[snafu(display("Illegal access to catalog: {} and schema: {}", catalog, schema))]
     QueryAccessDenied { catalog: String, schema: String },
 
-    #[snafu(display("Invalid system table definition: {err_msg}"))]
-    InvalidSystemTableDef { err_msg: String, location: Location },
-
-    #[snafu(display("{}: {}", msg, source))]
+    #[snafu(display("msg: {}", msg))]
     Datafusion {
         msg: String,
         source: DataFusionError,
         location: Location,
     },
 
-    #[snafu(display("Table schema mismatch, source: {}", source))]
+    #[snafu(display("Table schema mismatch"))]
     TableSchemaMismatch {
         location: Location,
         source: table::error::Error,
@@ -244,7 +226,7 @@ pub enum Error {
     #[snafu(display("A generic error has occurred, msg: {}", msg))]
     Generic { msg: String, location: Location },
 
-    #[snafu(display("Table metadata manager error: {}", source))]
+    #[snafu(display("Table metadata manager error"))]
     TableMetadataManager {
         source: common_meta::error::Error,
         location: Location,
@@ -259,10 +241,8 @@ impl ErrorExt for Error {
             Error::InvalidKey { .. }
             | Error::SchemaNotFound { .. }
             | Error::TableNotFound { .. }
-            | Error::IllegalManagerState { .. }
             | Error::CatalogNotFound { .. }
             | Error::InvalidEntryType { .. }
-            | Error::InvalidSystemTableDef { .. }
             | Error::ParallelOpenTable { .. } => StatusCode::Unexpected,
 
             Error::SystemCatalog { .. }
@@ -284,12 +264,15 @@ impl ErrorExt for Error {
                 StatusCode::InvalidArguments
             }
 
+            Error::ListCatalogs { source, .. } | Error::ListSchemas { source, .. } => {
+                source.status_code()
+            }
+
             Error::OpenSystemCatalog { source, .. }
             | Error::CreateSystemCatalog { source, .. }
             | Error::InsertCatalogRecord { source, .. }
             | Error::OpenTable { source, .. }
             | Error::CreateTable { source, .. }
-            | Error::DeregisterTable { source, .. }
             | Error::TableSchemaMismatch { source, .. } => source.status_code(),
 
             Error::MetaSrv { source, .. } => source.status_code(),

src/catalog/src/information_schema.rs

@@ -15,27 +15,31 @@
 mod columns;
 mod tables;
 
-use std::any::Any;
+use std::collections::HashMap;
 use std::sync::{Arc, Weak};
 
-use async_trait::async_trait;
+use common_catalog::consts::INFORMATION_SCHEMA_NAME;
 use common_error::ext::BoxedError;
 use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
 use datatypes::schema::SchemaRef;
 use futures_util::StreamExt;
 use snafu::ResultExt;
-use store_api::storage::ScanRequest;
+use store_api::data_source::DataSource;
+use store_api::storage::{ScanRequest, TableId};
 use table::error::{SchemaConversionSnafu, TablesRecordBatchSnafu};
-use table::metadata::TableType;
-use table::{Result as TableResult, Table, TableRef};
+use table::metadata::{
+    FilterPushDownType, TableInfoBuilder, TableInfoRef, TableMetaBuilder, TableType,
+};
+use table::thin_table::{ThinTable, ThinTableAdapter};
+use table::TableRef;
 
 use self::columns::InformationSchemaColumns;
 use crate::error::Result;
 use crate::information_schema::tables::InformationSchemaTables;
 use crate::CatalogManager;
 
-const TABLES: &str = "tables";
-const COLUMNS: &str = "columns";
+pub const TABLES: &str = "tables";
+pub const COLUMNS: &str = "columns";
 
 pub struct InformationSchemaProvider {
     catalog_name: String,
@@ -49,89 +53,124 @@ impl InformationSchemaProvider {
             catalog_manager,
         }
     }
-}
 
-impl InformationSchemaProvider {
-    pub fn table(&self, name: &str) -> Result<Option<TableRef>> {
-        let stream_builder = match name.to_ascii_lowercase().as_ref() {
-            TABLES => Arc::new(InformationSchemaTables::new(
+    /// Build a map of [TableRef] in information schema.
+    /// Including `tables` and `columns`.
+    pub fn build(
+        catalog_name: String,
+        catalog_manager: Weak<dyn CatalogManager>,
+    ) -> HashMap<String, TableRef> {
+        let provider = Self::new(catalog_name, catalog_manager);
+
+        let mut schema = HashMap::new();
+        schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap());
+        schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap());
+        schema
+    }
+
+    pub fn table(&self, name: &str) -> Option<TableRef> {
+        self.information_table(name).map(|table| {
+            let table_info = Self::table_info(self.catalog_name.clone(), &table);
+            let filter_pushdown = FilterPushDownType::Unsupported;
+            let thin_table = ThinTable::new(table_info, filter_pushdown);
+
+            let data_source = Arc::new(InformationTableDataSource::new(table));
+            Arc::new(ThinTableAdapter::new(thin_table, data_source)) as _
+        })
+    }
+
+    fn information_table(&self, name: &str) -> Option<InformationTableRef> {
+        match name.to_ascii_lowercase().as_str() {
+            TABLES => Some(Arc::new(InformationSchemaTables::new(
                 self.catalog_name.clone(),
                 self.catalog_manager.clone(),
-            )) as _,
-            COLUMNS => Arc::new(InformationSchemaColumns::new(
+            )) as _),
+            COLUMNS => Some(Arc::new(InformationSchemaColumns::new(
                 self.catalog_name.clone(),
                 self.catalog_manager.clone(),
-            )) as _,
-            _ => {
-                return Ok(None);
-            }
-        };
+            )) as _),
+            _ => None,
+        }
+    }
 
-        Ok(Some(Arc::new(InformationTable::new(stream_builder))))
+    fn table_info(catalog_name: String, table: &InformationTableRef) -> TableInfoRef {
+        let table_meta = TableMetaBuilder::default()
+            .schema(table.schema())
+            .primary_key_indices(vec![])
+            .next_column_id(0)
+            .build()
+            .unwrap();
+        let table_info = TableInfoBuilder::default()
+            .table_id(table.table_id())
+            .name(table.table_name().to_owned())
+            .catalog_name(catalog_name)
+            .schema_name(INFORMATION_SCHEMA_NAME.to_owned())
+            .meta(table_meta)
+            .table_type(table.table_type())
+            .build()
+            .unwrap();
+        Arc::new(table_info)
     }
 }
 
-// TODO(ruihang): make it a more generic trait:
-// https://github.com/GreptimeTeam/greptimedb/pull/1639#discussion_r1205001903
-pub trait InformationStreamBuilder: Send + Sync {
-    fn to_stream(&self) -> Result<SendableRecordBatchStream>;
+trait InformationTable {
+    fn table_id(&self) -> TableId;
+
+    fn table_name(&self) -> &'static str;
 
     fn schema(&self) -> SchemaRef;
-}
 
-pub struct InformationTable {
-    stream_builder: Arc<dyn InformationStreamBuilder>,
-}
+    fn to_stream(&self) -> Result<SendableRecordBatchStream>;
 
-impl InformationTable {
-    pub fn new(stream_builder: Arc<dyn InformationStreamBuilder>) -> Self {
-        Self { stream_builder }
+    fn table_type(&self) -> TableType {
+        TableType::Temporary
     }
 }
 
-#[async_trait]
-impl Table for InformationTable {
-    fn as_any(&self) -> &dyn Any {
-        self
+type InformationTableRef = Arc<dyn InformationTable + Send + Sync>;
+
+struct InformationTableDataSource {
+    table: InformationTableRef,
+}
+
+impl InformationTableDataSource {
+    fn new(table: InformationTableRef) -> Self {
+        Self { table }
     }
 
-    fn schema(&self) -> SchemaRef {
-        self.stream_builder.schema()
+    fn try_project(&self, projection: &[usize]) -> std::result::Result<SchemaRef, BoxedError> {
+        let schema = self
+            .table
+            .schema()
+            .try_project(projection)
+            .context(SchemaConversionSnafu)
+            .map_err(BoxedError::new)?;
+        Ok(Arc::new(schema))
     }
+}
 
-    fn table_info(&self) -> table::metadata::TableInfoRef {
-        unreachable!("Should not call table_info() of InformationTable directly")
-    }
-
-    fn table_type(&self) -> table::metadata::TableType {
-        TableType::View
-    }
-
-    async fn scan_to_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
+impl DataSource for InformationTableDataSource {
+    fn get_stream(
+        &self,
+        request: ScanRequest,
+    ) -> std::result::Result<SendableRecordBatchStream, BoxedError> {
         let projection = request.projection;
-        let projected_schema = if let Some(projection) = &projection {
-            Arc::new(
-                self.schema()
-                    .try_project(projection)
-                    .context(SchemaConversionSnafu)?,
-            )
-        } else {
-            self.schema()
+        let projected_schema = match &projection {
+            Some(projection) => self.try_project(projection)?,
+            None => self.table.schema(),
         };
+
         let stream = self
-            .stream_builder
+            .table
             .to_stream()
             .map_err(BoxedError::new)
-            .context(TablesRecordBatchSnafu)?
-            .map(move |batch| {
-                batch.and_then(|batch| {
-                    if let Some(projection) = &projection {
-                        batch.try_project(projection)
-                    } else {
-                        Ok(batch)
-                    }
-                })
+            .context(TablesRecordBatchSnafu)
+            .map_err(BoxedError::new)?
+            .map(move |batch| match &projection {
+                Some(p) => batch.and_then(|b| b.try_project(p)),
+                None => batch,
             });
+
         let stream = RecordBatchStreamAdaptor {
             schema: projected_schema,
             stream: Box::pin(stream),

src/catalog/src/information_schema/columns.rs

@@ -16,7 +16,8 @@ use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::{
-    SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
+    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME, SEMANTIC_TYPE_FIELD,
+    SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
 };
 use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
@@ -30,8 +31,10 @@ use datatypes::scalars::ScalarVectorBuilder;
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::vectors::{StringVectorBuilder, VectorRef};
 use snafu::{OptionExt, ResultExt};
+use store_api::storage::TableId;
 
-use super::InformationStreamBuilder;
+use super::tables::InformationSchemaTables;
+use super::{InformationTable, COLUMNS, TABLES};
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
@@ -52,19 +55,22 @@ const SEMANTIC_TYPE: &str = "semantic_type";
 
 impl InformationSchemaColumns {
     pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
-        let schema = Arc::new(Schema::new(vec![
+        Self {
+            schema: Self::schema(),
+            catalog_name,
+            catalog_manager,
+        }
+    }
+
+    fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
             ColumnSchema::new(TABLE_CATALOG, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(TABLE_SCHEMA, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(TABLE_NAME, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(COLUMN_NAME, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(DATA_TYPE, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(SEMANTIC_TYPE, ConcreteDataType::string_datatype(), false),
-        ]));
-        Self {
-            schema,
-            catalog_name,
-            catalog_manager,
-        }
+        ]))
     }
 
     fn builder(&self) -> InformationSchemaColumnsBuilder {
@@ -76,7 +82,15 @@ impl InformationSchemaColumns {
     }
 }
 
-impl InformationStreamBuilder for InformationSchemaColumns {
+impl InformationTable for InformationSchemaColumns {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_COLUMNS_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        COLUMNS
+    }
+
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -144,7 +158,7 @@ impl InformationSchemaColumnsBuilder {
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
             if !catalog_manager
-                .schema_exist(&catalog_name, &schema_name)
+                .schema_exists(&catalog_name, &schema_name)
                 .await?
             {
                 continue;
@@ -153,9 +167,28 @@ impl InformationSchemaColumnsBuilder {
                 .table_names(&catalog_name, &schema_name)
                 .await?
             {
-                let Some(table) = catalog_manager.table(&catalog_name, &schema_name, &table_name).await? else { continue };
-                let keys = &table.table_info().meta.primary_key_indices;
-                let schema = table.schema();
+                let (keys, schema) = if let Some(table) = catalog_manager
+                    .table(&catalog_name, &schema_name, &table_name)
+                    .await?
+                {
+                    let keys = &table.table_info().meta.primary_key_indices;
+                    let schema = table.schema();
+                    (keys.clone(), schema)
+                } else {
+                    // TODO: this specific branch is only a workaround for FrontendCatalogManager.
+                    if schema_name == INFORMATION_SCHEMA_NAME {
+                        if table_name == COLUMNS {
+                            (vec![], InformationSchemaColumns::schema())
+                        } else if table_name == TABLES {
+                            (vec![], InformationSchemaTables::schema())
+                        } else {
+                            continue;
+                        }
+                    } else {
+                        continue;
+                    }
+                };
+
                 for (idx, column) in schema.column_schemas().iter().enumerate() {
                     let semantic_type = if column.is_time_index() {
                         SEMANTIC_TYPE_TIME_INDEX

src/catalog/src/information_schema/tables.rs

@@ -15,7 +15,10 @@
 use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
-use common_catalog::consts::INFORMATION_SCHEMA_NAME;
+use common_catalog::consts::{
+    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME,
+    INFORMATION_SCHEMA_TABLES_TABLE_ID,
+};
 use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
@@ -27,12 +30,14 @@ use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder};
 use snafu::{OptionExt, ResultExt};
+use store_api::storage::TableId;
 use table::metadata::TableType;
 
+use super::{COLUMNS, TABLES};
 use crate::error::{
     CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
 };
-use crate::information_schema::InformationStreamBuilder;
+use crate::information_schema::InformationTable;
 use crate::CatalogManager;
 
 pub(super) struct InformationSchemaTables {
@@ -43,19 +48,22 @@ pub(super) struct InformationSchemaTables {
 
 impl InformationSchemaTables {
     pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
-        let schema = Arc::new(Schema::new(vec![
+        Self {
+            schema: Self::schema(),
+            catalog_name,
+            catalog_manager,
+        }
+    }
+
+    pub(crate) fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
             ColumnSchema::new("table_catalog", ConcreteDataType::string_datatype(), false),
             ColumnSchema::new("table_schema", ConcreteDataType::string_datatype(), false),
             ColumnSchema::new("table_name", ConcreteDataType::string_datatype(), false),
             ColumnSchema::new("table_type", ConcreteDataType::string_datatype(), false),
             ColumnSchema::new("table_id", ConcreteDataType::uint32_datatype(), true),
             ColumnSchema::new("engine", ConcreteDataType::string_datatype(), true),
-        ]));
-        Self {
-            schema,
-            catalog_name,
-            catalog_manager,
-        }
+        ]))
     }
 
     fn builder(&self) -> InformationSchemaTablesBuilder {
@@ -67,7 +75,15 @@ impl InformationSchemaTables {
     }
 }
 
-impl InformationStreamBuilder for InformationSchemaTables {
+impl InformationTable for InformationSchemaTables {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_TABLES_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        TABLES
+    }
+
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -137,11 +153,8 @@ impl InformationSchemaTablesBuilder {
             .context(UpgradeWeakCatalogManagerRefSnafu)?;
 
         for schema_name in catalog_manager.schema_names(&catalog_name).await? {
-            if schema_name == INFORMATION_SCHEMA_NAME {
-                continue;
-            }
             if !catalog_manager
-                .schema_exist(&catalog_name, &schema_name)
+                .schema_exists(&catalog_name, &schema_name)
                 .await?
             {
                 continue;
@@ -151,16 +164,43 @@ impl InformationSchemaTablesBuilder {
                 .table_names(&catalog_name, &schema_name)
                 .await?
             {
-                let Some(table) = catalog_manager.table(&catalog_name, &schema_name, &table_name).await? else { continue };
-                let table_info = table.table_info();
-                self.add_table(
-                    &catalog_name,
-                    &schema_name,
-                    &table_name,
-                    table.table_type(),
-                    Some(table_info.ident.table_id),
-                    Some(&table_info.meta.engine),
-                );
+                if let Some(table) = catalog_manager
+                    .table(&catalog_name, &schema_name, &table_name)
+                    .await?
+                {
+                    let table_info = table.table_info();
+                    self.add_table(
+                        &catalog_name,
+                        &schema_name,
+                        &table_name,
+                        table.table_type(),
+                        Some(table_info.ident.table_id),
+                        Some(&table_info.meta.engine),
+                    );
+                } else {
+                    // TODO: this specific branch is only a workaround for FrontendCatalogManager.
+                    if schema_name == INFORMATION_SCHEMA_NAME {
+                        if table_name == COLUMNS {
+                            self.add_table(
+                                &catalog_name,
+                                &schema_name,
+                                &table_name,
+                                TableType::Temporary,
+                                Some(INFORMATION_SCHEMA_COLUMNS_TABLE_ID),
+                                None,
+                            );
+                        } else if table_name == TABLES {
+                            self.add_table(
+                                &catalog_name,
+                                &schema_name,
+                                &table_name,
+                                TableType::Temporary,
+                                Some(INFORMATION_SCHEMA_TABLES_TABLE_ID),
+                                None,
+                            );
+                        }
+                    }
+                };
             }
         }
 

src/catalog/src/kvbackend.rs

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod immutable;
-mod procedure;
-#[cfg(test)]
-mod tests;
+pub use client::{CachedMetaKvBackend, MetaKvBackend};
 
-use table::metadata::TableVersion;
+mod client;
+mod manager;
 
-const INIT_TABLE_VERSION: TableVersion = 0;
+#[cfg(feature = "testing")]
+pub mod mock;
+pub use manager::KvBackendCatalogManager;

src/catalog/src/kvbackend/client.rs

@@ -18,8 +18,9 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use common_error::ext::BoxedError;
+use common_meta::cache_invalidator::KvCacheInvalidator;
 use common_meta::error::Error::{CacheNotGet, GetKvCache};
-use common_meta::error::{CacheNotGetSnafu, Error, MetaSrvSnafu, Result};
+use common_meta::error::{CacheNotGetSnafu, Error, ExternalSnafu, Result};
 use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
 use common_meta::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
@@ -28,12 +29,11 @@ use common_meta::rpc::store::{
     RangeRequest, RangeResponse,
 };
 use common_meta::rpc::KeyValue;
-use common_telemetry::timer;
+use common_telemetry::{debug, timer};
 use meta_client::client::MetaClient;
 use moka::future::{Cache, CacheBuilder};
 use snafu::{OptionExt, ResultExt};
 
-use super::KvCacheInvalidator;
 use crate::metrics::{METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET};
 
 const CACHE_MAX_CAPACITY: u64 = 10000;
@@ -197,7 +197,8 @@ impl KvBackend for CachedMetaKvBackend {
 #[async_trait::async_trait]
 impl KvCacheInvalidator for CachedMetaKvBackend {
     async fn invalidate_key(&self, key: &[u8]) {
-        self.cache.invalidate(key).await
+        self.cache.invalidate(key).await;
+        debug!("invalidated cache key: {}", String::from_utf8_lossy(key));
     }
 }
 
@@ -251,7 +252,7 @@ impl KvBackend for MetaKvBackend {
             .range(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
@@ -260,7 +261,7 @@ impl KvBackend for MetaKvBackend {
             .range(RangeRequest::new().with_key(key))
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)?;
+            .context(ExternalSnafu)?;
         Ok(response.take_kvs().get_mut(0).map(|kv| KeyValue {
             key: kv.take_key(),
             value: kv.take_value(),
@@ -272,7 +273,7 @@ impl KvBackend for MetaKvBackend {
             .batch_put(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn put(&self, req: PutRequest) -> Result<PutResponse> {
@@ -280,7 +281,7 @@ impl KvBackend for MetaKvBackend {
             .put(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
@@ -288,7 +289,7 @@ impl KvBackend for MetaKvBackend {
             .delete_range(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
@@ -296,7 +297,7 @@ impl KvBackend for MetaKvBackend {
             .batch_delete(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
@@ -304,7 +305,7 @@ impl KvBackend for MetaKvBackend {
             .batch_get(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn compare_and_put(
@@ -315,7 +316,7 @@ impl KvBackend for MetaKvBackend {
             .compare_and_put(request)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
@@ -323,7 +324,7 @@ impl KvBackend for MetaKvBackend {
             .move_value(req)
             .await
             .map_err(BoxedError::new)
-            .context(MetaSrvSnafu)
+            .context(ExternalSnafu)
     }
 
     fn as_any(&self) -> &dyn Any {

src/catalog/src/kvbackend/manager.rs

@@ -0,0 +1,301 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::collections::BTreeSet;
+use std::sync::{Arc, Weak};
+
+use common_catalog::consts::{DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, NUMBERS_TABLE_ID};
+use common_error::ext::BoxedError;
+use common_meta::cache_invalidator::{
+    CacheInvalidator, Context, KvCacheInvalidatorRef, TableMetadataCacheInvalidator,
+};
+use common_meta::datanode_manager::DatanodeManagerRef;
+use common_meta::error::Result as MetaResult;
+use common_meta::key::catalog_name::CatalogNameKey;
+use common_meta::key::schema_name::SchemaNameKey;
+use common_meta::key::table_name::TableNameKey;
+use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
+use common_meta::kv_backend::KvBackendRef;
+use common_meta::table_name::TableName;
+use futures_util::TryStreamExt;
+use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
+use snafu::prelude::*;
+use table::dist_table::DistTable;
+use table::metadata::TableId;
+use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+use table::TableRef;
+
+use crate::error::{
+    self as catalog_err, ListCatalogsSnafu, ListSchemasSnafu, Result as CatalogResult,
+    TableMetadataManagerSnafu,
+};
+use crate::information_schema::{InformationSchemaProvider, COLUMNS, TABLES};
+use crate::CatalogManager;
+
+/// Access all existing catalog, schema and tables.
+///
+/// The result comes from two source, all the user tables are presented in
+/// a kv-backend which persists the metadata of a table. And system tables
+/// comes from `SystemCatalog`, which is static and read-only.
+#[derive(Clone)]
+pub struct KvBackendCatalogManager {
+    // TODO(LFC): Maybe use a real implementation for Standalone mode.
+    // Now we use `NoopKvCacheInvalidator` for Standalone mode. In Standalone mode, the KV backend
+    // is implemented by RaftEngine. Maybe we need a cache for it?
+    table_metadata_cache_invalidator: TableMetadataCacheInvalidator,
+    partition_manager: PartitionRuleManagerRef,
+    table_metadata_manager: TableMetadataManagerRef,
+    datanode_manager: DatanodeManagerRef,
+    /// A sub-CatalogManager that handles system tables
+    system_catalog: SystemCatalog,
+}
+
+#[async_trait::async_trait]
+impl CacheInvalidator for KvBackendCatalogManager {
+    async fn invalidate_table_name(&self, ctx: &Context, table_name: TableName) -> MetaResult<()> {
+        self.table_metadata_cache_invalidator
+            .invalidate_table_name(ctx, table_name)
+            .await
+    }
+
+    async fn invalidate_table_id(&self, ctx: &Context, table_id: TableId) -> MetaResult<()> {
+        self.table_metadata_cache_invalidator
+            .invalidate_table_id(ctx, table_id)
+            .await
+    }
+}
+
+impl KvBackendCatalogManager {
+    pub fn new(
+        backend: KvBackendRef,
+        backend_cache_invalidator: KvCacheInvalidatorRef,
+        datanode_manager: DatanodeManagerRef,
+    ) -> Arc<Self> {
+        Arc::new_cyclic(|me| Self {
+            partition_manager: Arc::new(PartitionRuleManager::new(backend.clone())),
+            table_metadata_manager: Arc::new(TableMetadataManager::new(backend)),
+            table_metadata_cache_invalidator: TableMetadataCacheInvalidator::new(
+                backend_cache_invalidator.clone(),
+            ),
+            datanode_manager,
+            system_catalog: SystemCatalog {
+                catalog_manager: me.clone(),
+            },
+        })
+    }
+
+    pub fn partition_manager(&self) -> PartitionRuleManagerRef {
+        self.partition_manager.clone()
+    }
+
+    pub fn table_metadata_manager_ref(&self) -> &TableMetadataManagerRef {
+        &self.table_metadata_manager
+    }
+
+    pub fn datanode_manager(&self) -> DatanodeManagerRef {
+        self.datanode_manager.clone()
+    }
+
+    pub async fn invalidate_schema(&self, catalog: &str, schema: &str) {
+        self.table_metadata_cache_invalidator
+            .invalidate_schema(catalog, schema)
+            .await
+    }
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for KvBackendCatalogManager {
+    async fn catalog_names(&self) -> CatalogResult<Vec<String>> {
+        let stream = self
+            .table_metadata_manager
+            .catalog_manager()
+            .catalog_names()
+            .await;
+
+        let keys = stream
+            .try_collect::<Vec<_>>()
+            .await
+            .map_err(BoxedError::new)
+            .context(ListCatalogsSnafu)?;
+
+        Ok(keys)
+    }
+
+    async fn schema_names(&self, catalog: &str) -> CatalogResult<Vec<String>> {
+        let stream = self
+            .table_metadata_manager
+            .schema_manager()
+            .schema_names(catalog)
+            .await;
+        let mut keys = stream
+            .try_collect::<BTreeSet<_>>()
+            .await
+            .map_err(BoxedError::new)
+            .context(ListSchemasSnafu { catalog })?
+            .into_iter()
+            .collect::<Vec<_>>();
+
+        keys.extend_from_slice(&self.system_catalog.schema_names());
+
+        Ok(keys)
+    }
+
+    async fn table_names(&self, catalog: &str, schema: &str) -> CatalogResult<Vec<String>> {
+        let mut tables = self
+            .table_metadata_manager
+            .table_name_manager()
+            .tables(catalog, schema)
+            .await
+            .context(TableMetadataManagerSnafu)?
+            .into_iter()
+            .map(|(k, _)| k)
+            .collect::<Vec<String>>();
+        tables.extend_from_slice(&self.system_catalog.table_names(schema));
+
+        Ok(tables)
+    }
+
+    async fn catalog_exists(&self, catalog: &str) -> CatalogResult<bool> {
+        self.table_metadata_manager
+            .catalog_manager()
+            .exists(CatalogNameKey::new(catalog))
+            .await
+            .context(TableMetadataManagerSnafu)
+    }
+
+    async fn schema_exists(&self, catalog: &str, schema: &str) -> CatalogResult<bool> {
+        if self.system_catalog.schema_exist(schema) {
+            return Ok(true);
+        }
+
+        self.table_metadata_manager
+            .schema_manager()
+            .exists(SchemaNameKey::new(catalog, schema))
+            .await
+            .context(TableMetadataManagerSnafu)
+    }
+
+    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> CatalogResult<bool> {
+        if self.system_catalog.table_exist(schema, table) {
+            return Ok(true);
+        }
+
+        let key = TableNameKey::new(catalog, schema, table);
+        self.table_metadata_manager
+            .table_name_manager()
+            .get(key)
+            .await
+            .context(TableMetadataManagerSnafu)
+            .map(|x| x.is_some())
+    }
+
+    async fn table(
+        &self,
+        catalog: &str,
+        schema: &str,
+        table_name: &str,
+    ) -> CatalogResult<Option<TableRef>> {
+        if let Some(table) = self.system_catalog.table(catalog, schema, table_name) {
+            return Ok(Some(table));
+        }
+
+        let key = TableNameKey::new(catalog, schema, table_name);
+        let Some(table_name_value) = self
+            .table_metadata_manager
+            .table_name_manager()
+            .get(key)
+            .await
+            .context(TableMetadataManagerSnafu)?
+        else {
+            return Ok(None);
+        };
+        let table_id = table_name_value.table_id();
+
+        let Some(table_info_value) = self
+            .table_metadata_manager
+            .table_info_manager()
+            .get(table_id)
+            .await
+            .context(TableMetadataManagerSnafu)?
+        else {
+            return Ok(None);
+        };
+        let table_info = Arc::new(
+            table_info_value
+                .table_info
+                .try_into()
+                .context(catalog_err::InvalidTableInfoInCatalogSnafu)?,
+        );
+        Ok(Some(DistTable::table(table_info)))
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+// TODO: This struct can hold a static map of all system tables when
+// the upper layer (e.g., procedure) can inform the catalog manager
+// a new catalog is created.
+/// Existing system tables:
+/// - public.numbers
+/// - information_schema.tables
+/// - information_schema.columns
+#[derive(Clone)]
+struct SystemCatalog {
+    catalog_manager: Weak<KvBackendCatalogManager>,
+}
+
+impl SystemCatalog {
+    fn schema_names(&self) -> Vec<String> {
+        vec![INFORMATION_SCHEMA_NAME.to_string()]
+    }
+
+    fn table_names(&self, schema: &str) -> Vec<String> {
+        if schema == INFORMATION_SCHEMA_NAME {
+            vec![TABLES.to_string(), COLUMNS.to_string()]
+        } else if schema == DEFAULT_SCHEMA_NAME {
+            vec![NUMBERS_TABLE_NAME.to_string()]
+        } else {
+            vec![]
+        }
+    }
+
+    fn schema_exist(&self, schema: &str) -> bool {
+        schema == INFORMATION_SCHEMA_NAME
+    }
+
+    fn table_exist(&self, schema: &str, table: &str) -> bool {
+        if schema == INFORMATION_SCHEMA_NAME {
+            table == TABLES || table == COLUMNS
+        } else if schema == DEFAULT_SCHEMA_NAME {
+            table == NUMBERS_TABLE_NAME
+        } else {
+            false
+        }
+    }
+
+    fn table(&self, catalog: &str, schema: &str, table_name: &str) -> Option<TableRef> {
+        if schema == INFORMATION_SCHEMA_NAME {
+            let information_schema_provider =
+                InformationSchemaProvider::new(catalog.to_string(), self.catalog_manager.clone());
+            information_schema_provider.table(table_name)
+        } else if schema == DEFAULT_SCHEMA_NAME && table_name == NUMBERS_TABLE_NAME {
+            Some(NumbersTable::table(NUMBERS_TABLE_ID))
+        } else {
+            None
+        }
+    }
+}

src/catalog/src/kvbackend/mock.rs

@@ -23,6 +23,7 @@ use table::engine::{CloseTableResult, EngineContext, TableEngine};
 use table::metadata::TableId;
 use table::requests::{
     AlterTableRequest, CloseTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest,
+    TruncateTableRequest,
 };
 use table::test_util::MemTable;
 use table::TableRef;
@@ -54,14 +55,14 @@ impl TableEngine for MockTableEngine {
 
         let data = vec![Arc::new(StringVector::from(vec!["a", "b", "c"])) as _];
         let record_batch = RecordBatch::new(schema, data).unwrap();
-        let table: TableRef = Arc::new(MemTable::new_with_catalog(
+        let table = MemTable::new_with_catalog(
             &request.table_name,
             record_batch,
             table_id,
             request.catalog_name,
             request.schema_name,
             vec![0],
-        )) as Arc<_>;
+        );
 
         let mut tables = self.tables.write().unwrap();
         let _ = tables.insert(table_id, table.clone() as TableRef);
@@ -116,4 +117,12 @@ impl TableEngine for MockTableEngine {
     async fn close(&self) -> table::Result<()> {
         Ok(())
     }
+
+    async fn truncate_table(
+        &self,
+        _ctx: &EngineContext,
+        _request: TruncateTableRequest,
+    ) -> table::Result<bool> {
+        Ok(true)
+    }
 }

src/catalog/src/lib.rs

@@ -17,79 +17,38 @@
 #![feature(try_blocks)]
 
 use std::any::Any;
-use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use api::v1::meta::{RegionStat, TableIdent, TableName};
-use common_telemetry::{info, warn};
-use snafu::ResultExt;
-use table::engine::{EngineContext, TableEngineRef};
+use futures::future::BoxFuture;
 use table::metadata::TableId;
 use table::requests::CreateTableRequest;
 use table::TableRef;
 
-use crate::error::{CreateTableSnafu, Result};
+use crate::error::Result;
 
 pub mod error;
 pub mod information_schema;
-pub mod local;
+pub mod kvbackend;
+pub mod memory;
 mod metrics;
-pub mod remote;
-pub mod system;
 pub mod table_source;
-pub mod tables;
 
 #[async_trait::async_trait]
 pub trait CatalogManager: Send + Sync {
     fn as_any(&self) -> &dyn Any;
 
-    /// Starts a catalog manager.
-    async fn start(&self) -> Result<()>;
-
-    /// Registers a catalog to catalog manager, returns whether the catalog exist before.
-    async fn register_catalog(&self, name: String) -> Result<bool>;
-
-    /// Register a schema with catalog name and schema name. Retuens whether the
-    /// schema registered.
-    ///
-    /// # Errors
-    ///
-    /// This method will/should fail if catalog not exist
-    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool>;
-
-    /// Deregisters a database within given catalog/schema to catalog manager
-    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool>;
-
-    /// Registers a table within given catalog/schema to catalog manager,
-    /// returns whether the table registered.
-    ///
-    /// # Errors
-    ///
-    /// This method will/should fail if catalog or schema not exist
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
-
-    /// Deregisters a table within given catalog/schema to catalog manager
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()>;
-
-    /// Rename a table to [RenameTableRequest::new_table_name], returns whether the table is renamed.
-    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool>;
-
-    /// Register a system table, should be called before starting the manager.
-    async fn register_system_table(&self, request: RegisterSystemTableRequest)
-        -> error::Result<()>;
-
     async fn catalog_names(&self) -> Result<Vec<String>>;
 
     async fn schema_names(&self, catalog: &str) -> Result<Vec<String>>;
 
     async fn table_names(&self, catalog: &str, schema: &str) -> Result<Vec<String>>;
 
-    async fn catalog_exist(&self, catalog: &str) -> Result<bool>;
+    async fn catalog_exists(&self, catalog: &str) -> Result<bool>;
 
-    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool>;
+    async fn schema_exists(&self, catalog: &str, schema: &str) -> Result<bool>;
 
-    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
+    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
 
     /// Returns the table by catalog, schema and table name.
     async fn table(
@@ -103,7 +62,8 @@ pub trait CatalogManager: Send + Sync {
 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
 
 /// Hook called after system table opening.
-pub type OpenSystemTableHook = Arc<dyn Fn(TableRef) -> Result<()> + Send + Sync>;
+pub type OpenSystemTableHook =
+    Box<dyn Fn(TableRef) -> BoxFuture<'static, Result<()>> + Send + Sync>;
 
 /// Register system table request:
 /// - When system table is already created and registered, the hook will be called
@@ -162,102 +122,3 @@ pub struct RegisterSchemaRequest {
     pub catalog: String,
     pub schema: String,
 }
-
-pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
-    manager: &'a M,
-    engine: TableEngineRef,
-    sys_table_requests: &'a mut Vec<RegisterSystemTableRequest>,
-) -> Result<()> {
-    for req in sys_table_requests.drain(..) {
-        let catalog_name = &req.create_table_request.catalog_name;
-        let schema_name = &req.create_table_request.schema_name;
-        let table_name = &req.create_table_request.table_name;
-        let table_id = req.create_table_request.id;
-
-        let table = manager.table(catalog_name, schema_name, table_name).await?;
-        let table = if let Some(table) = table {
-            table
-        } else {
-            let table = engine
-                .create_table(&EngineContext::default(), req.create_table_request.clone())
-                .await
-                .with_context(|_| CreateTableSnafu {
-                    table_info: common_catalog::format_full_table_name(
-                        catalog_name,
-                        schema_name,
-                        table_name,
-                    ),
-                })?;
-            let _ = manager
-                .register_table(RegisterTableRequest {
-                    catalog: catalog_name.clone(),
-                    schema: schema_name.clone(),
-                    table_name: table_name.clone(),
-                    table_id,
-                    table: table.clone(),
-                })
-                .await?;
-            info!("Created and registered system table: {table_name}");
-            table
-        };
-        if let Some(hook) = req.open_hook {
-            (hook)(table)?;
-        }
-    }
-    Ok(())
-}
-
-/// The stat of regions in the datanode node.
-/// The number of regions can be got from len of vec.
-///
-/// Ignores any errors occurred during iterating regions. The intention of this method is to
-/// collect region stats that will be carried in Datanode's heartbeat to Metasrv, so it's a
-/// "try our best" job.
-pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<RegionStat>) {
-    let mut region_number: u64 = 0;
-    let mut region_stats = Vec::new();
-
-    let Ok(catalog_names) = catalog_manager.catalog_names().await else { return (region_number, region_stats) };
-    for catalog_name in catalog_names {
-        let Ok(schema_names) = catalog_manager.schema_names(&catalog_name).await else { continue };
-        for schema_name in schema_names {
-            let Ok(table_names) = catalog_manager.table_names(&catalog_name,&schema_name).await else { continue };
-            for table_name in table_names {
-                let Ok(Some(table)) = catalog_manager.table(&catalog_name, &schema_name, &table_name).await else { continue };
-
-                let table_info = table.table_info();
-                let region_numbers = &table_info.meta.region_numbers;
-                region_number += region_numbers.len() as u64;
-
-                let engine = &table_info.meta.engine;
-                let table_id = table_info.ident.table_id;
-
-                match table.region_stats() {
-                    Ok(stats) => {
-                        let stats = stats.into_iter().map(|stat| RegionStat {
-                            region_id: stat.region_id,
-                            table_ident: Some(TableIdent {
-                                table_id,
-                                table_name: Some(TableName {
-                                    catalog_name: catalog_name.clone(),
-                                    schema_name: schema_name.clone(),
-                                    table_name: table_name.clone(),
-                                }),
-                                engine: engine.clone(),
-                            }),
-                            approximate_bytes: stat.disk_usage_bytes as i64,
-                            attrs: HashMap::from([("engine_name".to_owned(), engine.clone())]),
-                            ..Default::default()
-                        });
-
-                        region_stats.extend(stats);
-                    }
-                    Err(e) => {
-                        warn!("Failed to get region status, err: {:?}", e);
-                    }
-                };
-            }
-        }
-    }
-    (region_number, region_stats)
-}

src/catalog/src/local/manager.rs

@@ -1,647 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::Arc;
-
-use common_catalog::consts::{
-    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
-    MITO_ENGINE, NUMBERS_TABLE_ID, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID,
-    SYSTEM_CATALOG_TABLE_NAME,
-};
-use common_catalog::format_full_table_name;
-use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use common_telemetry::{error, info};
-use datatypes::prelude::ScalarVector;
-use datatypes::vectors::{BinaryVector, UInt8Vector};
-use futures_util::lock::Mutex;
-use metrics::increment_gauge;
-use snafu::{ensure, OptionExt, ResultExt};
-use table::engine::manager::TableEngineManagerRef;
-use table::engine::EngineContext;
-use table::metadata::TableId;
-use table::requests::OpenTableRequest;
-use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
-use table::table::TableIdProvider;
-use table::TableRef;
-
-use crate::error::{
-    self, CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu,
-    Result, SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu,
-    SystemCatalogTypeMismatchSnafu, TableEngineNotFoundSnafu, TableExistsSnafu, TableNotExistSnafu,
-    TableNotFoundSnafu, UnimplementedSnafu,
-};
-use crate::information_schema::InformationSchemaProvider;
-use crate::local::memory::MemoryCatalogManager;
-use crate::system::{
-    decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
-    VALUE_INDEX,
-};
-use crate::tables::SystemCatalog;
-use crate::{
-    handle_system_table_request, CatalogManager, CatalogManagerRef, DeregisterSchemaRequest,
-    DeregisterTableRequest, RegisterSchemaRequest, RegisterSystemTableRequest,
-    RegisterTableRequest, RenameTableRequest,
-};
-
-/// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
-pub struct LocalCatalogManager {
-    system: Arc<SystemCatalog>,
-    catalogs: Arc<MemoryCatalogManager>,
-    engine_manager: TableEngineManagerRef,
-    next_table_id: AtomicU32,
-    init_lock: Mutex<bool>,
-    register_lock: Mutex<()>,
-    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
-}
-
-impl LocalCatalogManager {
-    /// Create a new [CatalogManager] with given user catalogs and mito engine
-    pub async fn try_new(engine_manager: TableEngineManagerRef) -> Result<Self> {
-        let engine = engine_manager
-            .engine(MITO_ENGINE)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: MITO_ENGINE,
-            })?;
-        let table = SystemCatalogTable::new(engine.clone()).await?;
-        let memory_catalog_manager = crate::local::memory::new_memory_catalog_manager()?;
-        let system_catalog = Arc::new(SystemCatalog::new(table));
-        Ok(Self {
-            system: system_catalog,
-            catalogs: memory_catalog_manager,
-            engine_manager,
-            next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
-            init_lock: Mutex::new(false),
-            register_lock: Mutex::new(()),
-            system_table_requests: Mutex::new(Vec::default()),
-        })
-    }
-
-    /// Scan all entries from system catalog table
-    pub async fn init(&self) -> Result<()> {
-        self.init_system_catalog().await?;
-        let system_records = self.system.information_schema.system.records().await?;
-        let entries = self.collect_system_catalog_entries(system_records).await?;
-        let max_table_id = self.handle_system_catalog_entries(entries).await?;
-
-        info!(
-            "All system catalog entries processed, max table id: {}",
-            max_table_id
-        );
-        self.next_table_id
-            .store((max_table_id + 1).max(MIN_USER_TABLE_ID), Ordering::Relaxed);
-        *self.init_lock.lock().await = true;
-
-        // Processing system table hooks
-        let mut sys_table_requests = self.system_table_requests.lock().await;
-        let engine = self
-            .engine_manager
-            .engine(MITO_ENGINE)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: MITO_ENGINE,
-            })?;
-
-        handle_system_table_request(self, engine, &mut sys_table_requests).await?;
-        Ok(())
-    }
-
-    async fn init_system_catalog(&self) -> Result<()> {
-        // register SystemCatalogTable
-        let _ = self
-            .catalogs
-            .register_catalog_sync(SYSTEM_CATALOG_NAME.to_string())?;
-        let _ = self.catalogs.register_schema_sync(RegisterSchemaRequest {
-            catalog: SYSTEM_CATALOG_NAME.to_string(),
-            schema: INFORMATION_SCHEMA_NAME.to_string(),
-        })?;
-        let register_table_req = RegisterTableRequest {
-            catalog: SYSTEM_CATALOG_NAME.to_string(),
-            schema: INFORMATION_SCHEMA_NAME.to_string(),
-            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
-            table_id: SYSTEM_CATALOG_TABLE_ID,
-            table: self.system.information_schema.system.clone(),
-        };
-        let _ = self.catalogs.register_table(register_table_req).await?;
-
-        // register default catalog and default schema
-        let _ = self
-            .catalogs
-            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())?;
-        let _ = self.catalogs.register_schema_sync(RegisterSchemaRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-        })?;
-
-        // Add numbers table for test
-        let numbers_table = Arc::new(NumbersTable::default());
-        let register_number_table_req = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: NUMBERS_TABLE_NAME.to_string(),
-            table_id: NUMBERS_TABLE_ID,
-            table: numbers_table,
-        };
-
-        let _ = self
-            .catalogs
-            .register_table(register_number_table_req)
-            .await?;
-
-        Ok(())
-    }
-
-    /// Collect stream of system catalog entries to `Vec<Entry>`
-    async fn collect_system_catalog_entries(
-        &self,
-        stream: SendableRecordBatchStream,
-    ) -> Result<Vec<Entry>> {
-        let record_batch = common_recordbatch::util::collect(stream)
-            .await
-            .context(ReadSystemCatalogSnafu)?;
-        let rbs = record_batch
-            .into_iter()
-            .map(Self::record_batch_to_entry)
-            .collect::<Result<Vec<_>>>()?;
-        Ok(rbs.into_iter().flat_map(Vec::into_iter).collect::<_>())
-    }
-
-    /// Convert `RecordBatch` to a vector of `Entry`.
-    fn record_batch_to_entry(rb: RecordBatch) -> Result<Vec<Entry>> {
-        ensure!(
-            rb.num_columns() >= 6,
-            SystemCatalogSnafu {
-                msg: format!("Length mismatch: {}", rb.num_columns())
-            }
-        );
-
-        let entry_type = rb
-            .column(ENTRY_TYPE_INDEX)
-            .as_any()
-            .downcast_ref::<UInt8Vector>()
-            .with_context(|| SystemCatalogTypeMismatchSnafu {
-                data_type: rb.column(ENTRY_TYPE_INDEX).data_type(),
-            })?;
-
-        let key = rb
-            .column(KEY_INDEX)
-            .as_any()
-            .downcast_ref::<BinaryVector>()
-            .with_context(|| SystemCatalogTypeMismatchSnafu {
-                data_type: rb.column(KEY_INDEX).data_type(),
-            })?;
-
-        let value = rb
-            .column(VALUE_INDEX)
-            .as_any()
-            .downcast_ref::<BinaryVector>()
-            .with_context(|| SystemCatalogTypeMismatchSnafu {
-                data_type: rb.column(VALUE_INDEX).data_type(),
-            })?;
-
-        let mut res = Vec::with_capacity(rb.num_rows());
-        for ((t, k), v) in entry_type
-            .iter_data()
-            .zip(key.iter_data())
-            .zip(value.iter_data())
-        {
-            let entry = decode_system_catalog(t, k, v)?;
-            res.push(entry);
-        }
-        Ok(res)
-    }
-
-    /// Processes records from system catalog table and returns the max table id persisted
-    /// in system catalog table.
-    async fn handle_system_catalog_entries(&self, entries: Vec<Entry>) -> Result<TableId> {
-        let entries = Self::sort_entries(entries);
-        let mut max_table_id = 0;
-        for entry in entries {
-            match entry {
-                Entry::Catalog(c) => {
-                    let _ = self
-                        .catalogs
-                        .register_catalog_if_absent(c.catalog_name.clone());
-                    info!("Register catalog: {}", c.catalog_name);
-                }
-                Entry::Schema(s) => {
-                    let req = RegisterSchemaRequest {
-                        catalog: s.catalog_name.clone(),
-                        schema: s.schema_name.clone(),
-                    };
-                    let _ = self.catalogs.register_schema_sync(req)?;
-                    info!("Registered schema: {:?}", s);
-                }
-                Entry::Table(t) => {
-                    max_table_id = max_table_id.max(t.table_id);
-                    if t.is_deleted {
-                        continue;
-                    }
-                    self.open_and_register_table(&t).await?;
-                    info!("Registered table: {:?}", t);
-                }
-            }
-        }
-        Ok(max_table_id)
-    }
-
-    /// Sort catalog entries to ensure catalog entries comes first, then schema entries,
-    /// and table entries is the last.
-    fn sort_entries(mut entries: Vec<Entry>) -> Vec<Entry> {
-        entries.sort();
-        entries
-    }
-
-    async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
-        self.check_catalog_schema_exist(&t.catalog_name, &t.schema_name)
-            .await?;
-
-        let context = EngineContext {};
-        let open_request = OpenTableRequest {
-            catalog_name: t.catalog_name.clone(),
-            schema_name: t.schema_name.clone(),
-            table_name: t.table_name.clone(),
-            table_id: t.table_id,
-            region_numbers: vec![0],
-        };
-        let engine = self
-            .engine_manager
-            .engine(&t.engine)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: &t.engine,
-            })?;
-
-        let table_ref = engine
-            .open_table(&context, open_request)
-            .await
-            .with_context(|_| OpenTableSnafu {
-                table_info: format!(
-                    "{}.{}.{}, id: {}",
-                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
-                ),
-            })?
-            .with_context(|| TableNotFoundSnafu {
-                table_info: format!(
-                    "{}.{}.{}, id: {}",
-                    &t.catalog_name, &t.schema_name, &t.table_name, t.table_id
-                ),
-            })?;
-
-        let register_request = RegisterTableRequest {
-            catalog: t.catalog_name.clone(),
-            schema: t.schema_name.clone(),
-            table_name: t.table_name.clone(),
-            table_id: t.table_id,
-            table: table_ref,
-        };
-        let _ = self.catalogs.register_table(register_request).await?;
-
-        Ok(())
-    }
-
-    async fn check_state(&self) -> Result<()> {
-        let started = self.init_lock.lock().await;
-        ensure!(
-            *started,
-            IllegalManagerStateSnafu {
-                msg: "Catalog manager not started",
-            }
-        );
-        Ok(())
-    }
-
-    async fn check_catalog_schema_exist(
-        &self,
-        catalog_name: &str,
-        schema_name: &str,
-    ) -> Result<()> {
-        if !self.catalogs.catalog_exist(catalog_name).await? {
-            return CatalogNotFoundSnafu { catalog_name }.fail()?;
-        }
-        if !self
-            .catalogs
-            .schema_exist(catalog_name, schema_name)
-            .await?
-        {
-            return SchemaNotFoundSnafu {
-                catalog: catalog_name,
-                schema: schema_name,
-            }
-            .fail()?;
-        }
-        Ok(())
-    }
-}
-
-#[async_trait::async_trait]
-impl TableIdProvider for LocalCatalogManager {
-    async fn next_table_id(&self) -> table::Result<TableId> {
-        Ok(self.next_table_id.fetch_add(1, Ordering::Relaxed))
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogManager for LocalCatalogManager {
-    /// Start [LocalCatalogManager] to load all information from system catalog table.
-    /// Make sure table engine is initialized before starting [MemoryCatalogManager].
-    async fn start(&self) -> Result<()> {
-        self.init().await
-    }
-
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        self.check_state().await?;
-
-        let catalog_name = request.catalog.clone();
-        let schema_name = request.schema.clone();
-
-        self.check_catalog_schema_exist(&catalog_name, &schema_name)
-            .await?;
-
-        {
-            let _lock = self.register_lock.lock().await;
-            if let Some(existing) = self
-                .catalogs
-                .table(&request.catalog, &request.schema, &request.table_name)
-                .await?
-            {
-                if existing.table_info().ident.table_id != request.table_id {
-                    error!(
-                        "Unexpected table register request: {:?}, existing: {:?}",
-                        request,
-                        existing.table_info()
-                    );
-                    return TableExistsSnafu {
-                        table: format_full_table_name(
-                            &catalog_name,
-                            &schema_name,
-                            &request.table_name,
-                        ),
-                    }
-                    .fail();
-                }
-                // Try to register table with same table id, just ignore.
-                Ok(false)
-            } else {
-                // table does not exist
-                let engine = request.table.table_info().meta.engine.to_string();
-                let table_name = request.table_name.clone();
-                let table_id = request.table_id;
-                let _ = self.catalogs.register_table(request).await?;
-                let _ = self
-                    .system
-                    .register_table(
-                        catalog_name.clone(),
-                        schema_name.clone(),
-                        table_name,
-                        table_id,
-                        engine,
-                    )
-                    .await?;
-                increment_gauge!(
-                    crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-                    1.0,
-                    &[crate::metrics::db_label(&catalog_name, &schema_name)],
-                );
-                Ok(true)
-            }
-        }
-    }
-
-    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        self.check_state().await?;
-
-        let catalog_name = &request.catalog;
-        let schema_name = &request.schema;
-
-        self.check_catalog_schema_exist(catalog_name, schema_name)
-            .await?;
-        ensure!(
-            self.catalogs
-                .table(catalog_name, schema_name, &request.new_table_name)
-                .await?
-                .is_none(),
-            TableExistsSnafu {
-                table: &request.new_table_name
-            }
-        );
-
-        let _lock = self.register_lock.lock().await;
-        let old_table = self
-            .catalogs
-            .table(catalog_name, schema_name, &request.table_name)
-            .await?
-            .context(TableNotExistSnafu {
-                table: &request.table_name,
-            })?;
-
-        let engine = old_table.table_info().meta.engine.to_string();
-        // rename table in system catalog
-        let _ = self
-            .system
-            .register_table(
-                catalog_name.clone(),
-                schema_name.clone(),
-                request.new_table_name.clone(),
-                request.table_id,
-                engine,
-            )
-            .await?;
-
-        self.catalogs.rename_table(request).await
-    }
-
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
-        self.check_state().await?;
-
-        {
-            let _ = self.register_lock.lock().await;
-
-            let DeregisterTableRequest {
-                catalog,
-                schema,
-                table_name,
-            } = &request;
-            let table_id = self
-                .catalogs
-                .table(catalog, schema, table_name)
-                .await?
-                .with_context(|| error::TableNotExistSnafu {
-                    table: format_full_table_name(catalog, schema, table_name),
-                })?
-                .table_info()
-                .ident
-                .table_id;
-
-            self.system.deregister_table(&request, table_id).await?;
-            self.catalogs.deregister_table(request).await
-        }
-    }
-
-    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        self.check_state().await?;
-
-        let catalog_name = &request.catalog;
-        let schema_name = &request.schema;
-
-        if !self.catalogs.catalog_exist(catalog_name).await? {
-            return CatalogNotFoundSnafu { catalog_name }.fail()?;
-        }
-
-        {
-            let _lock = self.register_lock.lock().await;
-            ensure!(
-                !self
-                    .catalogs
-                    .schema_exist(catalog_name, schema_name)
-                    .await?,
-                SchemaExistsSnafu {
-                    schema: schema_name,
-                }
-            );
-            let _ = self
-                .system
-                .register_schema(request.catalog.clone(), schema_name.clone())
-                .await?;
-            self.catalogs.register_schema_sync(request)
-        }
-    }
-
-    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
-        UnimplementedSnafu {
-            operation: "deregister schema",
-        }
-        .fail()
-    }
-
-    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
-        let catalog_name = request.create_table_request.catalog_name.clone();
-        let schema_name = request.create_table_request.schema_name.clone();
-
-        let mut sys_table_requests = self.system_table_requests.lock().await;
-        sys_table_requests.push(request);
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&catalog_name, &schema_name)],
-        );
-        Ok(())
-    }
-
-    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
-        self.catalogs.schema_exist(catalog, schema).await
-    }
-
-    async fn table(
-        &self,
-        catalog_name: &str,
-        schema_name: &str,
-        table_name: &str,
-    ) -> Result<Option<TableRef>> {
-        if schema_name == INFORMATION_SCHEMA_NAME {
-            let manager: CatalogManagerRef = self.catalogs.clone() as _;
-            let provider =
-                InformationSchemaProvider::new(catalog_name.to_string(), Arc::downgrade(&manager));
-            return provider.table(table_name);
-        }
-
-        self.catalogs
-            .table(catalog_name, schema_name, table_name)
-            .await
-    }
-
-    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
-        if catalog.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
-            Ok(true)
-        } else {
-            self.catalogs.catalog_exist(catalog).await
-        }
-    }
-
-    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
-        self.catalogs.table_exist(catalog, schema, table).await
-    }
-
-    async fn catalog_names(&self) -> Result<Vec<String>> {
-        self.catalogs.catalog_names().await
-    }
-
-    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
-        self.catalogs.schema_names(catalog_name).await
-    }
-
-    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
-        self.catalogs.table_names(catalog_name, schema_name).await
-    }
-
-    async fn register_catalog(&self, name: String) -> Result<bool> {
-        self.catalogs.register_catalog(name).await
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::assert_matches::assert_matches;
-
-    use mito::engine::MITO_ENGINE;
-
-    use super::*;
-    use crate::system::{CatalogEntry, SchemaEntry};
-
-    #[test]
-    fn test_sort_entry() {
-        let vec = vec![
-            Entry::Table(TableEntry {
-                catalog_name: "C1".to_string(),
-                schema_name: "S1".to_string(),
-                table_name: "T1".to_string(),
-                table_id: 1,
-                engine: MITO_ENGINE.to_string(),
-                is_deleted: false,
-            }),
-            Entry::Catalog(CatalogEntry {
-                catalog_name: "C2".to_string(),
-            }),
-            Entry::Schema(SchemaEntry {
-                catalog_name: "C1".to_string(),
-                schema_name: "S1".to_string(),
-            }),
-            Entry::Schema(SchemaEntry {
-                catalog_name: "C2".to_string(),
-                schema_name: "S2".to_string(),
-            }),
-            Entry::Catalog(CatalogEntry {
-                catalog_name: "".to_string(),
-            }),
-            Entry::Table(TableEntry {
-                catalog_name: "C1".to_string(),
-                schema_name: "S1".to_string(),
-                table_name: "T2".to_string(),
-                table_id: 2,
-                engine: MITO_ENGINE.to_string(),
-                is_deleted: false,
-            }),
-        ];
-        let res = LocalCatalogManager::sort_entries(vec);
-        assert_matches!(res[0], Entry::Catalog(..));
-        assert_matches!(res[1], Entry::Catalog(..));
-        assert_matches!(res[2], Entry::Schema(..));
-        assert_matches!(res[3], Entry::Schema(..));
-        assert_matches!(res[4], Entry::Table(..));
-        assert_matches!(res[5], Entry::Table(..));
-    }
-}

src/catalog/src/local/memory.rs

@@ -1,587 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::collections::hash_map::Entry;
-use std::collections::HashMap;
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::{Arc, RwLock};
-
-use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID};
-use metrics::{decrement_gauge, increment_gauge};
-use snafu::OptionExt;
-use table::metadata::TableId;
-use table::table::TableIdProvider;
-use table::TableRef;
-
-use crate::error::{
-    CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, TableNotFoundSnafu,
-};
-use crate::{
-    CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest, RegisterSchemaRequest,
-    RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
-};
-
-type SchemaEntries = HashMap<String, HashMap<String, TableRef>>;
-
-/// Simple in-memory list of catalogs
-pub struct MemoryCatalogManager {
-    /// Collection of catalogs containing schemas and ultimately Tables
-    pub catalogs: RwLock<HashMap<String, SchemaEntries>>,
-    pub table_id: AtomicU32,
-}
-
-impl Default for MemoryCatalogManager {
-    fn default() -> Self {
-        let manager = Self {
-            table_id: AtomicU32::new(MIN_USER_TABLE_ID),
-            catalogs: Default::default(),
-        };
-
-        let catalog = HashMap::from([(DEFAULT_SCHEMA_NAME.to_string(), HashMap::new())]);
-        let _ = manager
-            .catalogs
-            .write()
-            .unwrap()
-            .insert(DEFAULT_CATALOG_NAME.to_string(), catalog);
-
-        manager
-    }
-}
-
-#[async_trait::async_trait]
-impl TableIdProvider for MemoryCatalogManager {
-    async fn next_table_id(&self) -> table::error::Result<TableId> {
-        Ok(self.table_id.fetch_add(1, Ordering::Relaxed))
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogManager for MemoryCatalogManager {
-    async fn start(&self) -> Result<()> {
-        self.table_id.store(MIN_USER_TABLE_ID, Ordering::Relaxed);
-        Ok(())
-    }
-
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        self.register_table_sync(request)
-    }
-
-    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let schema = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?
-            .get_mut(&request.schema)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?;
-
-        // check old and new table names
-        if !schema.contains_key(&request.table_name) {
-            return TableNotFoundSnafu {
-                table_info: request.table_name.to_string(),
-            }
-            .fail()?;
-        }
-        if schema.contains_key(&request.new_table_name) {
-            return TableExistsSnafu {
-                table: &request.new_table_name,
-            }
-            .fail();
-        }
-
-        let table = schema.remove(&request.table_name).unwrap();
-        let _ = schema.insert(request.new_table_name, table);
-
-        Ok(true)
-    }
-
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let schema = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?
-            .get_mut(&request.schema)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?;
-        let result = schema.remove(&request.table_name);
-        if result.is_some() {
-            decrement_gauge!(
-                crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-                1.0,
-                &[crate::metrics::db_label(&request.catalog, &request.schema)],
-            );
-        }
-        Ok(())
-    }
-
-    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        self.register_schema_sync(request)
-    }
-
-    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let schemas = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?;
-        let table_count = schemas
-            .remove(&request.schema)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?
-            .len();
-        decrement_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            table_count as f64,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
-
-        decrement_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
-        Ok(true)
-    }
-
-    async fn register_system_table(&self, _request: RegisterSystemTableRequest) -> Result<()> {
-        // TODO(ruihang): support register system table request
-        Ok(())
-    }
-
-    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
-        Ok(self
-            .catalogs
-            .read()
-            .unwrap()
-            .get(catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: catalog,
-            })?
-            .contains_key(schema))
-    }
-
-    async fn table(
-        &self,
-        catalog: &str,
-        schema: &str,
-        table_name: &str,
-    ) -> Result<Option<TableRef>> {
-        let result = try {
-            self.catalogs
-                .read()
-                .unwrap()
-                .get(catalog)?
-                .get(schema)?
-                .get(table_name)
-                .cloned()?
-        };
-        Ok(result)
-    }
-
-    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
-        Ok(self.catalogs.read().unwrap().get(catalog).is_some())
-    }
-
-    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
-        let catalogs = self.catalogs.read().unwrap();
-        Ok(catalogs
-            .get(catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: catalog,
-            })?
-            .get(schema)
-            .with_context(|| SchemaNotFoundSnafu { catalog, schema })?
-            .contains_key(table))
-    }
-
-    async fn catalog_names(&self) -> Result<Vec<String>> {
-        Ok(self.catalogs.read().unwrap().keys().cloned().collect())
-    }
-
-    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
-        Ok(self
-            .catalogs
-            .read()
-            .unwrap()
-            .get(catalog_name)
-            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
-            .keys()
-            .cloned()
-            .collect())
-    }
-
-    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
-        Ok(self
-            .catalogs
-            .read()
-            .unwrap()
-            .get(catalog_name)
-            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
-            .get(schema_name)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: catalog_name,
-                schema: schema_name,
-            })?
-            .keys()
-            .cloned()
-            .collect())
-    }
-
-    async fn register_catalog(&self, name: String) -> Result<bool> {
-        self.register_catalog_sync(name)
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-impl MemoryCatalogManager {
-    /// Registers a catalog and return the catalog already exist
-    pub fn register_catalog_if_absent(&self, name: String) -> bool {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let entry = catalogs.entry(name);
-        match entry {
-            Entry::Occupied(_) => true,
-            Entry::Vacant(v) => {
-                let _ = v.insert(HashMap::new());
-                false
-            }
-        }
-    }
-
-    pub fn register_catalog_sync(&self, name: String) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-
-        match catalogs.entry(name) {
-            Entry::Vacant(e) => {
-                e.insert(HashMap::new());
-                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
-                Ok(true)
-            }
-            Entry::Occupied(_) => Ok(false),
-        }
-    }
-
-    pub fn register_schema_sync(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let catalog = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?;
-
-        match catalog.entry(request.schema) {
-            Entry::Vacant(e) => {
-                e.insert(HashMap::new());
-                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
-                Ok(true)
-            }
-            Entry::Occupied(_) => Ok(false),
-        }
-    }
-
-    pub fn register_table_sync(&self, request: RegisterTableRequest) -> Result<bool> {
-        let mut catalogs = self.catalogs.write().unwrap();
-        let schema = catalogs
-            .get_mut(&request.catalog)
-            .with_context(|| CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?
-            .get_mut(&request.schema)
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?;
-
-        if schema.contains_key(&request.table_name) {
-            return TableExistsSnafu {
-                table: &request.table_name,
-            }
-            .fail();
-        }
-        schema.insert(request.table_name, request.table);
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
-        Ok(true)
-    }
-
-    #[cfg(any(test, feature = "testing"))]
-    pub fn new_with_table(table: TableRef) -> Self {
-        let manager = Self::default();
-        let request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table.table_info().name.clone(),
-            table_id: table.table_info().ident.table_id,
-            table,
-        };
-        let _ = manager.register_table_sync(request).unwrap();
-        manager
-    }
-}
-
-/// Create a memory catalog list contains a numbers table for test
-pub fn new_memory_catalog_manager() -> Result<Arc<MemoryCatalogManager>> {
-    Ok(Arc::new(MemoryCatalogManager::default()))
-}
-
-#[cfg(test)]
-mod tests {
-    use common_catalog::consts::*;
-    use common_error::ext::ErrorExt;
-    use common_error::status_code::StatusCode;
-    use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
-
-    use super::*;
-
-    #[tokio::test]
-    async fn test_new_memory_catalog_list() {
-        let catalog_list = new_memory_catalog_manager().unwrap();
-
-        let register_request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: NUMBERS_TABLE_NAME.to_string(),
-            table_id: NUMBERS_TABLE_ID,
-            table: Arc::new(NumbersTable::default()),
-        };
-
-        let _ = catalog_list.register_table(register_request).await.unwrap();
-        let table = catalog_list
-            .table(
-                DEFAULT_CATALOG_NAME,
-                DEFAULT_SCHEMA_NAME,
-                NUMBERS_TABLE_NAME,
-            )
-            .await
-            .unwrap();
-        let _ = table.unwrap();
-        assert!(catalog_list
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "not_exists")
-            .await
-            .unwrap()
-            .is_none());
-    }
-
-    #[tokio::test]
-    async fn test_mem_manager_rename_table() {
-        let catalog = MemoryCatalogManager::default();
-        let table_name = "test_table";
-        assert!(!catalog
-            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap());
-        // register test table
-        let table_id = 2333;
-        let register_request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            table_id,
-            table: Arc::new(NumbersTable::new(table_id)),
-        };
-        assert!(catalog.register_table(register_request).await.unwrap());
-        assert!(catalog
-            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap());
-
-        // rename test table
-        let new_table_name = "test_table_renamed";
-        let rename_request = RenameTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            new_table_name: new_table_name.to_string(),
-            table_id,
-        };
-        let _ = catalog.rename_table(rename_request).await.unwrap();
-
-        // test old table name not exist
-        assert!(!catalog
-            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap());
-
-        // test new table name exists
-        assert!(catalog
-            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap());
-        let registered_table = catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(registered_table.table_info().ident.table_id, table_id);
-
-        let dup_register_request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: new_table_name.to_string(),
-            table_id: table_id + 1,
-            table: Arc::new(NumbersTable::new(table_id + 1)),
-        };
-        let result = catalog.register_table(dup_register_request).await;
-        let err = result.err().unwrap();
-        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
-    }
-
-    #[tokio::test]
-    async fn test_catalog_rename_table() {
-        let catalog = MemoryCatalogManager::default();
-        let table_name = "num";
-        let table_id = 2333;
-        let table: TableRef = Arc::new(NumbersTable::new(table_id));
-
-        // register table
-        let register_table_req = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            table_id,
-            table,
-        };
-        assert!(catalog.register_table(register_table_req).await.unwrap());
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap()
-            .is_some());
-
-        // rename table
-        let new_table_name = "numbers_new";
-        let rename_table_req = RenameTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            new_table_name: new_table_name.to_string(),
-            table_id,
-        };
-        assert!(catalog.rename_table(rename_table_req).await.unwrap());
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap()
-            .is_none());
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap()
-            .is_some());
-
-        let registered_table = catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(registered_table.table_info().ident.table_id, table_id);
-    }
-
-    #[test]
-    pub fn test_register_if_absent() {
-        let list = MemoryCatalogManager::default();
-        assert!(!list.register_catalog_if_absent("test_catalog".to_string(),));
-        assert!(list.register_catalog_if_absent("test_catalog".to_string()));
-    }
-
-    #[tokio::test]
-    pub async fn test_catalog_deregister_table() {
-        let catalog = MemoryCatalogManager::default();
-        let table_name = "foo_table";
-
-        let register_table_req = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            table_id: 2333,
-            table: Arc::new(NumbersTable::default()),
-        };
-        let _ = catalog.register_table(register_table_req).await.unwrap();
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap()
-            .is_some());
-
-        let deregister_table_req = DeregisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-        };
-        catalog
-            .deregister_table(deregister_table_req)
-            .await
-            .unwrap();
-        assert!(catalog
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
-            .await
-            .unwrap()
-            .is_none());
-    }
-
-    #[tokio::test]
-    async fn test_catalog_deregister_schema() {
-        let catalog = MemoryCatalogManager::default();
-
-        // Registers a catalog, a schema, and a table.
-        let catalog_name = "foo_catalog".to_string();
-        let schema_name = "foo_schema".to_string();
-        let table_name = "foo_table".to_string();
-        let schema = RegisterSchemaRequest {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-        };
-        let table = RegisterTableRequest {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-            table_name,
-            table_id: 0,
-            table: Arc::new(NumbersTable::default()),
-        };
-        catalog
-            .register_catalog(catalog_name.clone())
-            .await
-            .unwrap();
-        catalog.register_schema(schema).await.unwrap();
-        catalog.register_table(table).await.unwrap();
-
-        let request = DeregisterSchemaRequest {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-        };
-
-        assert!(catalog.deregister_schema(request).await.unwrap());
-        assert!(!catalog
-            .schema_exist(&catalog_name, &schema_name)
-            .await
-            .unwrap());
-    }
-}

src/catalog/src/memory.rs

@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod config;
-pub mod engine;
-pub mod error;
-mod manifest;
-mod metrics;
-pub mod table;
+pub mod manager;
+
+pub use manager::{new_memory_catalog_manager, MemoryCatalogManager};

src/catalog/src/memory/manager.rs

@@ -0,0 +1,369 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock, Weak};
+
+use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
+use metrics::{decrement_gauge, increment_gauge};
+use snafu::OptionExt;
+use table::TableRef;
+
+use crate::error::{CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu};
+use crate::information_schema::InformationSchemaProvider;
+use crate::{CatalogManager, DeregisterTableRequest, RegisterSchemaRequest, RegisterTableRequest};
+
+type SchemaEntries = HashMap<String, HashMap<String, TableRef>>;
+
+/// Simple in-memory list of catalogs
+#[derive(Clone)]
+pub struct MemoryCatalogManager {
+    /// Collection of catalogs containing schemas and ultimately Tables
+    catalogs: Arc<RwLock<HashMap<String, SchemaEntries>>>,
+}
+
+#[async_trait::async_trait]
+impl CatalogManager for MemoryCatalogManager {
+    async fn schema_exists(&self, catalog: &str, schema: &str) -> Result<bool> {
+        self.schema_exist_sync(catalog, schema)
+    }
+
+    async fn table(
+        &self,
+        catalog: &str,
+        schema: &str,
+        table_name: &str,
+    ) -> Result<Option<TableRef>> {
+        let result = try {
+            self.catalogs
+                .read()
+                .unwrap()
+                .get(catalog)?
+                .get(schema)?
+                .get(table_name)
+                .cloned()?
+        };
+        Ok(result)
+    }
+
+    async fn catalog_exists(&self, catalog: &str) -> Result<bool> {
+        self.catalog_exist_sync(catalog)
+    }
+
+    async fn table_exists(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+        let catalogs = self.catalogs.read().unwrap();
+        Ok(catalogs
+            .get(catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .get(schema)
+            .with_context(|| SchemaNotFoundSnafu { catalog, schema })?
+            .contains_key(table))
+    }
+
+    async fn catalog_names(&self) -> Result<Vec<String>> {
+        Ok(self.catalogs.read().unwrap().keys().cloned().collect())
+    }
+
+    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog_name)
+            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
+            .keys()
+            .cloned()
+            .collect())
+    }
+
+    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog_name)
+            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
+            .get(schema_name)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            })?
+            .keys()
+            .cloned()
+            .collect())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl MemoryCatalogManager {
+    pub fn new() -> Arc<Self> {
+        Arc::new(Self {
+            catalogs: Default::default(),
+        })
+    }
+
+    /// Creates a manager with some default setups
+    /// (e.g. default catalog/schema and information schema)
+    pub fn with_default_setup() -> Arc<Self> {
+        let manager = Arc::new(Self {
+            catalogs: Default::default(),
+        });
+
+        // Safety: default catalog/schema is registered in order so no CatalogNotFound error will occur
+        manager.register_catalog_sync(DEFAULT_CATALOG_NAME).unwrap();
+        manager
+            .register_schema_sync(RegisterSchemaRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+            })
+            .unwrap();
+
+        manager
+    }
+
+    fn schema_exist_sync(&self, catalog: &str, schema: &str) -> Result<bool> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .contains_key(schema))
+    }
+
+    fn catalog_exist_sync(&self, catalog: &str) -> Result<bool> {
+        Ok(self.catalogs.read().unwrap().get(catalog).is_some())
+    }
+
+    /// Registers a catalog if it does not exist and returns false if the schema exists.
+    pub fn register_catalog_sync(&self, name: &str) -> Result<bool> {
+        let name = name.to_string();
+
+        let mut catalogs = self.catalogs.write().unwrap();
+
+        match catalogs.entry(name.clone()) {
+            Entry::Vacant(e) => {
+                let arc_self = Arc::new(self.clone());
+                let catalog = arc_self.create_catalog_entry(name);
+                e.insert(catalog);
+                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
+                Ok(true)
+            }
+            Entry::Occupied(_) => Ok(false),
+        }
+    }
+
+    pub fn deregister_table_sync(&self, request: DeregisterTableRequest) -> Result<()> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .get_mut(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+        let result = schema.remove(&request.table_name);
+        if result.is_some() {
+            decrement_gauge!(
+                crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+                1.0,
+                &[crate::metrics::db_label(&request.catalog, &request.schema)],
+            );
+        }
+        Ok(())
+    }
+
+    /// Registers a schema if it does not exist.
+    /// It returns an error if the catalog does not exist,
+    /// and returns false if the schema exists.
+    pub fn register_schema_sync(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?;
+
+        match catalog.entry(request.schema) {
+            Entry::Vacant(e) => {
+                e.insert(HashMap::new());
+                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
+                Ok(true)
+            }
+            Entry::Occupied(_) => Ok(false),
+        }
+    }
+
+    /// Registers a schema and returns an error if the catalog or schema does not exist.
+    pub fn register_table_sync(&self, request: RegisterTableRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .get_mut(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+
+        if schema.contains_key(&request.table_name) {
+            return TableExistsSnafu {
+                table: &request.table_name,
+            }
+            .fail();
+        }
+        schema.insert(request.table_name, request.table);
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
+        Ok(true)
+    }
+
+    fn create_catalog_entry(self: &Arc<Self>, catalog: String) -> SchemaEntries {
+        let information_schema = InformationSchemaProvider::build(
+            catalog,
+            Arc::downgrade(self) as Weak<dyn CatalogManager>,
+        );
+        let mut catalog = HashMap::new();
+        catalog.insert(INFORMATION_SCHEMA_NAME.to_string(), information_schema);
+        catalog
+    }
+
+    #[cfg(any(test, feature = "testing"))]
+    pub fn new_with_table(table: TableRef) -> Arc<Self> {
+        let manager = Self::with_default_setup();
+        let catalog = &table.table_info().catalog_name;
+        let schema = &table.table_info().schema_name;
+
+        if !manager.catalog_exist_sync(catalog).unwrap() {
+            manager.register_catalog_sync(catalog).unwrap();
+        }
+
+        if !manager.schema_exist_sync(catalog, schema).unwrap() {
+            manager
+                .register_schema_sync(RegisterSchemaRequest {
+                    catalog: catalog.to_string(),
+                    schema: schema.to_string(),
+                })
+                .unwrap();
+        }
+
+        let request = RegisterTableRequest {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table_name: table.table_info().name.clone(),
+            table_id: table.table_info().ident.table_id,
+            table,
+        };
+        let _ = manager.register_table_sync(request).unwrap();
+        manager
+    }
+}
+
+/// Create a memory catalog list contains a numbers table for test
+pub fn new_memory_catalog_manager() -> Result<Arc<MemoryCatalogManager>> {
+    Ok(MemoryCatalogManager::with_default_setup())
+}
+
+#[cfg(test)]
+mod tests {
+    use common_catalog::consts::*;
+    use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_new_memory_catalog_list() {
+        let catalog_list = new_memory_catalog_manager().unwrap();
+
+        let register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: NUMBERS_TABLE_NAME.to_string(),
+            table_id: NUMBERS_TABLE_ID,
+            table: NumbersTable::table(NUMBERS_TABLE_ID),
+        };
+
+        catalog_list.register_table_sync(register_request).unwrap();
+        let table = catalog_list
+            .table(
+                DEFAULT_CATALOG_NAME,
+                DEFAULT_SCHEMA_NAME,
+                NUMBERS_TABLE_NAME,
+            )
+            .await
+            .unwrap();
+        let _ = table.unwrap();
+        assert!(catalog_list
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "not_exists")
+            .await
+            .unwrap()
+            .is_none());
+    }
+
+    #[test]
+    pub fn test_register_catalog_sync() {
+        let list = MemoryCatalogManager::with_default_setup();
+        assert!(list.register_catalog_sync("test_catalog").unwrap());
+        assert!(!list.register_catalog_sync("test_catalog").unwrap());
+    }
+
+    #[tokio::test]
+    pub async fn test_catalog_deregister_table() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+        let table_name = "foo_table";
+
+        let register_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id: 2333,
+            table: NumbersTable::table(2333),
+        };
+        catalog.register_table_sync(register_table_req).unwrap();
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_some());
+
+        let deregister_table_req = DeregisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+        };
+        catalog.deregister_table_sync(deregister_table_req).unwrap();
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_none());
+    }
+}

src/catalog/src/remote/manager.rs

@@ -1,430 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use common_catalog::consts::MITO_ENGINE;
-use common_meta::helper::{CatalogKey, SchemaKey};
-use common_meta::ident::TableIdent;
-use common_meta::key::datanode_table::DatanodeTableValue;
-use common_meta::key::TableMetadataManagerRef;
-use common_meta::kv_backend::KvBackendRef;
-use common_telemetry::{error, info, warn};
-use metrics::increment_gauge;
-use snafu::{ensure, OptionExt, ResultExt};
-use table::engine::manager::TableEngineManagerRef;
-use table::engine::EngineContext;
-use table::requests::OpenTableRequest;
-use table::TableRef;
-use tokio::sync::Mutex;
-
-use crate::error::{
-    OpenTableSnafu, ParallelOpenTableSnafu, Result, TableEngineNotFoundSnafu, TableExistsSnafu,
-    TableMetadataManagerSnafu, TableNotFoundSnafu, UnimplementedSnafu,
-};
-use crate::local::MemoryCatalogManager;
-use crate::remote::region_alive_keeper::RegionAliveKeepers;
-use crate::{
-    handle_system_table_request, CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest,
-    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
-};
-
-/// Catalog manager based on metasrv.
-pub struct RemoteCatalogManager {
-    node_id: u64,
-    backend: KvBackendRef,
-    engine_manager: TableEngineManagerRef,
-    system_table_requests: Mutex<Vec<RegisterSystemTableRequest>>,
-    region_alive_keepers: Arc<RegionAliveKeepers>,
-    memory_catalog_manager: Arc<MemoryCatalogManager>,
-    table_metadata_manager: TableMetadataManagerRef,
-}
-
-impl RemoteCatalogManager {
-    pub fn new(
-        engine_manager: TableEngineManagerRef,
-        node_id: u64,
-        backend: KvBackendRef,
-        region_alive_keepers: Arc<RegionAliveKeepers>,
-        table_metadata_manager: TableMetadataManagerRef,
-    ) -> Self {
-        Self {
-            engine_manager,
-            node_id,
-            backend,
-            system_table_requests: Default::default(),
-            region_alive_keepers,
-            memory_catalog_manager: Arc::new(MemoryCatalogManager::default()),
-            table_metadata_manager,
-        }
-    }
-
-    async fn initiate_catalogs(&self) -> Result<()> {
-        let tables = self
-            .table_metadata_manager
-            .datanode_table_manager()
-            .tables(self.node_id)
-            .await
-            .context(TableMetadataManagerSnafu)?;
-
-        let joins = tables
-            .into_iter()
-            .map(|datanode_table_value| {
-                let engine_manager = self.engine_manager.clone();
-                let memory_catalog_manager = self.memory_catalog_manager.clone();
-                let table_metadata_manager = self.table_metadata_manager.clone();
-                common_runtime::spawn_bg(async move {
-                    let table_id = datanode_table_value.table_id;
-                    if let Err(e) = open_and_register_table(
-                        engine_manager,
-                        datanode_table_value,
-                        memory_catalog_manager,
-                        table_metadata_manager,
-                    )
-                    .await
-                    {
-                        // Note that we don't return error here if table opened failed. This is because
-                        // we don't want those broken tables to impede the startup of Datanode.
-                        // However, this could be changed in the future.
-                        error!(e; "Failed to open or register table, id = {table_id}")
-                    }
-                })
-            })
-            .collect::<Vec<_>>();
-
-        let _ = futures::future::try_join_all(joins)
-            .await
-            .context(ParallelOpenTableSnafu)?;
-        Ok(())
-    }
-
-    fn build_schema_key(&self, catalog_name: String, schema_name: String) -> SchemaKey {
-        SchemaKey {
-            catalog_name,
-            schema_name,
-        }
-    }
-}
-
-async fn open_and_register_table(
-    engine_manager: TableEngineManagerRef,
-    datanode_table_value: DatanodeTableValue,
-    memory_catalog_manager: Arc<MemoryCatalogManager>,
-    table_metadata_manager: TableMetadataManagerRef,
-) -> Result<()> {
-    let context = EngineContext {};
-
-    let table_id = datanode_table_value.table_id;
-    let region_numbers = datanode_table_value.regions;
-
-    let table_info_value = table_metadata_manager
-        .table_info_manager()
-        .get(table_id)
-        .await
-        .context(TableMetadataManagerSnafu)?
-        .context(TableNotFoundSnafu {
-            table_info: format!("table id: {table_id}"),
-        })?;
-    let table_info = &table_info_value.table_info;
-    let catalog_name = table_info.catalog_name.clone();
-    let schema_name = table_info.schema_name.clone();
-    let table_name = table_info.name.clone();
-
-    let request = OpenTableRequest {
-        catalog_name: catalog_name.clone(),
-        schema_name: schema_name.clone(),
-        table_name: table_name.clone(),
-        table_id,
-        region_numbers: region_numbers.clone(),
-    };
-    let engine =
-        engine_manager
-            .engine(&table_info.meta.engine)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: &table_info.meta.engine,
-            })?;
-
-    let table_ident = TableIdent {
-        catalog: catalog_name,
-        schema: schema_name,
-        table: table_name,
-        table_id,
-        engine: table_info.meta.engine.clone(),
-    };
-
-    let table = engine
-        .open_table(&context, request)
-        .await
-        .with_context(|_| OpenTableSnafu {
-            table_info: table_ident.to_string(),
-        })?
-        .with_context(|| TableNotFoundSnafu {
-            table_info: table_ident.to_string(),
-        })?;
-    info!("Successfully opened table, {table_ident}");
-
-    if !memory_catalog_manager
-        .catalog_exist(&table_ident.catalog)
-        .await?
-    {
-        memory_catalog_manager.register_catalog_sync(table_ident.catalog.clone())?;
-    }
-
-    if !memory_catalog_manager
-        .schema_exist(&table_ident.catalog, &table_ident.schema)
-        .await?
-    {
-        memory_catalog_manager.register_schema_sync(RegisterSchemaRequest {
-            catalog: table_ident.catalog.clone(),
-            schema: table_ident.schema.clone(),
-        })?;
-    }
-
-    let request = RegisterTableRequest {
-        catalog: table_ident.catalog.clone(),
-        schema: table_ident.schema.clone(),
-        table_name: table_ident.table.clone(),
-        table_id,
-        table,
-    };
-    let registered = memory_catalog_manager.register_table_sync(request)?;
-    ensure!(
-        registered,
-        TableExistsSnafu {
-            table: table_ident.to_string(),
-        }
-    );
-    info!("Successfully registered table, {table_ident}");
-    Ok(())
-}
-
-#[async_trait]
-impl CatalogManager for RemoteCatalogManager {
-    async fn start(&self) -> Result<()> {
-        self.initiate_catalogs().await?;
-
-        let mut system_table_requests = self.system_table_requests.lock().await;
-        let engine = self
-            .engine_manager
-            .engine(MITO_ENGINE)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: MITO_ENGINE,
-            })?;
-        handle_system_table_request(self, engine, &mut system_table_requests).await?;
-        info!("All system table opened");
-        Ok(())
-    }
-
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        let table = request.table.clone();
-
-        let registered = self.memory_catalog_manager.register_table_sync(request)?;
-
-        if registered {
-            let table_info = table.table_info();
-            let table_ident = TableIdent {
-                catalog: table_info.catalog_name.clone(),
-                schema: table_info.schema_name.clone(),
-                table: table_info.name.clone(),
-                table_id: table_info.table_id(),
-                engine: table_info.meta.engine.clone(),
-            };
-            self.region_alive_keepers
-                .register_table(table_ident, table)
-                .await?;
-        }
-
-        Ok(registered)
-    }
-
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
-        let Some(table) = self
-            .memory_catalog_manager
-            .table(&request.catalog, &request.schema, &request.table_name)
-            .await? else { return Ok(()) };
-
-        let table_info = table.table_info();
-        let table_ident = TableIdent {
-            catalog: request.catalog.clone(),
-            schema: request.schema.clone(),
-            table: request.table_name.clone(),
-            table_id: table_info.ident.table_id,
-            engine: table_info.meta.engine.clone(),
-        };
-        if let Some(keeper) = self
-            .region_alive_keepers
-            .deregister_table(&table_ident)
-            .await
-        {
-            warn!(
-                "Table {} is deregistered from region alive keepers",
-                keeper.table_ident(),
-            );
-        }
-
-        self.memory_catalog_manager.deregister_table(request).await
-    }
-
-    async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        self.memory_catalog_manager.register_schema_sync(request)
-    }
-
-    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
-        UnimplementedSnafu {
-            operation: "deregister schema",
-        }
-        .fail()
-    }
-
-    async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        self.memory_catalog_manager.rename_table(request).await
-    }
-
-    async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
-        let catalog_name = request.create_table_request.catalog_name.clone();
-        let schema_name = request.create_table_request.schema_name.clone();
-
-        let mut requests = self.system_table_requests.lock().await;
-        requests.push(request);
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&catalog_name, &schema_name)],
-        );
-        Ok(())
-    }
-
-    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
-        if !self.catalog_exist(catalog).await? {
-            return Ok(false);
-        }
-
-        if self
-            .memory_catalog_manager
-            .schema_exist(catalog, schema)
-            .await?
-        {
-            return Ok(true);
-        }
-
-        let key = self
-            .build_schema_key(catalog.to_string(), schema.to_string())
-            .to_string();
-        let remote_schema_exists = self
-            .backend
-            .get(key.as_bytes())
-            .await
-            .context(TableMetadataManagerSnafu)?
-            .is_some();
-
-        // Create schema locally if remote schema exists. Since local schema is managed by memory
-        // catalog manager, creating a local schema is relatively cheap (just a HashMap).
-        // Besides, if this method ("schema_exist) is called, it's very likely that someone wants to
-        // create a table in this schema. We should create the schema now.
-        if remote_schema_exists
-            && self
-                .memory_catalog_manager
-                .register_schema(RegisterSchemaRequest {
-                    catalog: catalog.to_string(),
-                    schema: schema.to_string(),
-                })
-                .await?
-        {
-            info!("register schema '{catalog}/{schema}' on demand");
-        }
-
-        Ok(remote_schema_exists)
-    }
-
-    async fn table(
-        &self,
-        catalog_name: &str,
-        schema_name: &str,
-        table_name: &str,
-    ) -> Result<Option<TableRef>> {
-        self.memory_catalog_manager
-            .table(catalog_name, schema_name, table_name)
-            .await
-    }
-
-    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
-        if self.memory_catalog_manager.catalog_exist(catalog).await? {
-            return Ok(true);
-        }
-
-        let key = CatalogKey {
-            catalog_name: catalog.to_string(),
-        };
-
-        let remote_catalog_exists = self
-            .backend
-            .get(key.to_string().as_bytes())
-            .await
-            .context(TableMetadataManagerSnafu)?
-            .is_some();
-
-        // Create catalog locally if remote catalog exists. Since local catalog is managed by memory
-        // catalog manager, creating a local catalog is relatively cheap (just a HashMap).
-        // Besides, if this method ("catalog_exist) is called, it's very likely that someone wants to
-        // create a table in this catalog. We should create the catalog now.
-        if remote_catalog_exists
-            && self
-                .memory_catalog_manager
-                .register_catalog(catalog.to_string())
-                .await?
-        {
-            info!("register catalog '{catalog}' on demand");
-        }
-
-        Ok(remote_catalog_exists)
-    }
-
-    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
-        if !self.catalog_exist(catalog).await? {
-            return Ok(false);
-        }
-
-        if !self.schema_exist(catalog, schema).await? {
-            return Ok(false);
-        }
-
-        self.memory_catalog_manager
-            .table_exist(catalog, schema, table)
-            .await
-    }
-
-    async fn catalog_names(&self) -> Result<Vec<String>> {
-        self.memory_catalog_manager.catalog_names().await
-    }
-
-    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
-        self.memory_catalog_manager.schema_names(catalog_name).await
-    }
-
-    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
-        self.memory_catalog_manager
-            .table_names(catalog_name, schema_name)
-            .await
-    }
-
-    async fn register_catalog(&self, name: String) -> Result<bool> {
-        self.memory_catalog_manager.register_catalog_sync(name)
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}

src/catalog/src/remote/region_alive_keeper.rs

@@ -1,826 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::future::Future;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use common_meta::error::InvalidProtoMsgSnafu;
-use common_meta::heartbeat::handler::{
-    HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
-};
-use common_meta::ident::TableIdent;
-use common_meta::RegionIdent;
-use common_telemetry::{debug, error, info, warn};
-use snafu::{OptionExt, ResultExt};
-use store_api::storage::RegionNumber;
-use table::engine::manager::TableEngineManagerRef;
-use table::engine::{CloseTableResult, EngineContext, TableEngineRef};
-use table::requests::CloseTableRequest;
-use table::TableRef;
-use tokio::sync::{mpsc, oneshot, Mutex};
-use tokio::task::JoinHandle;
-use tokio::time::{Duration, Instant};
-
-use crate::error::{Result, TableEngineNotFoundSnafu};
-
-/// [RegionAliveKeepers] manages all [RegionAliveKeeper] in a scope of tables.
-pub struct RegionAliveKeepers {
-    table_engine_manager: TableEngineManagerRef,
-    keepers: Arc<Mutex<HashMap<TableIdent, Arc<RegionAliveKeeper>>>>,
-    heartbeat_interval_millis: u64,
-    started: AtomicBool,
-
-    /// The epoch when [RegionAliveKeepers] is created. It's used to get a monotonically non-decreasing
-    /// elapsed time when submitting heartbeats to Metasrv (because [Instant] is monotonically
-    /// non-decreasing). The heartbeat request will carry the duration since this epoch, and the
-    /// duration acts like an "invariant point" for region's keep alive lease.
-    epoch: Instant,
-}
-
-impl RegionAliveKeepers {
-    pub fn new(
-        table_engine_manager: TableEngineManagerRef,
-        heartbeat_interval_millis: u64,
-    ) -> Self {
-        Self {
-            table_engine_manager,
-            keepers: Arc::new(Mutex::new(HashMap::new())),
-            heartbeat_interval_millis,
-            started: AtomicBool::new(false),
-            epoch: Instant::now(),
-        }
-    }
-
-    pub async fn find_keeper(&self, table_ident: &TableIdent) -> Option<Arc<RegionAliveKeeper>> {
-        self.keepers.lock().await.get(table_ident).cloned()
-    }
-
-    pub async fn register_table(&self, table_ident: TableIdent, table: TableRef) -> Result<()> {
-        let keeper = self.find_keeper(&table_ident).await;
-        if keeper.is_some() {
-            return Ok(());
-        }
-
-        let table_engine = self
-            .table_engine_manager
-            .engine(&table_ident.engine)
-            .context(TableEngineNotFoundSnafu {
-                engine_name: &table_ident.engine,
-            })?;
-
-        let keeper = Arc::new(RegionAliveKeeper::new(
-            table_engine,
-            table_ident.clone(),
-            self.heartbeat_interval_millis,
-        ));
-        for r in table.table_info().meta.region_numbers.iter() {
-            keeper.register_region(*r).await;
-        }
-
-        let mut keepers = self.keepers.lock().await;
-        let _ = keepers.insert(table_ident.clone(), keeper.clone());
-
-        if self.started.load(Ordering::Relaxed) {
-            keeper.start().await;
-
-            info!("RegionAliveKeeper for table {table_ident} is started!");
-        } else {
-            info!("RegionAliveKeeper for table {table_ident} is registered but not started yet!");
-        }
-        Ok(())
-    }
-
-    pub async fn deregister_table(
-        &self,
-        table_ident: &TableIdent,
-    ) -> Option<Arc<RegionAliveKeeper>> {
-        self.keepers.lock().await.remove(table_ident).map(|x| {
-            info!("Deregister RegionAliveKeeper for table {table_ident}");
-            x
-        })
-    }
-
-    pub async fn register_region(&self, region_ident: &RegionIdent) {
-        let table_ident = &region_ident.table_ident;
-        let Some(keeper) = self.find_keeper(table_ident).await else {
-            // Alive keeper could be affected by lagging msg, just warn and ignore.
-            warn!("Alive keeper for region {region_ident} is not found!");
-            return;
-        };
-        keeper.register_region(region_ident.region_number).await
-    }
-
-    pub async fn deregister_region(&self, region_ident: &RegionIdent) {
-        let table_ident = &region_ident.table_ident;
-        let Some(keeper) = self.find_keeper(table_ident).await else {
-            // Alive keeper could be affected by lagging msg, just warn and ignore.
-            warn!("Alive keeper for region {region_ident} is not found!");
-            return;
-        };
-        let _ = keeper.deregister_region(region_ident.region_number).await;
-    }
-
-    pub async fn start(&self) {
-        let keepers = self.keepers.lock().await;
-        for keeper in keepers.values() {
-            keeper.start().await;
-        }
-        self.started.store(true, Ordering::Relaxed);
-
-        info!(
-            "RegionAliveKeepers for tables {:?} are started!",
-            keepers.keys().map(|x| x.to_string()).collect::<Vec<_>>(),
-        );
-    }
-
-    pub fn epoch(&self) -> Instant {
-        self.epoch
-    }
-}
-
-#[async_trait]
-impl HeartbeatResponseHandler for RegionAliveKeepers {
-    fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
-        !ctx.response.region_leases.is_empty()
-    }
-
-    async fn handle(
-        &self,
-        ctx: &mut HeartbeatResponseHandlerContext,
-    ) -> common_meta::error::Result<HandleControl> {
-        let leases = ctx.response.region_leases.drain(..).collect::<Vec<_>>();
-        for lease in leases {
-            let table_ident: TableIdent = match lease
-                .table_ident
-                .context(InvalidProtoMsgSnafu {
-                    err_msg: "'table_ident' is missing in RegionLease",
-                })
-                .and_then(|x| x.try_into())
-            {
-                Ok(x) => x,
-                Err(e) => {
-                    error!(e; "");
-                    continue;
-                }
-            };
-
-            let Some(keeper) = self.keepers.lock().await.get(&table_ident).cloned() else {
-                // Alive keeper could be affected by lagging msg, just warn and ignore.
-                warn!("Alive keeper for table {table_ident} is not found!");
-                continue;
-            };
-
-            let start_instant = self.epoch + Duration::from_millis(lease.duration_since_epoch);
-            let deadline = start_instant + Duration::from_secs(lease.lease_seconds);
-            keeper.keep_lived(lease.regions, deadline).await;
-        }
-        Ok(HandleControl::Continue)
-    }
-}
-
-/// [RegionAliveKeeper] starts a countdown for each region in a table. When deadline is reached,
-/// the region will be closed.
-/// The deadline is controlled by Metasrv. It works like "lease" for regions: a Datanode submits its
-/// opened regions to Metasrv, in heartbeats. If Metasrv decides some region could be resided in this
-/// Datanode, it will "extend" the region's "lease", with a deadline for [RegionAliveKeeper] to
-/// countdown.
-pub struct RegionAliveKeeper {
-    table_engine: TableEngineRef,
-    table_ident: TableIdent,
-    countdown_task_handles: Arc<Mutex<HashMap<RegionNumber, Arc<CountdownTaskHandle>>>>,
-    heartbeat_interval_millis: u64,
-    started: AtomicBool,
-}
-
-impl RegionAliveKeeper {
-    fn new(
-        table_engine: TableEngineRef,
-        table_ident: TableIdent,
-        heartbeat_interval_millis: u64,
-    ) -> Self {
-        Self {
-            table_engine,
-            table_ident,
-            countdown_task_handles: Arc::new(Mutex::new(HashMap::new())),
-            heartbeat_interval_millis,
-            started: AtomicBool::new(false),
-        }
-    }
-
-    async fn find_handle(&self, region: &RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
-        self.countdown_task_handles
-            .lock()
-            .await
-            .get(region)
-            .cloned()
-    }
-
-    async fn register_region(&self, region: RegionNumber) {
-        if self.find_handle(&region).await.is_some() {
-            return;
-        }
-
-        let countdown_task_handles = Arc::downgrade(&self.countdown_task_handles);
-        let on_task_finished = async move {
-            if let Some(x) = countdown_task_handles.upgrade() {
-                let _ = x.lock().await.remove(&region);
-            } // Else the countdown task handles map could be dropped because the keeper is dropped.
-        };
-        let handle = Arc::new(CountdownTaskHandle::new(
-            self.table_engine.clone(),
-            self.table_ident.clone(),
-            region,
-            || on_task_finished,
-        ));
-
-        let mut handles = self.countdown_task_handles.lock().await;
-        let _ = handles.insert(region, handle.clone());
-
-        if self.started.load(Ordering::Relaxed) {
-            handle.start(self.heartbeat_interval_millis).await;
-
-            info!(
-                "Region alive countdown for region {region} in table {} is started!",
-                self.table_ident
-            );
-        } else {
-            info!(
-                "Region alive countdown for region {region} in table {} is registered but not started yet!",
-                self.table_ident
-            );
-        }
-    }
-
-    async fn deregister_region(&self, region: RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
-        self.countdown_task_handles
-            .lock()
-            .await
-            .remove(&region)
-            .map(|x| {
-                info!(
-                    "Deregister alive countdown for region {region} in table {}",
-                    self.table_ident
-                );
-                x
-            })
-    }
-
-    async fn start(&self) {
-        let handles = self.countdown_task_handles.lock().await;
-        for handle in handles.values() {
-            handle.start(self.heartbeat_interval_millis).await;
-        }
-
-        self.started.store(true, Ordering::Relaxed);
-        info!(
-            "Region alive countdowns for regions {:?} in table {} are started!",
-            handles.keys().copied().collect::<Vec<_>>(),
-            self.table_ident
-        );
-    }
-
-    async fn keep_lived(&self, designated_regions: Vec<RegionNumber>, deadline: Instant) {
-        for region in designated_regions {
-            if let Some(handle) = self.find_handle(&region).await {
-                handle.reset_deadline(deadline).await;
-            }
-            // Else the region alive keeper might be triggered by lagging messages, we can safely ignore it.
-        }
-    }
-
-    pub async fn deadline(&self, region: RegionNumber) -> Option<Instant> {
-        let mut deadline = None;
-        if let Some(handle) = self.find_handle(&region).await {
-            let (s, r) = oneshot::channel();
-            if handle.tx.send(CountdownCommand::Deadline(s)).await.is_ok() {
-                deadline = r.await.ok()
-            }
-        }
-        deadline
-    }
-
-    pub fn table_ident(&self) -> &TableIdent {
-        &self.table_ident
-    }
-}
-
-#[derive(Debug)]
-enum CountdownCommand {
-    Start(u64),
-    Reset(Instant),
-    Deadline(oneshot::Sender<Instant>),
-}
-
-struct CountdownTaskHandle {
-    tx: mpsc::Sender<CountdownCommand>,
-    handler: JoinHandle<()>,
-    table_ident: TableIdent,
-    region: RegionNumber,
-}
-
-impl CountdownTaskHandle {
-    /// Creates a new [CountdownTaskHandle] and starts the countdown task.
-    /// # Params
-    /// - `on_task_finished`: a callback to be invoked when the task is finished. Note that it will not
-    ///   be invoked if the task is cancelled (by dropping the handle). This is because we want something
-    ///   meaningful to be done when the task is finished, e.g. deregister the handle from the map.
-    ///   While dropping the handle does not necessarily mean the task is finished.
-    fn new<Fut>(
-        table_engine: TableEngineRef,
-        table_ident: TableIdent,
-        region: RegionNumber,
-        on_task_finished: impl FnOnce() -> Fut + Send + 'static,
-    ) -> Self
-    where
-        Fut: Future<Output = ()> + Send,
-    {
-        let (tx, rx) = mpsc::channel(1024);
-
-        let mut countdown_task = CountdownTask {
-            table_engine,
-            table_ident: table_ident.clone(),
-            region,
-            rx,
-        };
-        let handler = common_runtime::spawn_bg(async move {
-            countdown_task.run().await;
-            on_task_finished().await;
-        });
-
-        Self {
-            tx,
-            handler,
-            table_ident,
-            region,
-        }
-    }
-
-    async fn start(&self, heartbeat_interval_millis: u64) {
-        if let Err(e) = self
-            .tx
-            .send(CountdownCommand::Start(heartbeat_interval_millis))
-            .await
-        {
-            warn!(
-                "Failed to start region alive keeper countdown: {e}. \
-                Maybe the task is stopped due to region been closed."
-            );
-        }
-    }
-
-    async fn reset_deadline(&self, deadline: Instant) {
-        if let Err(e) = self.tx.send(CountdownCommand::Reset(deadline)).await {
-            warn!(
-                "Failed to reset region alive keeper deadline: {e}. \
-                Maybe the task is stopped due to region been closed."
-            );
-        }
-    }
-}
-
-impl Drop for CountdownTaskHandle {
-    fn drop(&mut self) {
-        debug!(
-            "Aborting region alive countdown task for region {} in table {}",
-            self.region, self.table_ident,
-        );
-        self.handler.abort();
-    }
-}
-
-struct CountdownTask {
-    table_engine: TableEngineRef,
-    table_ident: TableIdent,
-    region: RegionNumber,
-    rx: mpsc::Receiver<CountdownCommand>,
-}
-
-impl CountdownTask {
-    async fn run(&mut self) {
-        // 30 years. See `Instant::far_future`.
-        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
-
-        // Make sure the alive countdown is not gonna happen before heartbeat task is started (the
-        // "start countdown" command will be sent from heartbeat task).
-        let countdown = tokio::time::sleep_until(far_future);
-        tokio::pin!(countdown);
-
-        let region = &self.region;
-        let table_ident = &self.table_ident;
-        loop {
-            tokio::select! {
-                command = self.rx.recv() => {
-                    match command {
-                        Some(CountdownCommand::Start(heartbeat_interval_millis)) => {
-                            // Set first deadline in 4 heartbeats (roughly after 20 seconds from now if heartbeat
-                            // interval is set to default 5 seconds), to make Datanode and Metasrv more tolerable to
-                            // network or other jitters during startup.
-                            let first_deadline = Instant::now() + Duration::from_millis(heartbeat_interval_millis) * 4;
-                            countdown.set(tokio::time::sleep_until(first_deadline));
-                        },
-                        Some(CountdownCommand::Reset(deadline)) => {
-                            if countdown.deadline() < deadline {
-                                debug!(
-                                    "Reset deadline of region {region} of table {table_ident} to approximately {} seconds later",
-                                    (deadline - Instant::now()).as_secs_f32(),
-                                );
-                                countdown.set(tokio::time::sleep_until(deadline));
-                            }
-                            // Else the countdown could be either:
-                            // - not started yet;
-                            // - during startup protection;
-                            // - received a lagging heartbeat message.
-                            // All can be safely ignored.
-                        },
-                        None => {
-                            info!(
-                                "The handle of countdown task for region {region} of table {table_ident} \
-                                is dropped, RegionAliveKeeper out."
-                            );
-                            break;
-                        },
-                        Some(CountdownCommand::Deadline(tx)) => {
-                            let _ = tx.send(countdown.deadline());
-                        }
-                    }
-                }
-                () = &mut countdown => {
-                    let result = self.close_region().await;
-                    warn!(
-                        "Region {region} of table {table_ident} is closed, result: {result:?}. \
-                        RegionAliveKeeper out.",
-                    );
-                    break;
-                }
-            }
-        }
-    }
-
-    async fn close_region(&self) -> CloseTableResult {
-        let ctx = EngineContext::default();
-        let region = self.region;
-        let table_ident = &self.table_ident;
-        loop {
-            let request = CloseTableRequest {
-                catalog_name: table_ident.catalog.clone(),
-                schema_name: table_ident.schema.clone(),
-                table_name: table_ident.table.clone(),
-                table_id: table_ident.table_id,
-                region_numbers: vec![region],
-                flush: true,
-            };
-            match self.table_engine.close_table(&ctx, request).await {
-                Ok(result) => return result,
-                // If region is failed to close, immediately retry. Maybe we should panic instead?
-                Err(e) => error!(e;
-                    "Failed to close region {region} of table {table_ident}. \
-                    For the integrity of data, retry closing and retry without wait.",
-                ),
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::sync::atomic::{AtomicBool, Ordering};
-    use std::sync::Arc;
-
-    use api::v1::meta::{HeartbeatResponse, RegionLease};
-    use common_meta::heartbeat::mailbox::HeartbeatMailbox;
-    use datatypes::schema::RawSchema;
-    use table::engine::manager::MemoryTableEngineManager;
-    use table::engine::TableEngine;
-    use table::requests::{CreateTableRequest, TableOptions};
-    use table::test_util::EmptyTable;
-
-    use super::*;
-    use crate::remote::mock::MockTableEngine;
-
-    async fn prepare_keepers() -> (TableIdent, RegionAliveKeepers) {
-        let table_engine = Arc::new(MockTableEngine::default());
-        let table_engine_manager = Arc::new(MemoryTableEngineManager::new(table_engine));
-        let keepers = RegionAliveKeepers::new(table_engine_manager, 5000);
-
-        let catalog = "my_catalog";
-        let schema = "my_schema";
-        let table = "my_table";
-        let table_ident = TableIdent {
-            catalog: catalog.to_string(),
-            schema: schema.to_string(),
-            table: table.to_string(),
-            table_id: 1,
-            engine: "MockTableEngine".to_string(),
-        };
-        let table = Arc::new(EmptyTable::new(CreateTableRequest {
-            id: 1,
-            catalog_name: catalog.to_string(),
-            schema_name: schema.to_string(),
-            table_name: table.to_string(),
-            desc: None,
-            schema: RawSchema {
-                column_schemas: vec![],
-                timestamp_index: None,
-                version: 0,
-            },
-            region_numbers: vec![1, 2, 3],
-            primary_key_indices: vec![],
-            create_if_not_exists: false,
-            table_options: TableOptions::default(),
-            engine: "MockTableEngine".to_string(),
-        }));
-        keepers
-            .register_table(table_ident.clone(), table)
-            .await
-            .unwrap();
-        assert!(keepers.keepers.lock().await.contains_key(&table_ident));
-
-        (table_ident, keepers)
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_handle_heartbeat_response() {
-        let (table_ident, keepers) = prepare_keepers().await;
-
-        keepers.start().await;
-        let startup_protection_until = Instant::now() + Duration::from_secs(21);
-
-        let duration_since_epoch = (Instant::now() - keepers.epoch).as_millis() as _;
-        let lease_seconds = 100;
-        let response = HeartbeatResponse {
-            region_leases: vec![RegionLease {
-                table_ident: Some(table_ident.clone().into()),
-                regions: vec![1, 3], // Not extending region 2's lease time.
-                duration_since_epoch,
-                lease_seconds,
-            }],
-            ..Default::default()
-        };
-        let keep_alive_until = keepers.epoch
-            + Duration::from_millis(duration_since_epoch)
-            + Duration::from_secs(lease_seconds);
-
-        let (tx, _) = mpsc::channel(8);
-        let mailbox = Arc::new(HeartbeatMailbox::new(tx));
-        let mut ctx = HeartbeatResponseHandlerContext::new(mailbox, response);
-
-        assert!(keepers.handle(&mut ctx).await.unwrap() == HandleControl::Continue);
-
-        // sleep to wait for background task spawned in `handle`
-        tokio::time::sleep(Duration::from_secs(1)).await;
-
-        async fn test(
-            keeper: &Arc<RegionAliveKeeper>,
-            region_number: RegionNumber,
-            startup_protection_until: Instant,
-            keep_alive_until: Instant,
-            is_kept_live: bool,
-        ) {
-            let deadline = keeper.deadline(region_number).await.unwrap();
-            if is_kept_live {
-                assert!(deadline > startup_protection_until && deadline == keep_alive_until);
-            } else {
-                assert!(deadline <= startup_protection_until);
-            }
-        }
-
-        let keeper = &keepers
-            .keepers
-            .lock()
-            .await
-            .get(&table_ident)
-            .cloned()
-            .unwrap();
-
-        // Test region 1 and 3 is kept lived. Their deadlines are updated to desired instant.
-        test(keeper, 1, startup_protection_until, keep_alive_until, true).await;
-        test(keeper, 3, startup_protection_until, keep_alive_until, true).await;
-
-        // Test region 2 is not kept lived. It's deadline is not updated: still during startup protection period.
-        test(keeper, 2, startup_protection_until, keep_alive_until, false).await;
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_region_alive_keepers() {
-        let (table_ident, keepers) = prepare_keepers().await;
-
-        keepers
-            .register_region(&RegionIdent {
-                cluster_id: 1,
-                datanode_id: 1,
-                table_ident: table_ident.clone(),
-                region_number: 4,
-            })
-            .await;
-
-        keepers.start().await;
-        for keeper in keepers.keepers.lock().await.values() {
-            let regions = {
-                let handles = keeper.countdown_task_handles.lock().await;
-                handles.keys().copied().collect::<Vec<_>>()
-            };
-            for region in regions {
-                // assert countdown tasks are started
-                let deadline = keeper.deadline(region).await.unwrap();
-                assert!(deadline <= Instant::now() + Duration::from_secs(20));
-            }
-        }
-
-        keepers
-            .deregister_region(&RegionIdent {
-                cluster_id: 1,
-                datanode_id: 1,
-                table_ident: table_ident.clone(),
-                region_number: 1,
-            })
-            .await;
-        let mut regions = keepers
-            .find_keeper(&table_ident)
-            .await
-            .unwrap()
-            .countdown_task_handles
-            .lock()
-            .await
-            .keys()
-            .copied()
-            .collect::<Vec<_>>();
-        regions.sort();
-        assert_eq!(regions, vec![2, 3, 4]);
-
-        let keeper = keepers.deregister_table(&table_ident).await.unwrap();
-        assert!(Arc::try_unwrap(keeper).is_ok(), "keeper is not dropped");
-        assert!(keepers.keepers.lock().await.is_empty());
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_region_alive_keeper() {
-        let table_engine = Arc::new(MockTableEngine::default());
-        let table_ident = TableIdent {
-            catalog: "my_catalog".to_string(),
-            schema: "my_schema".to_string(),
-            table: "my_table".to_string(),
-            table_id: 1024,
-            engine: "mito".to_string(),
-        };
-        let keeper = RegionAliveKeeper::new(table_engine, table_ident, 1000);
-
-        let region = 1;
-        assert!(keeper.find_handle(&region).await.is_none());
-        keeper.register_region(region).await;
-        let _ = keeper.find_handle(&region).await.unwrap();
-
-        let ten_seconds_later = || Instant::now() + Duration::from_secs(10);
-
-        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
-        assert!(keeper.find_handle(&2).await.is_none());
-        assert!(keeper.find_handle(&3).await.is_none());
-
-        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
-        // assert if keeper is not started, keep_lived is of no use
-        assert!(keeper.deadline(region).await.unwrap() > far_future);
-
-        keeper.start().await;
-        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
-        // assert keep_lived works if keeper is started
-        assert!(keeper.deadline(region).await.unwrap() <= ten_seconds_later());
-
-        let handle = keeper.deregister_region(region).await.unwrap();
-        assert!(Arc::try_unwrap(handle).is_ok(), "handle is not dropped");
-        assert!(keeper.find_handle(&region).await.is_none());
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_countdown_task_handle() {
-        let table_engine = Arc::new(MockTableEngine::default());
-        let table_ident = TableIdent {
-            catalog: "my_catalog".to_string(),
-            schema: "my_schema".to_string(),
-            table: "my_table".to_string(),
-            table_id: 1024,
-            engine: "mito".to_string(),
-        };
-        let finished = Arc::new(AtomicBool::new(false));
-        let finished_clone = finished.clone();
-        let handle = CountdownTaskHandle::new(
-            table_engine.clone(),
-            table_ident.clone(),
-            1,
-            || async move { finished_clone.store(true, Ordering::Relaxed) },
-        );
-        let tx = handle.tx.clone();
-
-        // assert countdown task is running
-        tx.send(CountdownCommand::Start(5000)).await.unwrap();
-        assert!(!finished.load(Ordering::Relaxed));
-
-        drop(handle);
-        tokio::time::sleep(Duration::from_secs(1)).await;
-
-        // assert countdown task is stopped
-        assert!(tx
-            .try_send(CountdownCommand::Reset(
-                Instant::now() + Duration::from_secs(10)
-            ))
-            .is_err());
-        // assert `on_task_finished` is not called (because the task is aborted by the handle's drop)
-        assert!(!finished.load(Ordering::Relaxed));
-
-        let finished = Arc::new(AtomicBool::new(false));
-        let finished_clone = finished.clone();
-        let handle = CountdownTaskHandle::new(table_engine, table_ident, 1, || async move {
-            finished_clone.store(true, Ordering::Relaxed)
-        });
-        handle.tx.send(CountdownCommand::Start(100)).await.unwrap();
-        tokio::time::sleep(Duration::from_secs(1)).await;
-        // assert `on_task_finished` is called when task is finished normally
-        assert!(finished.load(Ordering::Relaxed));
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_countdown_task_run() {
-        let ctx = &EngineContext::default();
-        let catalog = "my_catalog";
-        let schema = "my_schema";
-        let table = "my_table";
-        let table_id = 1;
-        let request = CreateTableRequest {
-            id: table_id,
-            catalog_name: catalog.to_string(),
-            schema_name: schema.to_string(),
-            table_name: table.to_string(),
-            desc: None,
-            schema: RawSchema {
-                column_schemas: vec![],
-                timestamp_index: None,
-                version: 0,
-            },
-            region_numbers: vec![],
-            primary_key_indices: vec![],
-            create_if_not_exists: false,
-            table_options: TableOptions::default(),
-            engine: "mito".to_string(),
-        };
-
-        let table_engine = Arc::new(MockTableEngine::default());
-        let _ = table_engine.create_table(ctx, request).await.unwrap();
-
-        let table_ident = TableIdent {
-            catalog: catalog.to_string(),
-            schema: schema.to_string(),
-            table: table.to_string(),
-            table_id,
-            engine: "mito".to_string(),
-        };
-        let (tx, rx) = mpsc::channel(10);
-        let mut task = CountdownTask {
-            table_engine: table_engine.clone(),
-            table_ident,
-            region: 1,
-            rx,
-        };
-        let _handle = common_runtime::spawn_bg(async move {
-            task.run().await;
-        });
-
-        async fn deadline(tx: &mpsc::Sender<CountdownCommand>) -> Instant {
-            let (s, r) = oneshot::channel();
-            tx.send(CountdownCommand::Deadline(s)).await.unwrap();
-            r.await.unwrap()
-        }
-
-        // if countdown task is not started, its deadline is set to far future
-        assert!(deadline(&tx).await > Instant::now() + Duration::from_secs(86400 * 365 * 29));
-
-        // start countdown in 250ms * 4 = 1s
-        tx.send(CountdownCommand::Start(250)).await.unwrap();
-        // assert deadline is correctly set
-        assert!(deadline(&tx).await <= Instant::now() + Duration::from_secs(1));
-
-        // reset countdown in 1.5s
-        tx.send(CountdownCommand::Reset(
-            Instant::now() + Duration::from_millis(1500),
-        ))
-        .await
-        .unwrap();
-
-        // assert the table is closed after deadline is reached
-        assert!(table_engine.table_exists(ctx, table_id));
-        // spare 500ms for the task to close the table
-        tokio::time::sleep(Duration::from_millis(2000)).await;
-        assert!(!table_engine.table_exists(ctx, table_id));
-    }
-}

src/catalog/src/system.rs

@@ -1,606 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use common_catalog::consts::{
-    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
-    SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
-};
-use common_recordbatch::SendableRecordBatchStream;
-use common_telemetry::debug;
-use common_time::util;
-use datatypes::prelude::{ConcreteDataType, ScalarVector, VectorRef};
-use datatypes::schema::{ColumnSchema, RawSchema, SchemaRef};
-use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
-use serde::{Deserialize, Serialize};
-use snafu::{ensure, OptionExt, ResultExt};
-use store_api::storage::ScanRequest;
-use table::engine::{EngineContext, TableEngineRef};
-use table::metadata::{TableId, TableInfoRef};
-use table::requests::{
-    CreateTableRequest, DeleteRequest, InsertRequest, OpenTableRequest, TableOptions,
-};
-use table::{Result as TableResult, Table, TableRef};
-
-use crate::error::{
-    self, CreateSystemCatalogSnafu, EmptyValueSnafu, Error, InvalidEntryTypeSnafu, InvalidKeySnafu,
-    OpenSystemCatalogSnafu, Result, ValueDeserializeSnafu,
-};
-use crate::DeregisterTableRequest;
-
-pub const ENTRY_TYPE_INDEX: usize = 0;
-pub const KEY_INDEX: usize = 1;
-pub const VALUE_INDEX: usize = 3;
-
-pub struct SystemCatalogTable(TableRef);
-
-#[async_trait::async_trait]
-impl Table for SystemCatalogTable {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.0.schema()
-    }
-
-    async fn scan_to_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
-        self.0.scan_to_stream(request).await
-    }
-
-    /// Insert values into table.
-    async fn insert(&self, request: InsertRequest) -> TableResult<usize> {
-        self.0.insert(request).await
-    }
-
-    fn table_info(&self) -> TableInfoRef {
-        self.0.table_info()
-    }
-
-    async fn delete(&self, request: DeleteRequest) -> TableResult<usize> {
-        self.0.delete(request).await
-    }
-
-    fn statistics(&self) -> Option<table::stats::TableStatistics> {
-        self.0.statistics()
-    }
-}
-
-impl SystemCatalogTable {
-    pub async fn new(engine: TableEngineRef) -> Result<Self> {
-        let request = OpenTableRequest {
-            catalog_name: SYSTEM_CATALOG_NAME.to_string(),
-            schema_name: INFORMATION_SCHEMA_NAME.to_string(),
-            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
-            table_id: SYSTEM_CATALOG_TABLE_ID,
-            region_numbers: vec![0],
-        };
-        let schema = build_system_catalog_schema();
-        let ctx = EngineContext::default();
-
-        if let Some(table) = engine
-            .open_table(&ctx, request)
-            .await
-            .context(OpenSystemCatalogSnafu)?
-        {
-            Ok(Self(table))
-        } else {
-            // system catalog table is not yet created, try to create
-            let request = CreateTableRequest {
-                id: SYSTEM_CATALOG_TABLE_ID,
-                catalog_name: SYSTEM_CATALOG_NAME.to_string(),
-                schema_name: INFORMATION_SCHEMA_NAME.to_string(),
-                table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
-                desc: Some("System catalog table".to_string()),
-                schema,
-                region_numbers: vec![0],
-                primary_key_indices: vec![ENTRY_TYPE_INDEX, KEY_INDEX],
-                create_if_not_exists: true,
-                table_options: TableOptions::default(),
-                engine: engine.name().to_string(),
-            };
-
-            let table = engine
-                .create_table(&ctx, request)
-                .await
-                .context(CreateSystemCatalogSnafu)?;
-            Ok(Self(table))
-        }
-    }
-
-    /// Create a stream of all entries inside system catalog table
-    pub async fn records(&self) -> Result<SendableRecordBatchStream> {
-        let full_projection = None;
-        let scan_req = ScanRequest {
-            sequence: None,
-            projection: full_projection,
-            filters: vec![],
-            output_ordering: None,
-            limit: None,
-        };
-        let stream = self
-            .scan_to_stream(scan_req)
-            .await
-            .context(error::SystemCatalogTableScanSnafu)?;
-        Ok(stream)
-    }
-}
-
-/// Build system catalog table schema.
-/// A system catalog table consists of 6 columns, namely
-/// - entry_type: type of entry in current row, can be any variant of [EntryType].
-/// - key: a binary encoded key of entry, differs according to different entry type.
-/// - timestamp: currently not used.
-/// - value: JSON-encoded value of entry's metadata.
-/// - gmt_created: create time of this metadata.
-/// - gmt_modified: last updated time of this metadata.
-fn build_system_catalog_schema() -> RawSchema {
-    let cols = vec![
-        ColumnSchema::new(
-            "entry_type".to_string(),
-            ConcreteDataType::uint8_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "key".to_string(),
-            ConcreteDataType::binary_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "timestamp".to_string(),
-            ConcreteDataType::timestamp_millisecond_datatype(),
-            false,
-        )
-        .with_time_index(true),
-        ColumnSchema::new(
-            "value".to_string(),
-            ConcreteDataType::binary_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "gmt_created".to_string(),
-            ConcreteDataType::timestamp_millisecond_datatype(),
-            false,
-        ),
-        ColumnSchema::new(
-            "gmt_modified".to_string(),
-            ConcreteDataType::timestamp_millisecond_datatype(),
-            false,
-        ),
-    ];
-
-    RawSchema::new(cols)
-}
-
-/// Formats key string for table entry in system catalog
-#[inline]
-pub fn format_table_entry_key(catalog: &str, schema: &str, table_id: TableId) -> String {
-    format!("{catalog}.{schema}.{table_id}")
-}
-
-pub fn build_table_insert_request(
-    catalog: String,
-    schema: String,
-    table_name: String,
-    table_id: TableId,
-    engine: String,
-) -> InsertRequest {
-    let entry_key = format_table_entry_key(&catalog, &schema, table_id);
-    build_insert_request(
-        EntryType::Table,
-        entry_key.as_bytes(),
-        serde_json::to_string(&TableEntryValue {
-            table_name,
-            engine,
-            is_deleted: false,
-        })
-        .unwrap()
-        .as_bytes(),
-    )
-}
-
-pub(crate) fn build_table_deletion_request(
-    request: &DeregisterTableRequest,
-    table_id: TableId,
-) -> InsertRequest {
-    let entry_key = format_table_entry_key(&request.catalog, &request.schema, table_id);
-    build_insert_request(
-        EntryType::Table,
-        entry_key.as_bytes(),
-        serde_json::to_string(&TableEntryValue {
-            table_name: "".to_string(),
-            engine: "".to_string(),
-            is_deleted: true,
-        })
-        .unwrap()
-        .as_bytes(),
-    )
-}
-
-fn build_primary_key_columns(entry_type: EntryType, key: &[u8]) -> HashMap<String, VectorRef> {
-    HashMap::from([
-        (
-            "entry_type".to_string(),
-            Arc::new(UInt8Vector::from_slice([entry_type as u8])) as VectorRef,
-        ),
-        (
-            "key".to_string(),
-            Arc::new(BinaryVector::from_slice(&[key])) as VectorRef,
-        ),
-        (
-            "timestamp".to_string(),
-            // Timestamp in key part is intentionally left to 0
-            Arc::new(TimestampMillisecondVector::from_slice([0])) as VectorRef,
-        ),
-    ])
-}
-
-pub fn build_schema_insert_request(catalog_name: String, schema_name: String) -> InsertRequest {
-    let full_schema_name = format!("{catalog_name}.{schema_name}");
-    build_insert_request(
-        EntryType::Schema,
-        full_schema_name.as_bytes(),
-        serde_json::to_string(&SchemaEntryValue {})
-            .unwrap()
-            .as_bytes(),
-    )
-}
-
-pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) -> InsertRequest {
-    let primary_key_columns = build_primary_key_columns(entry_type, key);
-
-    let mut columns_values = HashMap::with_capacity(6);
-    columns_values.extend(primary_key_columns.into_iter());
-
-    let _ = columns_values.insert(
-        "value".to_string(),
-        Arc::new(BinaryVector::from_slice(&[value])) as _,
-    );
-
-    let now = util::current_time_millis();
-    let _ = columns_values.insert(
-        "gmt_created".to_string(),
-        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
-    );
-
-    let _ = columns_values.insert(
-        "gmt_modified".to_string(),
-        Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
-    );
-
-    InsertRequest {
-        catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-        schema_name: DEFAULT_SCHEMA_NAME.to_string(),
-        table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
-        columns_values,
-        region_number: 0, // system catalog table has only one region
-    }
-}
-
-pub fn decode_system_catalog(
-    entry_type: Option<u8>,
-    key: Option<&[u8]>,
-    value: Option<&[u8]>,
-) -> Result<Entry> {
-    debug!(
-        "Decode system catalog entry: {:?}, {:?}, {:?}",
-        entry_type, key, value
-    );
-    let entry_type = entry_type.context(InvalidKeySnafu { key: None })?;
-    let key = String::from_utf8_lossy(key.context(InvalidKeySnafu { key: None })?);
-
-    match EntryType::try_from(entry_type)? {
-        EntryType::Catalog => {
-            // As for catalog entry, the key is a string with format: `<catalog_name>`
-            // and the value is current not used.
-            let catalog_name = key.to_string();
-            Ok(Entry::Catalog(CatalogEntry { catalog_name }))
-        }
-        EntryType::Schema => {
-            // As for schema entry, the key is a string with format: `<catalog_name>.<schema_name>`
-            // and the value is current not used.
-            let schema_parts = key.split('.').collect::<Vec<_>>();
-            ensure!(
-                schema_parts.len() == 2,
-                InvalidKeySnafu {
-                    key: Some(key.to_string())
-                }
-            );
-            Ok(Entry::Schema(SchemaEntry {
-                catalog_name: schema_parts[0].to_string(),
-                schema_name: schema_parts[1].to_string(),
-            }))
-        }
-
-        EntryType::Table => {
-            // As for table entry, the key is a string with format: `<catalog_name>.<schema_name>.<table_id>`
-            // and the value is a JSON string with format: `{"table_name": <table_name>}`
-            let table_parts = key.split('.').collect::<Vec<_>>();
-            ensure!(
-                table_parts.len() >= 3,
-                InvalidKeySnafu {
-                    key: Some(key.to_string())
-                }
-            );
-            let value = value.context(EmptyValueSnafu)?;
-            debug!("Table meta value: {}", String::from_utf8_lossy(value));
-            let table_meta: TableEntryValue =
-                serde_json::from_slice(value).context(ValueDeserializeSnafu)?;
-            let table_id = table_parts[2].parse::<TableId>().unwrap();
-            Ok(Entry::Table(TableEntry {
-                catalog_name: table_parts[0].to_string(),
-                schema_name: table_parts[1].to_string(),
-                table_name: table_meta.table_name,
-                table_id,
-                engine: table_meta.engine,
-                is_deleted: table_meta.is_deleted,
-            }))
-        }
-    }
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub enum EntryType {
-    Catalog = 1,
-    Schema = 2,
-    Table = 3,
-}
-
-impl TryFrom<u8> for EntryType {
-    type Error = Error;
-
-    fn try_from(value: u8) -> std::result::Result<Self, Self::Error> {
-        match value {
-            b if b == Self::Catalog as u8 => Ok(Self::Catalog),
-            b if b == Self::Schema as u8 => Ok(Self::Schema),
-            b if b == Self::Table as u8 => Ok(Self::Table),
-            b => InvalidEntryTypeSnafu {
-                entry_type: Some(b),
-            }
-            .fail(),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub enum Entry {
-    Catalog(CatalogEntry),
-    Schema(SchemaEntry),
-    Table(TableEntry),
-}
-
-#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub struct CatalogEntry {
-    pub catalog_name: String,
-}
-
-#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub struct SchemaEntry {
-    pub catalog_name: String,
-    pub schema_name: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub struct SchemaEntryValue;
-
-#[derive(Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub struct TableEntry {
-    pub catalog_name: String,
-    pub schema_name: String,
-    pub table_name: String,
-    pub table_id: TableId,
-    pub engine: String,
-    pub is_deleted: bool,
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-pub struct TableEntryValue {
-    pub table_name: String,
-
-    #[serde(default = "mito_engine")]
-    pub engine: String,
-
-    #[serde(default = "not_deleted")]
-    pub is_deleted: bool,
-}
-
-fn mito_engine() -> String {
-    MITO_ENGINE.to_string()
-}
-
-fn not_deleted() -> bool {
-    false
-}
-
-#[cfg(test)]
-mod tests {
-    use common_recordbatch::RecordBatches;
-    use common_test_util::temp_dir::{create_temp_dir, TempDir};
-    use datatypes::value::Value;
-    use log_store::NoopLogStore;
-    use mito::config::EngineConfig;
-    use mito::engine::{MitoEngine, MITO_ENGINE};
-    use object_store::ObjectStore;
-    use storage::compaction::noop::NoopCompactionScheduler;
-    use storage::config::EngineConfig as StorageEngineConfig;
-    use storage::EngineImpl;
-    use table::metadata::TableType;
-    use table::metadata::TableType::Base;
-
-    use super::*;
-
-    #[test]
-    pub fn test_decode_catalog_entry() {
-        let entry = decode_system_catalog(
-            Some(EntryType::Catalog as u8),
-            Some("some_catalog".as_bytes()),
-            None,
-        )
-        .unwrap();
-        if let Entry::Catalog(e) = entry {
-            assert_eq!("some_catalog", e.catalog_name);
-        } else {
-            panic!("Unexpected type: {entry:?}");
-        }
-    }
-
-    #[test]
-    pub fn test_decode_schema_entry() {
-        let entry = decode_system_catalog(
-            Some(EntryType::Schema as u8),
-            Some("some_catalog.some_schema".as_bytes()),
-            None,
-        )
-        .unwrap();
-
-        if let Entry::Schema(e) = entry {
-            assert_eq!("some_catalog", e.catalog_name);
-            assert_eq!("some_schema", e.schema_name);
-        } else {
-            panic!("Unexpected type: {entry:?}");
-        }
-    }
-
-    #[test]
-    pub fn test_decode_table() {
-        let entry = decode_system_catalog(
-            Some(EntryType::Table as u8),
-            Some("some_catalog.some_schema.42".as_bytes()),
-            Some("{\"table_name\":\"some_table\"}".as_bytes()),
-        )
-        .unwrap();
-
-        if let Entry::Table(e) = entry {
-            assert_eq!("some_catalog", e.catalog_name);
-            assert_eq!("some_schema", e.schema_name);
-            assert_eq!("some_table", e.table_name);
-            assert_eq!(42, e.table_id);
-        } else {
-            panic!("Unexpected type: {entry:?}");
-        }
-    }
-
-    #[test]
-    pub fn test_decode_mismatch() {
-        assert!(decode_system_catalog(
-            Some(EntryType::Table as u8),
-            Some("some_catalog.some_schema.42".as_bytes()),
-            None,
-        )
-        .is_err());
-    }
-
-    #[test]
-    pub fn test_entry_type() {
-        assert_eq!(EntryType::Catalog, EntryType::try_from(1).unwrap());
-        assert_eq!(EntryType::Schema, EntryType::try_from(2).unwrap());
-        assert_eq!(EntryType::Table, EntryType::try_from(3).unwrap());
-        assert!(EntryType::try_from(4).is_err());
-    }
-
-    pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) {
-        let dir = create_temp_dir("system-table-test");
-        let store_dir = dir.path().to_string_lossy();
-        let mut builder = object_store::services::Fs::default();
-        let _ = builder.root(&store_dir);
-        let object_store = ObjectStore::new(builder).unwrap().finish();
-        let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
-        let table_engine = Arc::new(MitoEngine::new(
-            EngineConfig::default(),
-            EngineImpl::new(
-                StorageEngineConfig::default(),
-                Arc::new(NoopLogStore::default()),
-                object_store.clone(),
-                noop_compaction_scheduler,
-            )
-            .unwrap(),
-            object_store,
-        ));
-        (dir, table_engine)
-    }
-
-    #[tokio::test]
-    async fn test_system_table_type() {
-        let (_dir, table_engine) = prepare_table_engine().await;
-        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
-        assert_eq!(Base, system_table.table_type());
-    }
-
-    #[tokio::test]
-    async fn test_system_table_info() {
-        let (_dir, table_engine) = prepare_table_engine().await;
-        let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
-        let info = system_table.table_info();
-        assert_eq!(TableType::Base, info.table_type);
-        assert_eq!(SYSTEM_CATALOG_TABLE_NAME, info.name);
-        assert_eq!(SYSTEM_CATALOG_TABLE_ID, info.ident.table_id);
-        assert_eq!(SYSTEM_CATALOG_NAME, info.catalog_name);
-        assert_eq!(INFORMATION_SCHEMA_NAME, info.schema_name);
-    }
-
-    #[tokio::test]
-    async fn test_system_catalog_table_records() {
-        let (_, table_engine) = prepare_table_engine().await;
-        let catalog_table = SystemCatalogTable::new(table_engine).await.unwrap();
-
-        let table_insertion = build_table_insert_request(
-            DEFAULT_CATALOG_NAME.to_string(),
-            DEFAULT_SCHEMA_NAME.to_string(),
-            "my_table".to_string(),
-            1,
-            MITO_ENGINE.to_string(),
-        );
-        let result = catalog_table.insert(table_insertion).await.unwrap();
-        assert_eq!(result, 1);
-
-        let records = catalog_table.records().await.unwrap();
-        let mut batches = RecordBatches::try_collect(records).await.unwrap().take();
-        assert_eq!(batches.len(), 1);
-        let batch = batches.remove(0);
-        assert_eq!(batch.num_rows(), 1);
-
-        let row = batch.rows().next().unwrap();
-        let Value::UInt8(entry_type) = row[0] else { unreachable!() };
-        let Value::Binary(key) = row[1].clone() else { unreachable!() };
-        let Value::Binary(value) = row[3].clone() else { unreachable!() };
-        let entry = decode_system_catalog(Some(entry_type), Some(&*key), Some(&*value)).unwrap();
-        let expected = Entry::Table(TableEntry {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: "my_table".to_string(),
-            table_id: 1,
-            engine: MITO_ENGINE.to_string(),
-            is_deleted: false,
-        });
-        assert_eq!(entry, expected);
-
-        let table_deletion = build_table_deletion_request(
-            &DeregisterTableRequest {
-                catalog: DEFAULT_CATALOG_NAME.to_string(),
-                schema: DEFAULT_SCHEMA_NAME.to_string(),
-                table_name: "my_table".to_string(),
-            },
-            1,
-        );
-        let result = catalog_table.insert(table_deletion).await.unwrap();
-        assert_eq!(result, 1);
-
-        let records = catalog_table.records().await.unwrap();
-        let batches = RecordBatches::try_collect(records).await.unwrap().take();
-        assert_eq!(batches.len(), 1);
-    }
-}

src/catalog/src/table_source.rs

@@ -45,8 +45,8 @@ impl DfTableSourceProvider {
             catalog_manager,
             disallow_cross_schema_query,
             resolved_tables: HashMap::new(),
-            default_catalog: query_ctx.current_catalog(),
-            default_schema: query_ctx.current_schema(),
+            default_catalog: query_ctx.current_catalog().to_owned(),
+            default_schema: query_ctx.current_schema().to_owned(),
         }
     }
 
@@ -123,14 +123,14 @@ mod tests {
     use session::context::QueryContext;
 
     use super::*;
-    use crate::local::MemoryCatalogManager;
+    use crate::memory::MemoryCatalogManager;
 
     #[test]
     fn test_validate_table_ref() {
         let query_ctx = &QueryContext::with("greptime", "public");
 
         let table_provider =
-            DfTableSourceProvider::new(Arc::new(MemoryCatalogManager::default()), true, query_ctx);
+            DfTableSourceProvider::new(MemoryCatalogManager::with_default_setup(), true, query_ctx);
 
         let table_ref = TableReference::Bare {
             table: Cow::Borrowed("table_name"),

src/catalog/src/tables.rs

@@ -1,101 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The `tables` table in system catalog keeps a record of all tables created by user.
-
-use std::sync::Arc;
-
-use common_telemetry::logging;
-use snafu::ResultExt;
-use table::metadata::TableId;
-use table::Table;
-
-use crate::error::{self, InsertCatalogRecordSnafu, Result as CatalogResult};
-use crate::system::{
-    build_schema_insert_request, build_table_deletion_request, build_table_insert_request,
-    SystemCatalogTable,
-};
-use crate::DeregisterTableRequest;
-
-pub struct InformationSchema {
-    pub system: Arc<SystemCatalogTable>,
-}
-
-pub struct SystemCatalog {
-    pub information_schema: Arc<InformationSchema>,
-}
-
-impl SystemCatalog {
-    pub(crate) fn new(system: SystemCatalogTable) -> Self {
-        let schema = InformationSchema {
-            system: Arc::new(system),
-        };
-        Self {
-            information_schema: Arc::new(schema),
-        }
-    }
-
-    pub async fn register_table(
-        &self,
-        catalog: String,
-        schema: String,
-        table_name: String,
-        table_id: TableId,
-        engine: String,
-    ) -> crate::error::Result<usize> {
-        let request = build_table_insert_request(catalog, schema, table_name, table_id, engine);
-        self.information_schema
-            .system
-            .insert(request)
-            .await
-            .context(InsertCatalogRecordSnafu)
-    }
-
-    pub(crate) async fn deregister_table(
-        &self,
-        request: &DeregisterTableRequest,
-        table_id: TableId,
-    ) -> CatalogResult<()> {
-        self.information_schema
-            .system
-            .insert(build_table_deletion_request(request, table_id))
-            .await
-            .map(|x| {
-                if x != 1 {
-                    let table = common_catalog::format_full_table_name(
-                        &request.catalog,
-                        &request.schema,
-                        &request.table_name
-                    );
-                    logging::warn!("Failed to delete table record from information_schema, unexpected returned result: {x}, table: {table}");
-                }
-            })
-            .with_context(|_| error::DeregisterTableSnafu {
-                request: request.clone(),
-            })
-    }
-
-    pub async fn register_schema(
-        &self,
-        catalog: String,
-        schema: String,
-    ) -> crate::error::Result<usize> {
-        let request = build_schema_insert_request(catalog, schema);
-        self.information_schema
-            .system
-            .insert(request)
-            .await
-            .context(InsertCatalogRecordSnafu)
-    }
-}

src/catalog/tests/local_catalog_tests.rs

@@ -1,176 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use catalog::local::LocalCatalogManager;
-    use catalog::{CatalogManager, RegisterTableRequest, RenameTableRequest};
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-    use common_telemetry::{error, info};
-    use common_test_util::temp_dir::TempDir;
-    use mito::config::EngineConfig;
-    use table::engine::manager::MemoryTableEngineManager;
-    use table::table::numbers::NumbersTable;
-    use table::TableRef;
-    use tokio::sync::Mutex;
-
-    async fn create_local_catalog_manager(
-    ) -> Result<(TempDir, LocalCatalogManager), catalog::error::Error> {
-        let (dir, object_store) =
-            mito::table::test_util::new_test_object_store("setup_mock_engine_and_table").await;
-        let mock_engine = Arc::new(mito::table::test_util::MockMitoEngine::new(
-            EngineConfig::default(),
-            mito::table::test_util::MockEngine::default(),
-            object_store,
-        ));
-        let engine_manager = Arc::new(MemoryTableEngineManager::new(mock_engine.clone()));
-        let catalog_manager = LocalCatalogManager::try_new(engine_manager).await.unwrap();
-        catalog_manager.start().await?;
-        Ok((dir, catalog_manager))
-    }
-
-    #[tokio::test]
-    async fn test_rename_table() {
-        common_telemetry::init_default_ut_logging();
-        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
-        // register table
-        let table_name = "test_table";
-        let table_id = 42;
-        let table = Arc::new(NumbersTable::new(table_id));
-        let request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            table_id,
-            table: table.clone(),
-        };
-        assert!(catalog_manager.register_table(request).await.unwrap());
-
-        // rename table
-        let new_table_name = "table_t";
-        let rename_table_req = RenameTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: table_name.to_string(),
-            new_table_name: new_table_name.to_string(),
-            table_id,
-        };
-        assert!(catalog_manager
-            .rename_table(rename_table_req)
-            .await
-            .unwrap());
-
-        let registered_table = catalog_manager
-            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(registered_table.table_info().ident.table_id, table_id);
-    }
-
-    #[tokio::test]
-    async fn test_duplicate_register() {
-        let (_dir, catalog_manager) = create_local_catalog_manager().await.unwrap();
-        let request = RegisterTableRequest {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: "test_table".to_string(),
-            table_id: 42,
-            table: Arc::new(NumbersTable::new(42)),
-        };
-        assert!(catalog_manager
-            .register_table(request.clone())
-            .await
-            .unwrap());
-
-        // register table with same table id will succeed with 0 as return val.
-        assert!(!catalog_manager.register_table(request).await.unwrap());
-
-        let err = catalog_manager
-            .register_table(RegisterTableRequest {
-                catalog: DEFAULT_CATALOG_NAME.to_string(),
-                schema: DEFAULT_SCHEMA_NAME.to_string(),
-                table_name: "test_table".to_string(),
-                table_id: 43,
-                table: Arc::new(NumbersTable::new(43)),
-            })
-            .await
-            .unwrap_err();
-        assert!(
-            err.to_string()
-                .contains("Table `greptime.public.test_table` already exists"),
-            "Actual error message: {err}",
-        );
-    }
-
-    #[test]
-    fn test_concurrent_register() {
-        common_telemetry::init_default_ut_logging();
-        let rt = Arc::new(tokio::runtime::Builder::new_multi_thread().build().unwrap());
-        let (_dir, catalog_manager) =
-            rt.block_on(async { create_local_catalog_manager().await.unwrap() });
-        let catalog_manager = Arc::new(catalog_manager);
-
-        let succeed: Arc<Mutex<Option<TableRef>>> = Arc::new(Mutex::new(None));
-
-        let mut handles = Vec::with_capacity(8);
-        for i in 0..8 {
-            let catalog = catalog_manager.clone();
-            let succeed = succeed.clone();
-            let handle = rt.spawn(async move {
-                let table_id = 42 + i;
-                let table = Arc::new(NumbersTable::new(table_id));
-                let req = RegisterTableRequest {
-                    catalog: DEFAULT_CATALOG_NAME.to_string(),
-                    schema: DEFAULT_SCHEMA_NAME.to_string(),
-                    table_name: "test_table".to_string(),
-                    table_id,
-                    table: table.clone(),
-                };
-                match catalog.register_table(req).await {
-                    Ok(res) => {
-                        if res {
-                            let mut succeed = succeed.lock().await;
-                            info!("Successfully registered table: {}", table_id);
-                            *succeed = Some(table);
-                        }
-                    }
-                    Err(_) => {
-                        error!("Failed to register table {}", table_id);
-                    }
-                }
-            });
-            handles.push(handle);
-        }
-
-        rt.block_on(async move {
-            for handle in handles {
-                handle.await.unwrap();
-            }
-            let guard = succeed.lock().await;
-            let table = guard.as_ref().unwrap();
-            let table_registered = catalog_manager
-                .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "test_table")
-                .await
-                .unwrap()
-                .unwrap();
-            assert_eq!(
-                table_registered.table_info().ident.table_id,
-                table.table_info().ident.table_id
-            );
-        });
-    }
-}

src/catalog/tests/remote_catalog_tests.rs

@@ -1,483 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#![feature(assert_matches)]
-
-#[cfg(test)]
-mod tests {
-    use std::assert_matches::assert_matches;
-    use std::collections::HashSet;
-    use std::sync::Arc;
-    use std::time::Duration;
-
-    use catalog::error::Error;
-    use catalog::remote::mock::MockTableEngine;
-    use catalog::remote::region_alive_keeper::RegionAliveKeepers;
-    use catalog::remote::{CachedMetaKvBackend, RemoteCatalogManager};
-    use catalog::{CatalogManager, RegisterSchemaRequest, RegisterTableRequest};
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
-    use common_meta::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
-    use common_meta::ident::TableIdent;
-    use common_meta::key::TableMetadataManager;
-    use common_meta::kv_backend::memory::MemoryKvBackend;
-    use common_meta::kv_backend::KvBackend;
-    use common_meta::rpc::store::{CompareAndPutRequest, PutRequest, RangeRequest};
-    use datatypes::schema::RawSchema;
-    use table::engine::manager::{MemoryTableEngineManager, TableEngineManagerRef};
-    use table::engine::{EngineContext, TableEngineRef};
-    use table::requests::CreateTableRequest;
-    use table::test_util::EmptyTable;
-    use tokio::time::Instant;
-
-    struct TestingComponents {
-        catalog_manager: Arc<RemoteCatalogManager>,
-        table_engine_manager: TableEngineManagerRef,
-        region_alive_keepers: Arc<RegionAliveKeepers>,
-    }
-
-    impl TestingComponents {
-        fn table_engine(&self) -> TableEngineRef {
-            self.table_engine_manager.engine(MITO_ENGINE).unwrap()
-        }
-    }
-
-    #[tokio::test]
-    async fn test_backend() {
-        let backend = MemoryKvBackend::<Error>::default();
-
-        let default_catalog_key = CatalogKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-        }
-        .to_string();
-        let req = PutRequest::new()
-            .with_key(default_catalog_key.as_bytes())
-            .with_value(CatalogValue.as_bytes().unwrap());
-        backend.put(req).await.unwrap();
-
-        let schema_key = SchemaKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
-        }
-        .to_string();
-        let req = PutRequest::new()
-            .with_key(schema_key.as_bytes())
-            .with_value(SchemaValue.as_bytes().unwrap());
-        backend.put(req).await.unwrap();
-
-        let req = RangeRequest::new().with_prefix(b"__c-".to_vec());
-        let res = backend
-            .range(req)
-            .await
-            .unwrap()
-            .kvs
-            .into_iter()
-            .map(|kv| String::from_utf8_lossy(kv.key()).to_string());
-        assert_eq!(
-            vec!["__c-greptime".to_string()],
-            res.into_iter().collect::<Vec<_>>()
-        );
-    }
-
-    #[tokio::test]
-    async fn test_cached_backend() {
-        let backend = CachedMetaKvBackend::wrap(Arc::new(MemoryKvBackend::default()));
-
-        let default_catalog_key = CatalogKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-        }
-        .to_string();
-        let req = PutRequest::new()
-            .with_key(default_catalog_key.as_bytes())
-            .with_value(CatalogValue.as_bytes().unwrap());
-        backend.put(req).await.unwrap();
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        let _ = ret.unwrap();
-
-        let req = CompareAndPutRequest::new()
-            .with_key(b"__c-greptime".to_vec())
-            .with_expect(CatalogValue.as_bytes().unwrap())
-            .with_value(b"123".to_vec());
-        let _ = backend.compare_and_put(req).await.unwrap();
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert_eq!(b"123", ret.as_ref().unwrap().value.as_slice());
-
-        let req = PutRequest::new()
-            .with_key(b"__c-greptime".to_vec())
-            .with_value(b"1234".to_vec());
-        let _ = backend.put(req).await;
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert_eq!(b"1234", ret.unwrap().value.as_slice());
-
-        backend.delete(b"__c-greptime", false).await.unwrap();
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert!(ret.is_none());
-    }
-
-    async fn prepare_components(node_id: u64) -> TestingComponents {
-        let backend = Arc::new(MemoryKvBackend::default());
-
-        let req = PutRequest::new()
-            .with_key(b"__c-greptime".to_vec())
-            .with_value(b"".to_vec());
-        backend.put(req).await.unwrap();
-
-        let req = PutRequest::new()
-            .with_key(b"__s-greptime-public".to_vec())
-            .with_value(b"".to_vec());
-        backend.put(req).await.unwrap();
-
-        let cached_backend = Arc::new(CachedMetaKvBackend::wrap(backend));
-
-        let table_engine = Arc::new(MockTableEngine::default());
-        let engine_manager = Arc::new(MemoryTableEngineManager::alias(
-            MITO_ENGINE.to_string(),
-            table_engine,
-        ));
-
-        let region_alive_keepers = Arc::new(RegionAliveKeepers::new(engine_manager.clone(), 5000));
-
-        let catalog_manager = RemoteCatalogManager::new(
-            engine_manager.clone(),
-            node_id,
-            cached_backend.clone(),
-            region_alive_keepers.clone(),
-            Arc::new(TableMetadataManager::new(cached_backend)),
-        );
-        catalog_manager.start().await.unwrap();
-
-        TestingComponents {
-            catalog_manager: Arc::new(catalog_manager),
-            table_engine_manager: engine_manager,
-            region_alive_keepers,
-        }
-    }
-
-    #[tokio::test]
-    async fn test_remote_catalog_default() {
-        common_telemetry::init_default_ut_logging();
-        let node_id = 42;
-        let TestingComponents {
-            catalog_manager, ..
-        } = prepare_components(node_id).await;
-        assert_eq!(
-            vec![DEFAULT_CATALOG_NAME.to_string()],
-            catalog_manager.catalog_names().await.unwrap()
-        );
-
-        assert_eq!(
-            vec![DEFAULT_SCHEMA_NAME.to_string()],
-            catalog_manager
-                .schema_names(DEFAULT_CATALOG_NAME)
-                .await
-                .unwrap()
-        );
-    }
-
-    #[tokio::test]
-    async fn test_remote_catalog_register_nonexistent() {
-        common_telemetry::init_default_ut_logging();
-        let node_id = 42;
-        let components = prepare_components(node_id).await;
-
-        // register a new table with an nonexistent catalog
-        let catalog_name = "nonexistent_catalog".to_string();
-        let schema_name = "nonexistent_schema".to_string();
-        let table_name = "fail_table".to_string();
-        // this schema has no effect
-        let table_schema = RawSchema::new(vec![]);
-        let table = components
-            .table_engine()
-            .create_table(
-                &EngineContext {},
-                CreateTableRequest {
-                    id: 1,
-                    catalog_name: catalog_name.clone(),
-                    schema_name: schema_name.clone(),
-                    table_name: table_name.clone(),
-                    desc: None,
-                    schema: table_schema,
-                    region_numbers: vec![0],
-                    primary_key_indices: vec![],
-                    create_if_not_exists: false,
-                    table_options: Default::default(),
-                    engine: MITO_ENGINE.to_string(),
-                },
-            )
-            .await
-            .unwrap();
-        let reg_req = RegisterTableRequest {
-            catalog: catalog_name,
-            schema: schema_name,
-            table_name,
-            table_id: 1,
-            table,
-        };
-        let res = components.catalog_manager.register_table(reg_req).await;
-
-        // because nonexistent_catalog does not exist yet.
-        assert_matches!(
-            res.err().unwrap(),
-            catalog::error::Error::CatalogNotFound { .. }
-        );
-    }
-
-    #[tokio::test]
-    async fn test_register_table() {
-        let node_id = 42;
-        let components = prepare_components(node_id).await;
-        assert_eq!(
-            vec![DEFAULT_SCHEMA_NAME.to_string()],
-            components
-                .catalog_manager
-                .schema_names(DEFAULT_CATALOG_NAME)
-                .await
-                .unwrap()
-        );
-
-        // register a new table with an nonexistent catalog
-        let catalog_name = DEFAULT_CATALOG_NAME.to_string();
-        let schema_name = DEFAULT_SCHEMA_NAME.to_string();
-        let table_name = "test_table".to_string();
-        let table_id = 1;
-        // this schema has no effect
-        let table_schema = RawSchema::new(vec![]);
-        let table = components
-            .table_engine()
-            .create_table(
-                &EngineContext {},
-                CreateTableRequest {
-                    id: table_id,
-                    catalog_name: catalog_name.clone(),
-                    schema_name: schema_name.clone(),
-                    table_name: table_name.clone(),
-                    desc: None,
-                    schema: table_schema,
-                    region_numbers: vec![0],
-                    primary_key_indices: vec![],
-                    create_if_not_exists: false,
-                    table_options: Default::default(),
-                    engine: MITO_ENGINE.to_string(),
-                },
-            )
-            .await
-            .unwrap();
-        let reg_req = RegisterTableRequest {
-            catalog: catalog_name,
-            schema: schema_name,
-            table_name: table_name.clone(),
-            table_id,
-            table,
-        };
-        assert!(components
-            .catalog_manager
-            .register_table(reg_req)
-            .await
-            .unwrap());
-        assert_eq!(
-            vec![table_name],
-            components
-                .catalog_manager
-                .table_names(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
-                .await
-                .unwrap()
-        );
-    }
-
-    #[tokio::test]
-    async fn test_register_catalog_schema_table() {
-        let node_id = 42;
-        let components = prepare_components(node_id).await;
-
-        let catalog_name = "test_catalog".to_string();
-        let schema_name = "nonexistent_schema".to_string();
-
-        // register catalog to catalog manager
-        assert!(components
-            .catalog_manager
-            .register_catalog(catalog_name.clone())
-            .await
-            .is_ok());
-        assert_eq!(
-            HashSet::<String>::from_iter(
-                vec![DEFAULT_CATALOG_NAME.to_string(), catalog_name.clone()].into_iter()
-            ),
-            HashSet::from_iter(
-                components
-                    .catalog_manager
-                    .catalog_names()
-                    .await
-                    .unwrap()
-                    .into_iter()
-            )
-        );
-
-        let table_to_register = components
-            .table_engine()
-            .create_table(
-                &EngineContext {},
-                CreateTableRequest {
-                    id: 2,
-                    catalog_name: catalog_name.clone(),
-                    schema_name: schema_name.clone(),
-                    table_name: "".to_string(),
-                    desc: None,
-                    schema: RawSchema::new(vec![]),
-                    region_numbers: vec![0],
-                    primary_key_indices: vec![],
-                    create_if_not_exists: false,
-                    table_options: Default::default(),
-                    engine: MITO_ENGINE.to_string(),
-                },
-            )
-            .await
-            .unwrap();
-
-        let reg_req = RegisterTableRequest {
-            catalog: catalog_name.clone(),
-            schema: schema_name.clone(),
-            table_name: " fail_table".to_string(),
-            table_id: 2,
-            table: table_to_register,
-        };
-        // this register will fail since schema does not exist yet
-        assert_matches!(
-            components
-                .catalog_manager
-                .register_table(reg_req.clone())
-                .await
-                .unwrap_err(),
-            catalog::error::Error::SchemaNotFound { .. }
-        );
-
-        let register_schema_request = RegisterSchemaRequest {
-            catalog: catalog_name.to_string(),
-            schema: schema_name.to_string(),
-        };
-        assert!(components
-            .catalog_manager
-            .register_schema(register_schema_request)
-            .await
-            .expect("Register schema should not fail"));
-        assert!(components
-            .catalog_manager
-            .register_table(reg_req)
-            .await
-            .unwrap());
-
-        assert_eq!(
-            HashSet::from([schema_name.clone()]),
-            components
-                .catalog_manager
-                .schema_names(&catalog_name)
-                .await
-                .unwrap()
-                .into_iter()
-                .collect()
-        )
-    }
-
-    #[tokio::test]
-    async fn test_register_table_before_and_after_region_alive_keeper_started() {
-        let components = prepare_components(42).await;
-        let catalog_manager = &components.catalog_manager;
-        let region_alive_keepers = &components.region_alive_keepers;
-
-        let table_before = TableIdent {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table: "table_before".to_string(),
-            table_id: 1,
-            engine: MITO_ENGINE.to_string(),
-        };
-        let request = RegisterTableRequest {
-            catalog: table_before.catalog.clone(),
-            schema: table_before.schema.clone(),
-            table_name: table_before.table.clone(),
-            table_id: table_before.table_id,
-            table: Arc::new(EmptyTable::new(CreateTableRequest {
-                id: table_before.table_id,
-                catalog_name: table_before.catalog.clone(),
-                schema_name: table_before.schema.clone(),
-                table_name: table_before.table.clone(),
-                desc: None,
-                schema: RawSchema::new(vec![]),
-                region_numbers: vec![0],
-                primary_key_indices: vec![],
-                create_if_not_exists: false,
-                table_options: Default::default(),
-                engine: MITO_ENGINE.to_string(),
-            })),
-        };
-        assert!(catalog_manager.register_table(request).await.unwrap());
-
-        let keeper = region_alive_keepers
-            .find_keeper(&table_before)
-            .await
-            .unwrap();
-        let deadline = keeper.deadline(0).await.unwrap();
-        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
-        // assert region alive countdown is not started
-        assert!(deadline > far_future);
-
-        region_alive_keepers.start().await;
-
-        let table_after = TableIdent {
-            catalog: DEFAULT_CATALOG_NAME.to_string(),
-            schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table: "table_after".to_string(),
-            table_id: 2,
-            engine: MITO_ENGINE.to_string(),
-        };
-        let request = RegisterTableRequest {
-            catalog: table_after.catalog.clone(),
-            schema: table_after.schema.clone(),
-            table_name: table_after.table.clone(),
-            table_id: table_after.table_id,
-            table: Arc::new(EmptyTable::new(CreateTableRequest {
-                id: table_after.table_id,
-                catalog_name: table_after.catalog.clone(),
-                schema_name: table_after.schema.clone(),
-                table_name: table_after.table.clone(),
-                desc: None,
-                schema: RawSchema::new(vec![]),
-                region_numbers: vec![0],
-                primary_key_indices: vec![],
-                create_if_not_exists: false,
-                table_options: Default::default(),
-                engine: MITO_ENGINE.to_string(),
-            })),
-        };
-        assert!(catalog_manager.register_table(request).await.unwrap());
-
-        let keeper = region_alive_keepers
-            .find_keeper(&table_after)
-            .await
-            .unwrap();
-        let deadline = keeper.deadline(0).await.unwrap();
-        // assert countdown is started for the table registered after [RegionAliveKeepers] started
-        assert!(deadline <= Instant::now() + Duration::from_secs(20));
-
-        let keeper = region_alive_keepers
-            .find_keeper(&table_before)
-            .await
-            .unwrap();
-        let deadline = keeper.deadline(0).await.unwrap();
-        // assert countdown is started for the table registered before [RegionAliveKeepers] started, too
-        assert!(deadline <= Instant::now() + Duration::from_secs(20));
-    }
-}

src/client/Cargo.toml

@@ -8,39 +8,42 @@ license.workspace = true
 testing = []
 
 [dependencies]
-api = { path = "../api" }
+api = { workspace = true }
 arrow-flight.workspace = true
 async-stream.workspace = true
-common-base = { path = "../common/base" }
-common-catalog = { path = "../common/catalog" }
-common-error = { path = "../common/error" }
-common-grpc = { path = "../common/grpc" }
-common-query = { path = "../common/query" }
-common-recordbatch = { path = "../common/recordbatch" }
-common-time = { path = "../common/time" }
-common-meta = { path = "../common/meta" }
-common-telemetry = { path = "../common/telemetry" }
+async-trait.workspace = true
+common-base = { workspace = true }
+common-catalog = { workspace = true }
+common-error = { workspace = true }
+common-grpc = { workspace = true }
+common-meta = { workspace = true }
+common-query = { workspace = true }
+common-recordbatch = { workspace = true }
+common-telemetry = { workspace = true }
+common-time = { workspace = true }
 datafusion.workspace = true
-datatypes = { path = "../datatypes" }
+datatypes = { workspace = true }
+derive_builder.workspace = true
 enum_dispatch = "0.3"
 futures-util.workspace = true
 moka = { version = "0.9", features = ["future"] }
 parking_lot = "0.12"
 prost.workspace = true
 rand.workspace = true
+session = { workspace = true }
 snafu.workspace = true
 tokio-stream = { version = "0.1", features = ["net"] }
 tokio.workspace = true
 tonic.workspace = true
 
 [dev-dependencies]
-common-grpc-expr = { path = "../common/grpc-expr" }
-datanode = { path = "../datanode" }
+common-grpc-expr = { workspace = true }
+datanode = { workspace = true }
 derive-new = "0.5"
-substrait = { path = "../common/substrait" }
+prost.workspace = true
+substrait = { workspace = true }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-prost.workspace = true
 
 [dev-dependencies.substrait_proto]
 package = "substrait"

src/client/examples/logical.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, TableId};
+use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, SemanticType, TableId};
 use client::{Client, Database};
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
 use prost::Message;
@@ -41,21 +41,27 @@ async fn run() {
         column_defs: vec![
             ColumnDef {
                 name: "timestamp".to_string(),
-                datatype: ColumnDataType::TimestampMillisecond as i32,
+                data_type: ColumnDataType::TimestampMillisecond as i32,
                 is_nullable: false,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Timestamp as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "key".to_string(),
-                datatype: ColumnDataType::Uint64 as i32,
+                data_type: ColumnDataType::Uint64 as i32,
                 is_nullable: false,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Tag as i32,
+                comment: String::new(),
             },
             ColumnDef {
                 name: "value".to_string(),
-                datatype: ColumnDataType::Uint64 as i32,
+                data_type: ColumnDataType::Uint64 as i32,
                 is_nullable: false,
                 default_constraint: vec![],
+                semantic_type: SemanticType::Field as i32,
+                comment: String::new(),
             },
         ],
         time_index: "timestamp".to_string(),
@@ -63,7 +69,6 @@ async fn run() {
         create_if_not_exists: false,
         table_options: Default::default(),
         table_id: Some(TableId { id: 1024 }),
-        region_numbers: vec![0],
         engine: MITO_ENGINE.to_string(),
     };
 
@@ -73,7 +78,7 @@ async fn run() {
 
     let logical = mock_logical_plan();
     event!(Level::INFO, "plan size: {:#?}", logical.len());
-    let result = db.logical_plan(logical, None).await.unwrap();
+    let result = db.logical_plan(logical, 0).await.unwrap();
 
     event!(Level::INFO, "result: {:#?}", result);
 }

src/client/examples/stream_ingest.rs

@@ -131,7 +131,7 @@ fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
         Column {
             column_name: "ts".to_owned(),
             values: Some(column::Values {
-                ts_millisecond_values: timestamp_millis,
+                timestamp_millisecond_values: timestamp_millis,
                 ..Default::default()
             }),
             semantic_type: SemanticType::Timestamp as i32,
@@ -177,6 +177,5 @@ fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
         table_name: "weather_demo".to_owned(),
         columns,
         row_count: rows as u32,
-        ..Default::default()
     }
 }

src/client/src/client.rs

@@ -17,6 +17,7 @@ use std::sync::Arc;
 use api::v1::greptime_database_client::GreptimeDatabaseClient;
 use api::v1::health_check_client::HealthCheckClient;
 use api::v1::prometheus_gateway_client::PrometheusGatewayClient;
+use api::v1::region::region_client::RegionClient as PbRegionClient;
 use api::v1::HealthCheckRequest;
 use arrow_flight::flight_service_client::FlightServiceClient;
 use common_grpc::channel_manager::ChannelManager;
@@ -82,11 +83,6 @@ impl Client {
         Default::default()
     }
 
-    pub fn with_manager(channel_manager: ChannelManager) -> Self {
-        let inner = Arc::new(Inner::with_manager(channel_manager));
-        Self { inner }
-    }
-
     pub fn with_urls<U, A>(urls: A) -> Self
     where
         U: AsRef<str>,
@@ -142,21 +138,40 @@ impl Client {
         Ok((addr, channel))
     }
 
+    fn max_grpc_recv_message_size(&self) -> usize {
+        self.inner.channel_manager.config().max_recv_message_size
+    }
+
+    fn max_grpc_send_message_size(&self) -> usize {
+        self.inner.channel_manager.config().max_send_message_size
+    }
+
     pub(crate) fn make_flight_client(&self) -> Result<FlightClient> {
         let (addr, channel) = self.find_channel()?;
         Ok(FlightClient {
             addr,
-            client: FlightServiceClient::new(channel),
+            client: FlightServiceClient::new(channel)
+                .max_decoding_message_size(self.max_grpc_recv_message_size())
+                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
     pub(crate) fn make_database_client(&self) -> Result<DatabaseClient> {
         let (_, channel) = self.find_channel()?;
         Ok(DatabaseClient {
-            inner: GreptimeDatabaseClient::new(channel),
+            inner: GreptimeDatabaseClient::new(channel)
+                .max_decoding_message_size(self.max_grpc_recv_message_size())
+                .max_encoding_message_size(self.max_grpc_send_message_size()),
         })
     }
 
+    pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
+        let (_, channel) = self.find_channel()?;
+        Ok(PbRegionClient::new(channel)
+            .max_decoding_message_size(self.max_grpc_recv_message_size())
+            .max_encoding_message_size(self.max_grpc_send_message_size()))
+    }
+
     pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {
         let (_, channel) = self.find_channel()?;
         Ok(PrometheusGatewayClient::new(channel))

src/client/src/client_manager.rs

@@ -13,12 +13,15 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
+use std::sync::Arc;
 use std::time::Duration;
 
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
+use common_meta::datanode_manager::{Datanode, DatanodeManager};
 use common_meta::peer::Peer;
 use moka::future::{Cache, CacheBuilder};
 
+use crate::region::RegionRequester;
 use crate::Client;
 
 pub struct DatanodeClients {
@@ -40,6 +43,15 @@ impl Debug for DatanodeClients {
     }
 }
 
+#[async_trait::async_trait]
+impl DatanodeManager for DatanodeClients {
+    async fn datanode(&self, datanode: &Peer) -> Arc<dyn Datanode> {
+        let client = self.get_client(datanode).await;
+
+        Arc::new(RegionRequester::new(client))
+    }
+}
+
 impl DatanodeClients {
     pub fn new(config: ChannelConfig) -> Self {
         Self {

src/client/src/database.rs

@@ -17,20 +17,23 @@ use api::v1::ddl_request::Expr as DdlExpr;
 use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
-    AlterExpr, AuthHeader, CompactTableExpr, CreateTableExpr, DdlRequest, DeleteRequest,
-    DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest,
-    RequestHeader, TruncateTableExpr,
+    AlterExpr, AuthHeader, CreateTableExpr, DdlRequest, DeleteRequests, DropTableExpr,
+    GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest, RequestHeader,
+    RowInsertRequests, TruncateTableExpr,
 };
-use arrow_flight::{FlightData, Ticket};
+use arrow_flight::Ticket;
+use async_stream::stream;
 use common_error::ext::{BoxedError, ErrorExt};
-use common_grpc::flight::{flight_messages_to_recordbatches, FlightDecoder, FlightMessage};
+use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_query::Output;
+use common_recordbatch::error::ExternalSnafu;
+use common_recordbatch::RecordBatchStreamAdaptor;
 use common_telemetry::{logging, timer};
-use futures_util::{TryFutureExt, TryStreamExt};
+use futures_util::StreamExt;
 use prost::Message;
 use snafu::{ensure, ResultExt};
 
-use crate::error::{ConvertFlightDataSnafu, IllegalFlightMessagesSnafu, ServerSnafu};
+use crate::error::{ConvertFlightDataSnafu, Error, IllegalFlightMessagesSnafu, ServerSnafu};
 use crate::{error, from_grpc_response, metrics, Client, Result, StreamInserter};
 
 #[derive(Clone, Debug, Default)]
@@ -112,6 +115,11 @@ impl Database {
         self.handle(Request::Inserts(requests)).await
     }
 
+    pub async fn row_insert(&self, requests: RowInsertRequests) -> Result<u32> {
+        let _timer = timer!(metrics::METRIC_GRPC_INSERT);
+        self.handle(Request::RowInserts(requests)).await
+    }
+
     pub fn streaming_inserter(&self) -> Result<StreamInserter> {
         self.streaming_inserter_with_channel_size(65536)
     }
@@ -132,20 +140,20 @@ impl Database {
         Ok(stream_inserter)
     }
 
-    pub async fn delete(&self, request: DeleteRequest) -> Result<u32> {
+    pub async fn delete(&self, request: DeleteRequests) -> Result<u32> {
         let _timer = timer!(metrics::METRIC_GRPC_DELETE);
-        self.handle(Request::Delete(request)).await
+        self.handle(Request::Deletes(request)).await
     }
 
     async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = self.client.make_database_client()?.inner;
-        let request = self.to_rpc_request(request, None);
+        let request = self.to_rpc_request(request, 0);
         let response = client.handle(request).await?.into_inner();
         from_grpc_response(response)
     }
 
     #[inline]
-    fn to_rpc_request(&self, request: Request, trace_id: Option<u64>) -> GreptimeRequest {
+    fn to_rpc_request(&self, request: Request, trace_id: u64) -> GreptimeRequest {
         GreptimeRequest {
             header: Some(RequestHeader {
                 catalog: self.catalog.clone(),
@@ -153,7 +161,7 @@ impl Database {
                 authorization: self.ctx.auth_header.clone(),
                 dbname: self.dbname.clone(),
                 trace_id,
-                span_id: None,
+                span_id: 0,
             }),
             request: Some(request),
         }
@@ -165,16 +173,12 @@ impl Database {
             Request::Query(QueryRequest {
                 query: Some(Query::Sql(sql.to_string())),
             }),
-            None,
+            0,
         )
         .await
     }
 
-    pub async fn logical_plan(
-        &self,
-        logical_plan: Vec<u8>,
-        trace_id: Option<u64>,
-    ) -> Result<Output> {
+    pub async fn logical_plan(&self, logical_plan: Vec<u8>, trace_id: u64) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_LOGICAL_PLAN);
         self.do_get(
             Request::Query(QueryRequest {
@@ -202,7 +206,7 @@ impl Database {
                     step: step.to_string(),
                 })),
             }),
-            None,
+            0,
         )
         .await
     }
@@ -213,7 +217,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::CreateTable(expr)),
             }),
-            None,
+            0,
         )
         .await
     }
@@ -224,7 +228,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::Alter(expr)),
             }),
-            None,
+            0,
         )
         .await
     }
@@ -235,29 +239,7 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::DropTable(expr)),
             }),
-            None,
-        )
-        .await
-    }
-
-    pub async fn flush_table(&self, expr: FlushTableExpr) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_FLUSH_TABLE);
-        self.do_get(
-            Request::Ddl(DdlRequest {
-                expr: Some(DdlExpr::FlushTable(expr)),
-            }),
-            None,
-        )
-        .await
-    }
-
-    pub async fn compact_table(&self, expr: CompactTableExpr) -> Result<Output> {
-        let _timer = timer!(metrics::METRIC_GRPC_COMPACT_TABLE);
-        self.do_get(
-            Request::Ddl(DdlRequest {
-                expr: Some(DdlExpr::CompactTable(expr)),
-            }),
-            None,
+            0,
         )
         .await
     }
@@ -268,12 +250,12 @@ impl Database {
             Request::Ddl(DdlRequest {
                 expr: Some(DdlExpr::TruncateTable(expr)),
             }),
-            None,
+            0,
         )
         .await
     }
 
-    async fn do_get(&self, request: Request, trace_id: Option<u64>) -> Result<Output> {
+    async fn do_get(&self, request: Request, trace_id: u64) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
         let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
         let request = self.to_rpc_request(request, trace_id);
@@ -283,55 +265,81 @@ impl Database {
 
         let mut client = self.client.make_flight_client()?;
 
-        let flight_data: Vec<FlightData> = client
-            .mut_inner()
-            .do_get(request)
-            .and_then(|response| response.into_inner().try_collect())
-            .await
-            .map_err(|e| {
-                let tonic_code = e.code();
-                let e: error::Error = e.into();
-                let code = e.status_code();
-                let msg = e.to_string();
-                ServerSnafu { code, msg }
-                    .fail::<()>()
-                    .map_err(BoxedError::new)
-                    .context(error::FlightGetSnafu {
-                        tonic_code,
-                        addr: client.addr(),
-                    })
-                    .map_err(|error| {
-                        logging::error!(
-                            "Failed to do Flight get, addr: {}, code: {}, source: {}",
-                            client.addr(),
-                            tonic_code,
-                            error
-                        );
-                        error
-                    })
-                    .unwrap_err()
-            })?;
-
-        let decoder = &mut FlightDecoder::default();
-        let flight_messages = flight_data
-            .into_iter()
-            .map(|x| decoder.try_decode(x).context(ConvertFlightDataSnafu))
-            .collect::<Result<Vec<_>>>()?;
-
-        let output = if let Some(FlightMessage::AffectedRows(rows)) = flight_messages.get(0) {
-            ensure!(
-                flight_messages.len() == 1,
-                IllegalFlightMessagesSnafu {
-                    reason: "Expect 'AffectedRows' Flight messages to be one and only!"
-                }
+        let response = client.mut_inner().do_get(request).await.map_err(|e| {
+            let tonic_code = e.code();
+            let e: error::Error = e.into();
+            let code = e.status_code();
+            let msg = e.to_string();
+            let error = Error::FlightGet {
+                tonic_code,
+                addr: client.addr().to_string(),
+                source: BoxedError::new(ServerSnafu { code, msg }.build()),
+            };
+            logging::error!(
+                "Failed to do Flight get, addr: {}, code: {}, source: {}",
+                client.addr(),
+                tonic_code,
+                error
             );
-            Output::AffectedRows(*rows)
-        } else {
-            let recordbatches = flight_messages_to_recordbatches(flight_messages)
-                .context(ConvertFlightDataSnafu)?;
-            Output::RecordBatches(recordbatches)
+            error
+        })?;
+
+        let flight_data_stream = response.into_inner();
+        let mut decoder = FlightDecoder::default();
+
+        let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
+            flight_data
+                .map_err(Error::from)
+                .and_then(|data| decoder.try_decode(data).context(ConvertFlightDataSnafu))
+        });
+
+        let Some(first_flight_message) = flight_message_stream.next().await else {
+            return IllegalFlightMessagesSnafu {
+                reason: "Expect the response not to be empty",
+            }
+            .fail();
         };
-        Ok(output)
+
+        let first_flight_message = first_flight_message?;
+
+        match first_flight_message {
+            FlightMessage::AffectedRows(rows) => {
+                ensure!(
+                    flight_message_stream.next().await.is_none(),
+                    IllegalFlightMessagesSnafu {
+                        reason: "Expect 'AffectedRows' Flight messages to be the one and the only!"
+                    }
+                );
+                Ok(Output::AffectedRows(rows))
+            }
+            FlightMessage::Recordbatch(_) => IllegalFlightMessagesSnafu {
+                reason: "The first flight message cannot be a RecordBatch message",
+            }
+            .fail(),
+            FlightMessage::Schema(schema) => {
+                let stream = Box::pin(stream!({
+                    while let Some(flight_message) = flight_message_stream.next().await {
+                        let flight_message = flight_message
+                            .map_err(BoxedError::new)
+                            .context(ExternalSnafu)?;
+                        let FlightMessage::Recordbatch(record_batch) = flight_message else {
+                            yield IllegalFlightMessagesSnafu {reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"}
+                                        .fail()
+                                        .map_err(BoxedError::new)
+                                        .context(ExternalSnafu);
+                            break;
+                        };
+                        yield Ok(record_batch);
+                    }
+                }));
+                let record_batch_stream = RecordBatchStreamAdaptor {
+                    schema,
+                    stream,
+                    output_ordering: None,
+                };
+                Ok(Output::Stream(Box::pin(record_batch_stream)))
+            }
+        }
     }
 }
 
@@ -342,106 +350,11 @@ pub struct FlightContext {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
-    use api::helper::ColumnDataTypeWrapper;
     use api::v1::auth_header::AuthScheme;
-    use api::v1::{AuthHeader, Basic, Column};
-    use common_grpc::select::{null_mask, values};
-    use common_grpc_expr::column_to_vector;
-    use datatypes::prelude::{Vector, VectorRef};
-    use datatypes::vectors::{
-        BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
-        Int16Vector, Int32Vector, Int64Vector, Int8Vector, StringVector, UInt16Vector,
-        UInt32Vector, UInt64Vector, UInt8Vector,
-    };
+    use api::v1::{AuthHeader, Basic};
 
     use crate::database::FlightContext;
 
-    #[test]
-    fn test_column_to_vector() {
-        let mut column = create_test_column(Arc::new(BooleanVector::from(vec![true])));
-        column.datatype = -100;
-        let result = column_to_vector(&column, 1);
-        assert!(result.is_err());
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Column datatype error, source: Unknown proto column datatype: -100"
-        );
-
-        macro_rules! test_with_vector {
-            ($vector: expr) => {
-                let vector = Arc::new($vector);
-                let column = create_test_column(vector.clone());
-                let result = column_to_vector(&column, vector.len() as u32).unwrap();
-                assert_eq!(result, vector as VectorRef);
-            };
-        }
-
-        test_with_vector!(BooleanVector::from(vec![Some(true), None, Some(false)]));
-        test_with_vector!(Int8Vector::from(vec![Some(i8::MIN), None, Some(i8::MAX)]));
-        test_with_vector!(Int16Vector::from(vec![
-            Some(i16::MIN),
-            None,
-            Some(i16::MAX)
-        ]));
-        test_with_vector!(Int32Vector::from(vec![
-            Some(i32::MIN),
-            None,
-            Some(i32::MAX)
-        ]));
-        test_with_vector!(Int64Vector::from(vec![
-            Some(i64::MIN),
-            None,
-            Some(i64::MAX)
-        ]));
-        test_with_vector!(UInt8Vector::from(vec![Some(u8::MIN), None, Some(u8::MAX)]));
-        test_with_vector!(UInt16Vector::from(vec![
-            Some(u16::MIN),
-            None,
-            Some(u16::MAX)
-        ]));
-        test_with_vector!(UInt32Vector::from(vec![
-            Some(u32::MIN),
-            None,
-            Some(u32::MAX)
-        ]));
-        test_with_vector!(UInt64Vector::from(vec![
-            Some(u64::MIN),
-            None,
-            Some(u64::MAX)
-        ]));
-        test_with_vector!(Float32Vector::from(vec![
-            Some(f32::MIN),
-            None,
-            Some(f32::MAX)
-        ]));
-        test_with_vector!(Float64Vector::from(vec![
-            Some(f64::MIN),
-            None,
-            Some(f64::MAX)
-        ]));
-        test_with_vector!(BinaryVector::from(vec![
-            Some(b"".to_vec()),
-            None,
-            Some(b"hello".to_vec())
-        ]));
-        test_with_vector!(StringVector::from(vec![Some(""), None, Some("foo"),]));
-        test_with_vector!(DateVector::from(vec![Some(1), None, Some(3)]));
-        test_with_vector!(DateTimeVector::from(vec![Some(4), None, Some(6)]));
-    }
-
-    fn create_test_column(vector: VectorRef) -> Column {
-        let wrapper: ColumnDataTypeWrapper = vector.data_type().try_into().unwrap();
-        Column {
-            column_name: "test".to_string(),
-            semantic_type: 1,
-            values: Some(values(&[vector.clone()]).unwrap()),
-            null_mask: null_mask(&[vector.clone()], vector.len()),
-            datatype: wrapper.datatype() as i32,
-        }
-    }
-
     #[test]
     fn test_flight_ctx() {
         let mut ctx = FlightContext::default();

src/client/src/error.rs

@@ -13,11 +13,10 @@
 // limitations under the License.
 
 use std::any::Any;
-use std::str::FromStr;
 
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use common_error::{INNER_ERROR_CODE, INNER_ERROR_MSG};
+use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
 use snafu::{Location, Snafu};
 use tonic::{Code, Status};
 
@@ -27,20 +26,26 @@ pub enum Error {
     #[snafu(display("Illegal Flight messages, reason: {}", reason))]
     IllegalFlightMessages { reason: String, location: Location },
 
-    #[snafu(display("Failed to do Flight get, code: {}, source: {}", tonic_code, source))]
+    #[snafu(display("Failed to do Flight get, code: {}", tonic_code))]
     FlightGet {
         addr: String,
         tonic_code: Code,
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to convert FlightData, source: {}", source))]
+    #[snafu(display("Failure occurs during handling request"))]
+    HandleRequest {
+        location: Location,
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to convert FlightData"))]
     ConvertFlightData {
         location: Location,
         source: common_grpc::Error,
     },
 
-    #[snafu(display("Column datatype error, source: {}", source))]
+    #[snafu(display("Column datatype error"))]
     ColumnDataType {
         location: Location,
         source: api::error::Error,
@@ -52,17 +57,16 @@ pub enum Error {
     #[snafu(display("Missing required field in protobuf, field: {}", field))]
     MissingField { field: String, location: Location },
 
-    #[snafu(display(
-        "Failed to create gRPC channel, peer address: {}, source: {}",
-        addr,
-        source
-    ))]
+    #[snafu(display("Failed to create gRPC channel, peer address: {}", addr))]
     CreateChannel {
         addr: String,
         location: Location,
         source: common_grpc::error::Error,
     },
 
+    #[snafu(display("Failed to request RegionServer, code: {}", code))]
+    RegionServer { code: Code, source: BoxedError },
+
     // Server error carried in Tonic Status's metadata.
     #[snafu(display("{}", msg))]
     Server { code: StatusCode, msg: String },
@@ -86,7 +90,9 @@ impl ErrorExt for Error {
             | Error::ClientStreaming { .. } => StatusCode::Internal,
 
             Error::Server { code, .. } => *code,
-            Error::FlightGet { source, .. } => source.status_code(),
+            Error::FlightGet { source, .. }
+            | Error::HandleRequest { source, .. }
+            | Error::RegionServer { source, .. } => source.status_code(),
             Error::CreateChannel { source, .. } | Error::ConvertFlightData { source, .. } => {
                 source.status_code()
             }
@@ -107,11 +113,18 @@ impl From<Status> for Error {
                 .and_then(|v| String::from_utf8(v.as_bytes().to_vec()).ok())
         }
 
-        let code = get_metadata_value(&e, INNER_ERROR_CODE)
-            .and_then(|s| StatusCode::from_str(&s).ok())
+        let code = get_metadata_value(&e, GREPTIME_ERROR_CODE)
+            .and_then(|s| {
+                if let Ok(code) = s.parse::<u32>() {
+                    StatusCode::from_u32(code)
+                } else {
+                    None
+                }
+            })
             .unwrap_or(StatusCode::Unknown);
 
-        let msg = get_metadata_value(&e, INNER_ERROR_MSG).unwrap_or(e.to_string());
+        let msg =
+            get_metadata_value(&e, GREPTIME_ERROR_MSG).unwrap_or_else(|| e.message().to_string());
 
         Self::Server { code, msg }
     }

src/client/src/lib.rs

@@ -18,6 +18,7 @@ mod database;
 pub mod error;
 pub mod load_balance;
 mod metrics;
+pub mod region;
 mod stream_insert;
 
 pub use api;

src/client/src/metrics.rs

@@ -21,7 +21,6 @@ pub const METRIC_GRPC_SQL: &str = "grpc.sql";
 pub const METRIC_GRPC_LOGICAL_PLAN: &str = "grpc.logical_plan";
 pub const METRIC_GRPC_ALTER: &str = "grpc.alter";
 pub const METRIC_GRPC_DROP_TABLE: &str = "grpc.drop_table";
-pub const METRIC_GRPC_FLUSH_TABLE: &str = "grpc.flush_table";
-pub const METRIC_GRPC_COMPACT_TABLE: &str = "grpc.compact_table";
 pub const METRIC_GRPC_TRUNCATE_TABLE: &str = "grpc.truncate_table";
 pub const METRIC_GRPC_DO_GET: &str = "grpc.do_get";
+pub(crate) const METRIC_REGION_REQUEST_GRPC: &str = "grpc.region_request";

src/client/src/region.rs

@@ -0,0 +1,264 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::region::{QueryRequest, RegionRequest, RegionResponse};
+use api::v1::ResponseHeader;
+use arrow_flight::Ticket;
+use async_stream::stream;
+use async_trait::async_trait;
+use common_error::ext::{BoxedError, ErrorExt};
+use common_error::status_code::StatusCode;
+use common_grpc::flight::{FlightDecoder, FlightMessage};
+use common_meta::datanode_manager::{AffectedRows, Datanode};
+use common_meta::error::{self as meta_error, Result as MetaResult};
+use common_recordbatch::error::ExternalSnafu;
+use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
+use common_telemetry::{error, timer};
+use prost::Message;
+use snafu::{location, Location, OptionExt, ResultExt};
+use tokio_stream::StreamExt;
+
+use crate::error::Error::RegionServer;
+use crate::error::{
+    self, ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
+    MissingFieldSnafu, Result, ServerSnafu,
+};
+use crate::{metrics, Client, Error};
+
+#[derive(Debug)]
+pub struct RegionRequester {
+    client: Client,
+}
+
+#[async_trait]
+impl Datanode for RegionRequester {
+    async fn handle(&self, request: RegionRequest) -> MetaResult<AffectedRows> {
+        self.handle_inner(request).await.map_err(|err| {
+            if matches!(err, RegionServer { .. }) {
+                meta_error::Error::RetryLater {
+                    source: BoxedError::new(err),
+                }
+            } else {
+                meta_error::Error::External {
+                    source: BoxedError::new(err),
+                    location: location!(),
+                }
+            }
+        })
+    }
+
+    async fn handle_query(&self, request: QueryRequest) -> MetaResult<SendableRecordBatchStream> {
+        let ticket = Ticket {
+            ticket: request.encode_to_vec().into(),
+        };
+        self.do_get_inner(ticket)
+            .await
+            .map_err(BoxedError::new)
+            .context(meta_error::ExternalSnafu)
+    }
+}
+
+impl RegionRequester {
+    pub fn new(client: Client) -> Self {
+        Self { client }
+    }
+
+    pub async fn do_get_inner(&self, ticket: Ticket) -> Result<SendableRecordBatchStream> {
+        let mut flight_client = self.client.make_flight_client()?;
+        let response = flight_client
+            .mut_inner()
+            .do_get(ticket)
+            .await
+            .map_err(|e| {
+                let tonic_code = e.code();
+                let e: error::Error = e.into();
+                let code = e.status_code();
+                let msg = e.to_string();
+                let error = Error::FlightGet {
+                    tonic_code,
+                    addr: flight_client.addr().to_string(),
+                    source: BoxedError::new(ServerSnafu { code, msg }.build()),
+                };
+                error!(
+                    e; "Failed to do Flight get, addr: {}, code: {}",
+                    flight_client.addr(),
+                    tonic_code
+                );
+                error
+            })?;
+
+        let flight_data_stream = response.into_inner();
+        let mut decoder = FlightDecoder::default();
+
+        let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
+            flight_data
+                .map_err(Error::from)
+                .and_then(|data| decoder.try_decode(data).context(ConvertFlightDataSnafu))
+        });
+
+        let Some(first_flight_message) = flight_message_stream.next().await else {
+            return IllegalFlightMessagesSnafu {
+                reason: "Expect the response not to be empty",
+            }
+            .fail();
+        };
+        let FlightMessage::Schema(schema) = first_flight_message? else {
+            return IllegalFlightMessagesSnafu {
+                reason: "Expect schema to be the first flight message",
+            }
+            .fail();
+        };
+
+        let stream = Box::pin(stream!({
+            while let Some(flight_message) = flight_message_stream.next().await {
+                let flight_message = flight_message
+                    .map_err(BoxedError::new)
+                    .context(ExternalSnafu)?;
+                let FlightMessage::Recordbatch(record_batch) = flight_message else {
+                    yield IllegalFlightMessagesSnafu {
+                            reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"
+                        }
+                        .fail()
+                        .map_err(BoxedError::new)
+                        .context(ExternalSnafu);
+                    break;
+                };
+                yield Ok(record_batch);
+            }
+        }));
+        let record_batch_stream = RecordBatchStreamAdaptor {
+            schema,
+            stream,
+            output_ordering: None,
+        };
+        Ok(Box::pin(record_batch_stream))
+    }
+
+    async fn handle_inner(&self, request: RegionRequest) -> Result<AffectedRows> {
+        let request_type = request
+            .body
+            .as_ref()
+            .with_context(|| MissingFieldSnafu { field: "body" })?
+            .as_ref()
+            .to_string();
+
+        let _timer = timer!(
+            metrics::METRIC_REGION_REQUEST_GRPC,
+            &[("request_type", request_type)]
+        );
+
+        let mut client = self.client.raw_region_client()?;
+
+        let RegionResponse {
+            header,
+            affected_rows,
+        } = client
+            .handle(request)
+            .await
+            .map_err(|e| {
+                let code = e.code();
+                let err: error::Error = e.into();
+                // Uses `Error::RegionServer` instead of `Error::Server`
+                error::Error::RegionServer {
+                    code,
+                    source: BoxedError::new(err),
+                }
+            })?
+            .into_inner();
+
+        check_response_header(header)?;
+
+        Ok(affected_rows)
+    }
+
+    pub async fn handle(&self, request: RegionRequest) -> Result<AffectedRows> {
+        self.handle_inner(request).await
+    }
+}
+
+pub fn check_response_header(header: Option<ResponseHeader>) -> Result<()> {
+    let status = header
+        .and_then(|header| header.status)
+        .context(IllegalDatabaseResponseSnafu {
+            err_msg: "either response header or status is missing",
+        })?;
+
+    if StatusCode::is_success(status.status_code) {
+        Ok(())
+    } else {
+        let code =
+            StatusCode::from_u32(status.status_code).context(IllegalDatabaseResponseSnafu {
+                err_msg: format!("unknown server status: {:?}", status),
+            })?;
+        ServerSnafu {
+            code,
+            msg: status.err_msg,
+        }
+        .fail()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use api::v1::Status as PbStatus;
+
+    use super::*;
+    use crate::Error::{IllegalDatabaseResponse, Server};
+
+    #[test]
+    fn test_check_response_header() {
+        let result = check_response_header(None);
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader { status: None }));
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: StatusCode::Success as u32,
+                err_msg: "".to_string(),
+            }),
+        }));
+        assert!(result.is_ok());
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: u32::MAX,
+                err_msg: "".to_string(),
+            }),
+        }));
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: StatusCode::Internal as u32,
+                err_msg: "blabla".to_string(),
+            }),
+        }));
+        let Server { code, msg } = result.unwrap_err() else {
+            unreachable!()
+        };
+        assert_eq!(code, StatusCode::Internal);
+        assert_eq!(msg, "blabla");
+    }
+}

src/client/src/stream_insert.rs

@@ -16,6 +16,7 @@ use api::v1::greptime_database_client::GreptimeDatabaseClient;
 use api::v1::greptime_request::Request;
 use api::v1::{
     AuthHeader, GreptimeRequest, GreptimeResponse, InsertRequest, InsertRequests, RequestHeader,
+    RowInsertRequest, RowInsertRequests,
 };
 use tokio::sync::mpsc;
 use tokio::task::JoinHandle;
@@ -84,6 +85,18 @@ impl StreamInserter {
         })
     }
 
+    pub async fn row_insert(&self, requests: Vec<RowInsertRequest>) -> Result<()> {
+        let inserts = RowInsertRequests { inserts: requests };
+        let request = self.to_rpc_request(Request::RowInserts(inserts));
+
+        self.sender.send(request).await.map_err(|e| {
+            error::ClientStreamingSnafu {
+                err_msg: e.to_string(),
+            }
+            .build()
+        })
+    }
+
     pub async fn finish(self) -> Result<u32> {
         drop(self.sender);
 

src/cmd/Cargo.toml

@@ -13,56 +13,66 @@ path = "src/bin/greptime.rs"
 default = ["metrics-process"]
 tokio-console = ["common-telemetry/tokio-console"]
 metrics-process = ["servers/metrics-process"]
-greptimedb-telemetry = [
-    "datanode/greptimedb-telemetry",
-    "meta-srv/greptimedb-telemetry",
-]
 
 [dependencies]
 anymap = "1.0.0-beta.2"
 async-trait.workspace = true
-catalog = { path = "../catalog" }
+auth.workspace = true
+catalog = { workspace = true }
 chrono.workspace = true
 clap = { version = "3.1", features = ["derive"] }
-client = { path = "../client" }
-common-base = { path = "../common/base" }
-common-error = { path = "../common/error" }
-common-query = { path = "../common/query" }
-common-meta = { path = "../common/meta" }
-common-recordbatch = { path = "../common/recordbatch" }
-common-telemetry = { path = "../common/telemetry", features = [
+client = { workspace = true }
+common-base = { workspace = true }
+common-catalog = { workspace = true }
+common-config = { workspace = true }
+common-error = { workspace = true }
+common-meta = { workspace = true }
+common-procedure = { workspace = true }
+common-query = { workspace = true }
+common-recordbatch = { workspace = true }
+common-telemetry = { workspace = true, features = [
     "deadlock_detection",
 ] }
 config = "0.13"
-datanode = { path = "../datanode" }
-datatypes = { path = "../datatypes" }
+datanode = { workspace = true }
+datatypes = { workspace = true }
 either = "1.8"
 etcd-client.workspace = true
-frontend = { path = "../frontend" }
+file-engine = { workspace = true }
+frontend = { workspace = true }
 futures.workspace = true
-meta-client = { path = "../meta-client" }
-meta-srv = { path = "../meta-srv" }
+lazy_static.workspace = true
+meta-client = { workspace = true }
+meta-srv = { workspace = true }
 metrics.workspace = true
+mito2 = { workspace = true }
 nu-ansi-term = "0.46"
-partition = { path = "../partition" }
-query = { path = "../query" }
+partition = { workspace = true }
+prost.workspace = true
+query = { workspace = true }
 rand.workspace = true
+regex.workspace = true
 rustyline = "10.1"
 serde.workspace = true
-servers = { path = "../servers" }
-session = { path = "../session" }
+serde_json.workspace = true
+servers = { workspace = true }
+session = { workspace = true }
 snafu.workspace = true
-substrait = { path = "../common/substrait" }
-table = { path = "../table" }
-tikv-jemallocator = "0.5"
+substrait = { workspace = true }
+table = { workspace = true }
 tokio.workspace = true
 
+[target.'cfg(not(windows))'.dependencies]
+tikv-jemallocator = "0.5"
+
 [dev-dependencies]
-common-test-util = { path = "../common/test-util" }
-rexpect = "0.5"
-temp-env = "0.3"
+common-test-util = { workspace = true }
 serde.workspace = true
+temp-env = "0.3"
 toml.workspace = true
 
+[target.'cfg(not(windows))'.dev-dependencies]
+rexpect = "0.5"
+
 [build-dependencies]
-common-version = { path = "../common/version" }
+common-version = { workspace = true }

src/cmd/src/bin/greptime.rs

@@ -116,7 +116,7 @@ impl SubCommand {
                 Ok(Application::Metasrv(app))
             }
             (SubCommand::Standalone(cmd), Options::Standalone(opts)) => {
-                let app = cmd.build(opts.fe_opts, opts.dn_opts).await?;
+                let app = cmd.build(*opts).await?;
                 Ok(Application::Standalone(app))
             }
             (SubCommand::Cli(cmd), Options::Cli(_)) => {
@@ -187,6 +187,7 @@ fn log_env_flags() {
     }
 }
 
+#[cfg(not(windows))]
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 

src/cmd/src/cli.rs

@@ -16,6 +16,8 @@ mod bench;
 mod cmd;
 mod helper;
 mod repl;
+// TODO(weny): Removes it
+#[allow(deprecated)]
 mod upgrade;
 
 use async_trait::async_trait;

src/cmd/src/cli/bench.rs

@@ -12,53 +12,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod datanode_table;
-mod table_info;
-mod table_name;
-mod table_region;
-
+use std::collections::BTreeMap;
 use std::future::Future;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Duration;
 
 use async_trait::async_trait;
 use clap::Parser;
-use common_meta::key::table_region::RegionDistribution;
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
+use common_meta::peer::Peer;
+use common_meta::rpc::router::{Region, RegionRoute};
 use common_meta::table_name::TableName;
 use common_telemetry::info;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use meta_srv::service::store::etcd::EtcdStore;
 use meta_srv::service::store::kv::KvBackendAdapter;
-use rand::prelude::SliceRandom;
+use rand::Rng;
 use table::metadata::{RawTableInfo, RawTableMeta, TableId, TableIdent, TableType};
 
-use crate::cli::bench::datanode_table::DatanodeTableBencher;
-use crate::cli::bench::table_info::TableInfoBencher;
-use crate::cli::bench::table_name::TableNameBencher;
-use crate::cli::bench::table_region::TableRegionBencher;
+use self::metadata::TableMetadataBencher;
 use crate::cli::{Instance, Tool};
 use crate::error::Result;
 
-async fn bench<F, Fut>(desc: &str, f: F, count: u32)
-where
-    F: Fn(u32) -> Fut,
-    Fut: Future<Output = ()>,
-{
-    let mut total = Duration::default();
-
-    for i in 1..=count {
-        let start = Instant::now();
-
-        f(i).await;
-
-        total += start.elapsed();
-    }
-
-    let cost = total.as_millis() as f64 / count as f64;
-    info!("{desc}, average operation cost: {cost:.2} ms");
-}
+mod metadata;
 
 async fn bench_self_recorded<F, Fut>(desc: &str, f: F, count: u32)
 where
@@ -107,31 +84,11 @@ struct BenchTableMetadata {
 #[async_trait]
 impl Tool for BenchTableMetadata {
     async fn do_work(&self) -> Result<()> {
-        info!("Start benching table name manager ...");
-        TableNameBencher::new(self.table_metadata_manager.table_name_manager(), self.count)
-            .start()
-            .await;
-
-        info!("Start benching table info manager ...");
-        TableInfoBencher::new(self.table_metadata_manager.table_info_manager(), self.count)
-            .start()
-            .await;
-
-        info!("Start benching table region manager ...");
-        TableRegionBencher::new(
-            self.table_metadata_manager.table_region_manager(),
-            self.count,
-        )
-        .start()
-        .await;
-
-        info!("Start benching datanode table manager ...");
-        DatanodeTableBencher::new(
-            self.table_metadata_manager.datanode_table_manager(),
-            self.count,
-        )
-        .start()
-        .await;
+        let bencher = TableMetadataBencher::new(self.table_metadata_manager.clone(), self.count);
+        bencher.bench_create().await;
+        bencher.bench_get().await;
+        bencher.bench_rename().await;
+        bencher.bench_delete().await;
         Ok(())
     }
 }
@@ -163,7 +120,6 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
         created_on: chrono::DateTime::default(),
         primary_key_indices: vec![],
         next_column_id: columns as u32 + 1,
-        engine_options: Default::default(),
         value_indices: vec![],
         options: Default::default(),
         region_numbers: (1..=100).collect(),
@@ -184,16 +140,25 @@ fn create_table_info(table_id: TableId, table_name: TableName) -> RawTableInfo {
     }
 }
 
-fn create_region_distribution() -> RegionDistribution {
-    let mut regions = (1..=100).collect::<Vec<u32>>();
-    regions.shuffle(&mut rand::thread_rng());
+fn create_region_routes() -> Vec<RegionRoute> {
+    let mut regions = Vec::with_capacity(100);
+    let mut rng = rand::thread_rng();
 
-    let mut region_distribution = RegionDistribution::new();
-    for datanode_id in 0..10 {
-        region_distribution.insert(
-            datanode_id as u64,
-            regions[datanode_id * 10..(datanode_id + 1) * 10].to_vec(),
-        );
+    for region_id in 0..64u64 {
+        regions.push(RegionRoute {
+            region: Region {
+                id: region_id.into(),
+                name: String::new(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer {
+                id: rng.gen_range(0..10),
+                addr: String::new(),
+            }),
+            follower_peers: vec![],
+        });
     }
-    region_distribution
+
+    regions
 }

src/cmd/src/cli/bench/datanode_table.rs

@@ -1,131 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableManager};
-
-use super::bench;
-
-pub struct DatanodeTableBencher<'a> {
-    datanode_table_manager: &'a DatanodeTableManager,
-    count: u32,
-}
-
-impl<'a> DatanodeTableBencher<'a> {
-    pub fn new(datanode_table_manager: &'a DatanodeTableManager, count: u32) -> Self {
-        Self {
-            datanode_table_manager,
-            count,
-        }
-    }
-
-    pub async fn start(&self) {
-        self.bench_create().await;
-        self.bench_get().await;
-        self.bench_move_region().await;
-        self.bench_tables().await;
-        self.bench_remove().await;
-    }
-
-    async fn bench_create(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: create {} datanode table keys",
-            self.count
-        );
-        bench(
-            &desc,
-            |i| async move {
-                self.datanode_table_manager
-                    .create(1, i, vec![1, 2, 3, 4])
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_get(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: get {} datanode table keys",
-            self.count
-        );
-        bench(
-            &desc,
-            |i| async move {
-                let key = DatanodeTableKey::new(1, i);
-                assert!(self
-                    .datanode_table_manager
-                    .get(&key)
-                    .await
-                    .unwrap()
-                    .is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_move_region(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: move {} datanode table regions",
-            self.count
-        );
-        bench(
-            &desc,
-            |i| async move {
-                self.datanode_table_manager
-                    .move_region(1, 2, i, 1)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_tables(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: list {} datanode table keys",
-            self.count
-        );
-        bench(
-            &desc,
-            |_| async move {
-                assert!(!self
-                    .datanode_table_manager
-                    .tables(1)
-                    .await
-                    .unwrap()
-                    .is_empty());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_remove(&self) {
-        let desc = format!(
-            "DatanodeTableBencher: remove {} datanode table keys",
-            self.count
-        );
-        bench(
-            &desc,
-            |i| async move {
-                self.datanode_table_manager.remove(1, i).await.unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-}

src/cmd/src/cli/bench/metadata.rs

@@ -0,0 +1,136 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Instant;
+
+use common_meta::key::TableMetadataManagerRef;
+use common_meta::table_name::TableName;
+
+use super::{bench_self_recorded, create_region_routes, create_table_info};
+
+pub struct TableMetadataBencher {
+    table_metadata_manager: TableMetadataManagerRef,
+    count: u32,
+}
+
+impl TableMetadataBencher {
+    pub fn new(table_metadata_manager: TableMetadataManagerRef, count: u32) -> Self {
+        Self {
+            table_metadata_manager,
+            count,
+        }
+    }
+
+    pub async fn bench_create(&self) {
+        let desc = format!(
+            "TableMetadataBencher: creating {} table metadata",
+            self.count
+        );
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let table_name = format!("bench_table_name_{}", i);
+                let table_name = TableName::new("bench_catalog", "bench_schema", table_name);
+                let table_info = create_table_info(i, table_name);
+                let region_routes = create_region_routes();
+
+                let start = Instant::now();
+
+                self.table_metadata_manager
+                    .create_table_metadata(table_info, region_routes)
+                    .await
+                    .unwrap();
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    pub async fn bench_get(&self) {
+        let desc = format!(
+            "TableMetadataBencher: getting {} table info and region routes",
+            self.count
+        );
+
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let start = Instant::now();
+                self.table_metadata_manager
+                    .get_full_table_info(i)
+                    .await
+                    .unwrap();
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    pub async fn bench_delete(&self) {
+        let desc = format!(
+            "TableMetadataBencher: deleting {} table metadata",
+            self.count
+        );
+
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let (table_info, table_route) = self
+                    .table_metadata_manager
+                    .get_full_table_info(i)
+                    .await
+                    .unwrap();
+                let start = Instant::now();
+                let _ = self
+                    .table_metadata_manager
+                    .delete_table_metadata(&table_info.unwrap(), &table_route.unwrap())
+                    .await;
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+
+    pub async fn bench_rename(&self) {
+        let desc = format!("TableMetadataBencher: renaming {} table", self.count);
+
+        bench_self_recorded(
+            &desc,
+            |i| async move {
+                let (table_info, _) = self
+                    .table_metadata_manager
+                    .get_full_table_info(i)
+                    .await
+                    .unwrap();
+
+                let new_table_name = format!("renamed_{}", i);
+
+                let start = Instant::now();
+                let _ = self
+                    .table_metadata_manager
+                    .rename_table(table_info.unwrap(), new_table_name)
+                    .await;
+
+                start.elapsed()
+            },
+            self.count,
+        )
+        .await;
+    }
+}

src/cmd/src/cli/bench/table_info.rs

@@ -1,111 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::Instant;
-
-use common_meta::key::table_info::TableInfoManager;
-use common_meta::table_name::TableName;
-
-use super::{bench, bench_self_recorded, create_table_info};
-
-pub struct TableInfoBencher<'a> {
-    table_info_manager: &'a TableInfoManager,
-    count: u32,
-}
-
-impl<'a> TableInfoBencher<'a> {
-    pub fn new(table_info_manager: &'a TableInfoManager, count: u32) -> Self {
-        Self {
-            table_info_manager,
-            count,
-        }
-    }
-
-    pub async fn start(&self) {
-        self.bench_create().await;
-        self.bench_get().await;
-        self.bench_compare_and_put().await;
-        self.bench_remove().await;
-    }
-
-    async fn bench_create(&self) {
-        let desc = format!("TableInfoBencher: create {} table infos", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_{}", i);
-                let table_name = TableName::new("bench_catalog", "bench_schema", table_name);
-                let table_info = create_table_info(i, table_name);
-                self.table_info_manager
-                    .create(i, &table_info)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_get(&self) {
-        let desc = format!("TableInfoBencher: get {} table infos", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                assert!(self.table_info_manager.get(i).await.unwrap().is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_compare_and_put(&self) {
-        let desc = format!(
-            "TableInfoBencher: compare_and_put {} table infos",
-            self.count
-        );
-        bench_self_recorded(
-            &desc,
-            |i| async move {
-                let table_info_value = self.table_info_manager.get(i).await.unwrap().unwrap();
-
-                let mut new_table_info = table_info_value.table_info.clone();
-                new_table_info.ident.version += 1;
-
-                let start = Instant::now();
-
-                self.table_info_manager
-                    .compare_and_put(i, Some(table_info_value), new_table_info)
-                    .await
-                    .unwrap()
-                    .unwrap();
-
-                start.elapsed()
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_remove(&self) {
-        let desc = format!("TableInfoBencher: remove {} table infos", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                self.table_info_manager.remove(i).await.unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-}

src/cmd/src/cli/bench/table_name.rs

@@ -1,131 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_meta::key::table_name::{TableNameKey, TableNameManager};
-
-use super::bench;
-
-pub struct TableNameBencher<'a> {
-    table_name_manager: &'a TableNameManager,
-    count: u32,
-}
-
-impl<'a> TableNameBencher<'a> {
-    pub fn new(table_name_manager: &'a TableNameManager, count: u32) -> Self {
-        Self {
-            table_name_manager,
-            count,
-        }
-    }
-
-    pub async fn start(&self) {
-        self.bench_create().await;
-        self.bench_rename().await;
-        self.bench_get().await;
-        self.bench_tables().await;
-        self.bench_remove().await;
-    }
-
-    async fn bench_create(&self) {
-        let desc = format!("TableNameBencher: create {} table names", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_{}", i);
-                let table_name_key = create_table_name_key(&table_name);
-                self.table_name_manager
-                    .create(&table_name_key, i)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_rename(&self) {
-        let desc = format!("TableNameBencher: rename {} table names", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_{}", i);
-                let new_table_name = format!("bench_table_name_new_{}", i);
-                let table_name_key = create_table_name_key(&table_name);
-                self.table_name_manager
-                    .rename(table_name_key, i, &new_table_name)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_get(&self) {
-        let desc = format!("TableNameBencher: get {} table names", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_new_{}", i);
-                let table_name_key = create_table_name_key(&table_name);
-                assert!(self
-                    .table_name_manager
-                    .get(table_name_key)
-                    .await
-                    .unwrap()
-                    .is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_tables(&self) {
-        let desc = format!("TableNameBencher: list all {} table names", self.count);
-        bench(
-            &desc,
-            |_| async move {
-                assert!(!self
-                    .table_name_manager
-                    .tables("bench_catalog", "bench_schema")
-                    .await
-                    .unwrap()
-                    .is_empty());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_remove(&self) {
-        let desc = format!("TableNameBencher: remove {} table names", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                let table_name = format!("bench_table_name_new_{}", i);
-                let table_name_key = create_table_name_key(&table_name);
-                self.table_name_manager
-                    .remove(table_name_key)
-                    .await
-                    .unwrap();
-            },
-            self.count,
-        )
-        .await;
-    }
-}
-
-fn create_table_name_key(table_name: &str) -> TableNameKey {
-    TableNameKey::new("bench_catalog", "bench_schema", table_name)
-}

src/cmd/src/cli/bench/table_region.rs

@@ -1,112 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::Instant;
-
-use common_meta::key::table_region::TableRegionManager;
-
-use super::{bench, bench_self_recorded, create_region_distribution};
-
-pub struct TableRegionBencher<'a> {
-    table_region_manager: &'a TableRegionManager,
-    count: u32,
-}
-
-impl<'a> TableRegionBencher<'a> {
-    pub fn new(table_region_manager: &'a TableRegionManager, count: u32) -> Self {
-        Self {
-            table_region_manager,
-            count,
-        }
-    }
-
-    pub async fn start(&self) {
-        self.bench_create().await;
-        self.bench_get().await;
-        self.bench_compare_and_put().await;
-        self.bench_remove().await;
-    }
-
-    async fn bench_create(&self) {
-        let desc = format!("TableRegionBencher: create {} table regions", self.count);
-        bench_self_recorded(
-            &desc,
-            |i| async move {
-                let region_distribution = create_region_distribution();
-
-                let start = Instant::now();
-
-                self.table_region_manager
-                    .create(i, &region_distribution)
-                    .await
-                    .unwrap();
-
-                start.elapsed()
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_get(&self) {
-        let desc = format!("TableRegionBencher: get {} table regions", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                assert!(self.table_region_manager.get(i).await.unwrap().is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_compare_and_put(&self) {
-        let desc = format!(
-            "TableRegionBencher: compare_and_put {} table regions",
-            self.count
-        );
-        bench_self_recorded(
-            &desc,
-            |i| async move {
-                let table_region_value = self.table_region_manager.get(i).await.unwrap().unwrap();
-
-                let new_region_distribution = create_region_distribution();
-
-                let start = Instant::now();
-
-                self.table_region_manager
-                    .compare_and_put(i, Some(table_region_value), new_region_distribution)
-                    .await
-                    .unwrap()
-                    .unwrap();
-
-                start.elapsed()
-            },
-            self.count,
-        )
-        .await;
-    }
-
-    async fn bench_remove(&self) {
-        let desc = format!("TableRegionBencher: remove {} table regions", self.count);
-        bench(
-            &desc,
-            |i| async move {
-                assert!(self.table_region_manager.remove(i).await.unwrap().is_some());
-            },
-            self.count,
-        )
-        .await;
-    }
-}

src/cmd/src/cli/repl.rs

@@ -16,20 +16,16 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
 
-use catalog::remote::CachedMetaKvBackend;
+use catalog::kvbackend::{CachedMetaKvBackend, KvBackendCatalogManager};
 use client::client_manager::DatanodeClients;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_base::Plugins;
 use common_error::ext::ErrorExt;
-use common_meta::key::TableMetadataManager;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
 use common_telemetry::logging;
 use either::Either;
-use frontend::catalog::FrontendCatalogManager;
 use meta_client::client::MetaClientBuilder;
-use partition::manager::PartitionRuleManager;
-use partition::route::TableRoutes;
 use query::datafusion::DatafusionQueryEngine;
 use query::logical_optimizer::LogicalOptimizer;
 use query::parser::QueryLanguageParser;
@@ -180,7 +176,7 @@ impl Repl {
                 .encode(&plan)
                 .context(SubstraitEncodeLogicalPlanSnafu)?;
 
-            self.database.logical_plan(plan.to_vec(), None).await
+            self.database.logical_plan(plan.to_vec(), 0).await
         } else {
             self.database.sql(&sql).await
         }
@@ -254,24 +250,18 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
 
     let cached_meta_backend = Arc::new(CachedMetaKvBackend::new(meta_client.clone()));
 
-    let table_routes = Arc::new(TableRoutes::new(meta_client));
-    let partition_manager = Arc::new(PartitionRuleManager::new(table_routes));
-
     let datanode_clients = Arc::new(DatanodeClients::default());
 
-    let catalog_list = Arc::new(FrontendCatalogManager::new(
+    let catalog_list = KvBackendCatalogManager::new(
         cached_meta_backend.clone(),
         cached_meta_backend.clone(),
-        partition_manager,
         datanode_clients,
-        Arc::new(TableMetadataManager::new(cached_meta_backend)),
-    ));
+    );
     let plugins: Arc<Plugins> = Default::default();
     let state = Arc::new(QueryEngineState::new(
         catalog_list,
+        None,
         false,
-        None,
-        None,
         plugins.clone(),
     ));
 

src/cmd/src/cli/upgrade.rs

@@ -12,26 +12,37 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 use async_trait::async_trait;
 use clap::Parser;
-use common_meta::helper::TableGlobalValue;
+use client::api::v1::meta::TableRouteValue;
+use common_meta::ddl::utils::region_storage_path;
+use common_meta::error as MetaError;
+use common_meta::key::catalog_name::{CatalogNameKey, CatalogNameValue};
 use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue};
+use common_meta::key::schema_name::{SchemaNameKey, SchemaNameValue};
 use common_meta::key::table_info::{TableInfoKey, TableInfoValue};
 use common_meta::key::table_name::{TableNameKey, TableNameValue};
-use common_meta::key::table_region::{RegionDistribution, TableRegionKey, TableRegionValue};
-use common_meta::key::TableMetaKey;
-use common_meta::rpc::store::{
-    BatchDeleteRequest, BatchPutRequest, PutRequest, RangeRequest, RangeResponse,
-};
+use common_meta::key::table_region::{TableRegionKey, TableRegionValue};
+use common_meta::key::table_route::{TableRouteKey, TableRouteValue as NextTableRouteValue};
+use common_meta::key::{RegionDistribution, TableMetaKey};
+use common_meta::range_stream::PaginationStream;
+use common_meta::rpc::router::TableRoute;
+use common_meta::rpc::store::{BatchDeleteRequest, BatchPutRequest, PutRequest, RangeRequest};
+use common_meta::rpc::KeyValue;
 use common_meta::util::get_prefix_end_key;
 use common_telemetry::info;
 use etcd_client::Client;
+use futures::TryStreamExt;
 use meta_srv::service::store::etcd::EtcdStore;
-use meta_srv::service::store::kv::KvStoreRef;
+use meta_srv::service::store::kv::{KvBackendAdapter, KvStoreRef};
+use prost::Message;
 use snafu::ResultExt;
+use v1_helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue};
 
 use crate::cli::{Instance, Tool};
-use crate::error::{ConnectEtcdSnafu, Result};
+use crate::error::{self, ConnectEtcdSnafu, Result};
 
 #[derive(Debug, Default, Parser)]
 pub struct UpgradeCommand {
@@ -39,6 +50,15 @@ pub struct UpgradeCommand {
     etcd_addr: String,
     #[clap(long)]
     dryrun: bool,
+
+    #[clap(long)]
+    skip_table_global_keys: bool,
+    #[clap(long)]
+    skip_catalog_keys: bool,
+    #[clap(long)]
+    skip_schema_keys: bool,
+    #[clap(long)]
+    skip_table_route_keys: bool,
 }
 
 impl UpgradeCommand {
@@ -51,6 +71,10 @@ impl UpgradeCommand {
         let tool = MigrateTableMetadata {
             etcd_store: EtcdStore::with_etcd_client(client),
             dryrun: self.dryrun,
+            skip_catalog_keys: self.skip_catalog_keys,
+            skip_table_global_keys: self.skip_table_global_keys,
+            skip_schema_keys: self.skip_schema_keys,
+            skip_table_route_keys: self.skip_table_route_keys,
         };
         Ok(Instance::Tool(Box::new(tool)))
     }
@@ -59,60 +83,246 @@ impl UpgradeCommand {
 struct MigrateTableMetadata {
     etcd_store: KvStoreRef,
     dryrun: bool,
+
+    skip_table_global_keys: bool,
+
+    skip_catalog_keys: bool,
+
+    skip_schema_keys: bool,
+
+    skip_table_route_keys: bool,
 }
 
 #[async_trait]
 impl Tool for MigrateTableMetadata {
+    // migrates database's metadata from 0.3 to 0.4.
     async fn do_work(&self) -> Result<()> {
-        let mut key = b"__tg".to_vec();
-        let range_end = get_prefix_end_key(&key);
-
-        let mut processed_keys = 0;
-        loop {
-            info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
-
-            let req = RangeRequest::new()
-                .with_range(key, range_end.clone())
-                .with_limit(1000);
-            let resp = self.etcd_store.range(req).await.unwrap();
-            for kv in resp.kvs.iter() {
-                let key = String::from_utf8_lossy(kv.key());
-                let value = TableGlobalValue::from_bytes(kv.value())
-                    .unwrap_or_else(|e| panic!("table global value is corrupted: {e}, key: {key}"));
-
-                self.create_table_name_key(&value).await;
-
-                self.create_datanode_table_keys(&value).await;
-
-                self.split_table_global_value(&key, value).await;
-            }
-
-            self.delete_migrated_keys(&resp).await;
-
-            processed_keys += resp.kvs.len();
-
-            if resp.more {
-                key = get_prefix_end_key(resp.kvs.last().unwrap().key());
-            } else {
-                break;
-            }
+        if !self.skip_table_global_keys {
+            self.migrate_table_global_values().await?;
+        }
+        if !self.skip_catalog_keys {
+            self.migrate_catalog_keys().await?;
+        }
+        if !self.skip_schema_keys {
+            self.migrate_schema_keys().await?;
+        }
+        if !self.skip_table_route_keys {
+            self.migrate_table_route_keys().await?;
         }
-        info!("Total migrated TableGlobalKeys: {processed_keys}");
         Ok(())
     }
 }
 
+const PAGE_SIZE: usize = 1000;
+
 impl MigrateTableMetadata {
-    async fn delete_migrated_keys(&self, resp: &RangeResponse) {
-        info!("Deleting {} TableGlobalKeys", resp.kvs.len());
-        let req = BatchDeleteRequest {
-            keys: resp.kvs.iter().map(|kv| kv.key().to_vec()).collect(),
-            prev_kv: false,
-        };
+    async fn migrate_table_route_keys(&self) -> Result<()> {
+        let key = b"__meta_table_route".to_vec();
+        let range_end = get_prefix_end_key(&key);
+        let mut keys = Vec::new();
+        info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
+
+        let mut stream = PaginationStream::new(
+            KvBackendAdapter::wrap(self.etcd_store.clone()),
+            RangeRequest::new().with_range(key, range_end),
+            PAGE_SIZE,
+            Arc::new(|kv: KeyValue| {
+                let value =
+                    TableRouteValue::decode(&kv.value[..]).context(MetaError::DecodeProtoSnafu)?;
+                Ok((kv.key, value))
+            }),
+        );
+
+        while let Some((key, value)) = stream.try_next().await.context(error::IterStreamSnafu)? {
+            let table_id = self.migrate_table_route_key(value).await?;
+            keys.push(key);
+            keys.push(TableRegionKey::new(table_id).as_raw_key())
+        }
+
+        info!("Total migrated TableRouteKeys: {}", keys.len() / 2);
+        self.delete_migrated_keys(keys).await;
+
+        Ok(())
+    }
+
+    async fn migrate_table_route_key(&self, value: TableRouteValue) -> Result<u32> {
+        let table_route = TableRoute::try_from_raw(
+            &value.peers,
+            value.table_route.expect("expected table_route"),
+        )
+        .unwrap();
+
+        let new_table_value = NextTableRouteValue::new(table_route.region_routes);
+
+        let table_id = table_route.table.id as u32;
+        let new_key = TableRouteKey::new(table_id);
+        info!("Creating '{new_key}'");
+
         if self.dryrun {
             info!("Dryrun: do nothing");
         } else {
-            self.etcd_store.batch_delete(req).await.unwrap();
+            self.etcd_store
+                .put(
+                    PutRequest::new()
+                        .with_key(new_key.as_raw_key())
+                        .with_value(new_table_value.try_as_raw_value().unwrap()),
+                )
+                .await
+                .unwrap();
+        }
+
+        Ok(table_id)
+    }
+
+    async fn migrate_schema_keys(&self) -> Result<()> {
+        // The schema key prefix.
+        let key = b"__s".to_vec();
+        let range_end = get_prefix_end_key(&key);
+
+        let mut keys = Vec::new();
+        info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
+        let mut stream = PaginationStream::new(
+            KvBackendAdapter::wrap(self.etcd_store.clone()),
+            RangeRequest::new().with_range(key, range_end),
+            PAGE_SIZE,
+            Arc::new(|kv: KeyValue| {
+                let key_str =
+                    std::str::from_utf8(&kv.key).context(MetaError::ConvertRawKeySnafu)?;
+                let key = v1SchemaKey::parse(key_str)
+                    .unwrap_or_else(|e| panic!("schema key is corrupted: {e}, key: {key_str}"));
+
+                Ok((key, ()))
+            }),
+        );
+        while let Some((key, _)) = stream.try_next().await.context(error::IterStreamSnafu)? {
+            let _ = self.migrate_schema_key(&key).await;
+            keys.push(key.to_string().as_bytes().to_vec());
+        }
+        info!("Total migrated SchemaKeys: {}", keys.len());
+        self.delete_migrated_keys(keys).await;
+
+        Ok(())
+    }
+
+    async fn migrate_schema_key(&self, key: &v1SchemaKey) -> Result<()> {
+        let new_key = SchemaNameKey::new(&key.catalog_name, &key.schema_name);
+        let schema_name_value = SchemaNameValue::default();
+
+        info!("Creating '{new_key}'");
+
+        if self.dryrun {
+            info!("Dryrun: do nothing");
+        } else {
+            self.etcd_store
+                .put(
+                    PutRequest::new()
+                        .with_key(new_key.as_raw_key())
+                        .with_value(schema_name_value.try_as_raw_value().unwrap()),
+                )
+                .await
+                .unwrap();
+        }
+
+        Ok(())
+    }
+
+    async fn migrate_catalog_keys(&self) -> Result<()> {
+        // The catalog key prefix.
+        let key = b"__c".to_vec();
+        let range_end = get_prefix_end_key(&key);
+
+        let mut keys = Vec::new();
+        info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
+        let mut stream = PaginationStream::new(
+            KvBackendAdapter::wrap(self.etcd_store.clone()),
+            RangeRequest::new().with_range(key, range_end),
+            PAGE_SIZE,
+            Arc::new(|kv: KeyValue| {
+                let key_str =
+                    std::str::from_utf8(&kv.key).context(MetaError::ConvertRawKeySnafu)?;
+                let key = v1CatalogKey::parse(key_str)
+                    .unwrap_or_else(|e| panic!("catalog key is corrupted: {e}, key: {key_str}"));
+
+                Ok((key, ()))
+            }),
+        );
+        while let Some((key, _)) = stream.try_next().await.context(error::IterStreamSnafu)? {
+            let _ = self.migrate_catalog_key(&key).await;
+            keys.push(key.to_string().as_bytes().to_vec());
+        }
+        info!("Total migrated CatalogKeys: {}", keys.len());
+        self.delete_migrated_keys(keys).await;
+
+        Ok(())
+    }
+
+    async fn migrate_catalog_key(&self, key: &v1CatalogKey) {
+        let new_key = CatalogNameKey::new(&key.catalog_name);
+        let catalog_name_value = CatalogNameValue;
+
+        info!("Creating '{new_key}'");
+
+        if self.dryrun {
+            info!("Dryrun: do nothing");
+        } else {
+            self.etcd_store
+                .put(
+                    PutRequest::new()
+                        .with_key(new_key.as_raw_key())
+                        .with_value(catalog_name_value.try_as_raw_value().unwrap()),
+                )
+                .await
+                .unwrap();
+        }
+    }
+
+    async fn migrate_table_global_values(&self) -> Result<()> {
+        let key = b"__tg".to_vec();
+        let range_end = get_prefix_end_key(&key);
+
+        let mut keys = Vec::new();
+
+        info!("Start scanning key from: {}", String::from_utf8_lossy(&key));
+        let mut stream = PaginationStream::new(
+            KvBackendAdapter::wrap(self.etcd_store.clone()),
+            RangeRequest::new().with_range(key, range_end.clone()),
+            PAGE_SIZE,
+            Arc::new(|kv: KeyValue| {
+                let key = String::from_utf8_lossy(kv.key()).to_string();
+                let value = TableGlobalValue::from_bytes(kv.value())
+                    .unwrap_or_else(|e| panic!("table global value is corrupted: {e}, key: {key}"));
+
+                Ok((key, value))
+            }),
+        );
+        while let Some((key, value)) = stream.try_next().await.context(error::IterStreamSnafu)? {
+            self.create_table_name_key(&value).await;
+
+            self.create_datanode_table_keys(&value).await;
+
+            self.split_table_global_value(&key, value).await;
+
+            keys.push(key.as_bytes().to_vec());
+        }
+
+        info!("Total migrated TableGlobalKeys: {}", keys.len());
+        self.delete_migrated_keys(keys).await;
+
+        Ok(())
+    }
+
+    async fn delete_migrated_keys(&self, keys: Vec<Vec<u8>>) {
+        for keys in keys.chunks(PAGE_SIZE) {
+            info!("Deleting {} keys", keys.len());
+            let req = BatchDeleteRequest {
+                keys: keys.to_vec(),
+                prev_kv: false,
+            };
+            if self.dryrun {
+                info!("Dryrun: do nothing");
+            } else {
+                self.etcd_store.batch_delete(req).await.unwrap();
+            }
         }
     }
 
@@ -177,6 +387,11 @@ impl MigrateTableMetadata {
 
     async fn create_datanode_table_keys(&self, value: &TableGlobalValue) {
         let table_id = value.table_id();
+        let engine = value.table_info.meta.engine.as_str();
+        let region_storage_path = region_storage_path(
+            &value.table_info.catalog_name,
+            &value.table_info.schema_name,
+        );
         let region_distribution: RegionDistribution =
             value.regions_id_map.clone().into_iter().collect();
 
@@ -185,7 +400,15 @@ impl MigrateTableMetadata {
             .map(|(datanode_id, regions)| {
                 let k = DatanodeTableKey::new(datanode_id, table_id);
                 info!("Creating DatanodeTableKey '{k}' => {regions:?}");
-                (k, DatanodeTableValue::new(table_id, regions))
+                (
+                    k,
+                    DatanodeTableValue::new(
+                        table_id,
+                        regions,
+                        engine.to_string(),
+                        region_storage_path.clone(),
+                    ),
+                )
             })
             .collect::<Vec<_>>();
 
@@ -200,3 +423,148 @@ impl MigrateTableMetadata {
         }
     }
 }
+
+#[deprecated(since = "0.4.0", note = "Used for migrate old version(v0.3) metadata")]
+mod v1_helper {
+    use std::collections::HashMap;
+    use std::fmt::{Display, Formatter};
+
+    use err::{DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu};
+    use lazy_static::lazy_static;
+    use regex::Regex;
+    use serde::{Deserialize, Serialize};
+    use snafu::{ensure, OptionExt, ResultExt};
+    use table::metadata::{RawTableInfo, TableId};
+
+    pub const CATALOG_KEY_PREFIX: &str = "__c";
+    pub const SCHEMA_KEY_PREFIX: &str = "__s";
+
+    /// The pattern of a valid catalog, schema or table name.
+    const NAME_PATTERN: &str = "[a-zA-Z_:][a-zA-Z0-9_:]*";
+
+    lazy_static! {
+        static ref CATALOG_KEY_PATTERN: Regex =
+            Regex::new(&format!("^{CATALOG_KEY_PREFIX}-({NAME_PATTERN})$")).unwrap();
+    }
+
+    lazy_static! {
+        static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!(
+            "^{SCHEMA_KEY_PREFIX}-({NAME_PATTERN})-({NAME_PATTERN})$"
+        ))
+        .unwrap();
+    }
+
+    /// Table global info contains necessary info for a datanode to create table regions, including
+    /// table id, table meta(schema...), region id allocation across datanodes.
+    #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+    pub struct TableGlobalValue {
+        /// Id of datanode that created the global table info kv. only for debugging.
+        pub node_id: u64,
+        /// Allocation of region ids across all datanodes.
+        pub regions_id_map: HashMap<u64, Vec<u32>>,
+        pub table_info: RawTableInfo,
+    }
+
+    impl TableGlobalValue {
+        pub fn table_id(&self) -> TableId {
+            self.table_info.ident.table_id
+        }
+    }
+
+    pub struct CatalogKey {
+        pub catalog_name: String,
+    }
+
+    impl Display for CatalogKey {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            f.write_str(CATALOG_KEY_PREFIX)?;
+            f.write_str("-")?;
+            f.write_str(&self.catalog_name)
+        }
+    }
+
+    impl CatalogKey {
+        pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+            let key = s.as_ref();
+            let captures = CATALOG_KEY_PATTERN
+                .captures(key)
+                .context(InvalidCatalogSnafu { key })?;
+            ensure!(captures.len() == 2, InvalidCatalogSnafu { key });
+            Ok(Self {
+                catalog_name: captures[1].to_string(),
+            })
+        }
+    }
+
+    #[derive(Debug, Serialize, Deserialize)]
+    pub struct CatalogValue;
+
+    pub struct SchemaKey {
+        pub catalog_name: String,
+        pub schema_name: String,
+    }
+
+    impl Display for SchemaKey {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            f.write_str(SCHEMA_KEY_PREFIX)?;
+            f.write_str("-")?;
+            f.write_str(&self.catalog_name)?;
+            f.write_str("-")?;
+            f.write_str(&self.schema_name)
+        }
+    }
+
+    impl SchemaKey {
+        pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+            let key = s.as_ref();
+            let captures = SCHEMA_KEY_PATTERN
+                .captures(key)
+                .context(InvalidCatalogSnafu { key })?;
+            ensure!(captures.len() == 3, InvalidCatalogSnafu { key });
+            Ok(Self {
+                catalog_name: captures[1].to_string(),
+                schema_name: captures[2].to_string(),
+            })
+        }
+    }
+
+    #[derive(Debug, Serialize, Deserialize)]
+    pub struct SchemaValue;
+
+    macro_rules! define_catalog_value {
+    ( $($val_ty: ty), *) => {
+            $(
+                impl $val_ty {
+                    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
+                        serde_json::from_str(s.as_ref())
+                            .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() })
+                    }
+
+                    pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self, Error> {
+                         Self::parse(&String::from_utf8_lossy(bytes.as_ref()))
+                    }
+                }
+            )*
+        }
+    }
+
+    define_catalog_value!(TableGlobalValue);
+
+    mod err {
+        use snafu::{Location, Snafu};
+
+        #[derive(Debug, Snafu)]
+        #[snafu(visibility(pub))]
+        pub enum Error {
+            #[snafu(display("Invalid catalog info: {}", key))]
+            InvalidCatalog { key: String, location: Location },
+
+            #[snafu(display("Failed to deserialize catalog entry value: {}", raw))]
+            DeserializeCatalogEntryValue {
+                raw: String,
+                location: Location,
+                source: serde_json::error::Error,
+            },
+        }
+    }
+}

src/cmd/src/datanode.rs

@@ -16,7 +16,8 @@ use std::time::Duration;
 
 use clap::Parser;
 use common_telemetry::logging;
-use datanode::datanode::{Datanode, DatanodeOptions, FileConfig, ObjectStoreConfig};
+use datanode::config::DatanodeOptions;
+use datanode::datanode::{Datanode, DatanodeBuilder};
 use meta_client::MetaClientOptions;
 use servers::Mode;
 use snafu::ResultExt;
@@ -91,8 +92,6 @@ struct StartCommand {
     #[clap(long)]
     data_home: Option<String>,
     #[clap(long)]
-    wal_dir: Option<String>,
-    #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
     http_timeout: Option<u64>,
@@ -129,7 +128,7 @@ impl StartCommand {
         }
 
         if let Some(metasrv_addrs) = &self.metasrv_addr {
-            opts.meta_client_options
+            opts.meta_client
                 .get_or_insert_with(MetaClientOptions::default)
                 .metasrv_addrs = metasrv_addrs.clone();
             opts.mode = Mode::Distributed;
@@ -143,25 +142,19 @@ impl StartCommand {
         }
 
         if let Some(data_home) = &self.data_home {
-            opts.storage.store = ObjectStoreConfig::File(FileConfig {
-                data_home: data_home.clone(),
-            });
-        }
-
-        if let Some(wal_dir) = &self.wal_dir {
-            opts.wal.dir = Some(wal_dir.clone());
+            opts.storage.data_home = data_home.clone();
         }
 
         if let Some(http_addr) = &self.http_addr {
-            opts.http_opts.addr = http_addr.clone();
+            opts.http.addr = http_addr.clone();
         }
 
         if let Some(http_timeout) = self.http_timeout {
-            opts.http_opts.timeout = Duration::from_secs(http_timeout)
+            opts.http.timeout = Duration::from_secs(http_timeout)
         }
 
         // Disable dashboard in datanode.
-        opts.http_opts.disable_dashboard = true;
+        opts.http.disable_dashboard = true;
 
         Ok(Options::Datanode(Box::new(opts)))
     }
@@ -170,7 +163,8 @@ impl StartCommand {
         logging::info!("Datanode start command: {:#?}", self);
         logging::info!("Datanode options: {:#?}", opts);
 
-        let datanode = Datanode::new(opts, Default::default())
+        let datanode = DatanodeBuilder::new(opts, None, Default::default())
+            .build()
             .await
             .context(StartDatanodeSnafu)?;
 
@@ -185,7 +179,7 @@ mod tests {
 
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::datanode::{CompactionConfig, ObjectStoreConfig, RegionManifestConfig};
+    use datanode::config::{CompactionConfig, FileConfig, ObjectStoreConfig, RegionManifestConfig};
     use servers::Mode;
 
     use super::*;
@@ -202,7 +196,7 @@ mod tests {
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
 
-            [meta_client_options]
+            [meta_client]
             metasrv_addrs = ["127.0.0.1:3002"]
             timeout_millis = 3000
             connect_timeout_millis = 5000
@@ -229,7 +223,6 @@ mod tests {
             [storage.manifest]
             checkpoint_margin = 9
             gc_duration = '7s'
-            checkpoint_on_startup = true
             compress = true
 
             [logging]
@@ -243,13 +236,14 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Datanode(options) =
-            cmd.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
+        let Options::Datanode(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
+        else {
+            unreachable!()
+        };
 
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
         assert_eq!(Some(42), options.node_id);
 
-        assert_eq!("/other/wal", options.wal.dir.unwrap());
         assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
         assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
         assert_eq!(1024 * 1024 * 1024 * 50, options.wal.purge_threshold.0);
@@ -261,23 +255,18 @@ mod tests {
             connect_timeout_millis,
             tcp_nodelay,
             ddl_timeout_millis,
-        } = options.meta_client_options.unwrap();
+        } = options.meta_client.unwrap();
 
         assert_eq!(vec!["127.0.0.1:3002".to_string()], metasrv_addr);
         assert_eq!(5000, connect_timeout_millis);
         assert_eq!(10000, ddl_timeout_millis);
         assert_eq!(3000, timeout_millis);
         assert!(tcp_nodelay);
-
-        match &options.storage.store {
-            ObjectStoreConfig::File(FileConfig { data_home, .. }) => {
-                assert_eq!("/tmp/greptimedb/", data_home)
-            }
-            ObjectStoreConfig::S3 { .. } => unreachable!(),
-            ObjectStoreConfig::Oss { .. } => unreachable!(),
-            ObjectStoreConfig::Azblob { .. } => unreachable!(),
-            ObjectStoreConfig::Gcs { .. } => unreachable!(),
-        };
+        assert_eq!("/tmp/greptimedb/", options.storage.data_home);
+        assert!(matches!(
+            &options.storage.store,
+            ObjectStoreConfig::File(FileConfig { .. })
+        ));
 
         assert_eq!(
             CompactionConfig {
@@ -292,7 +281,6 @@ mod tests {
             RegionManifestConfig {
                 checkpoint_margin: Some(9),
                 gc_duration: Some(Duration::from_secs(7)),
-                checkpoint_on_startup: true,
                 compress: true
             },
             options.storage.manifest,
@@ -365,7 +353,7 @@ mod tests {
             rpc_hostname = "127.0.0.1"
             rpc_runtime_size = 8
 
-            [meta_client_options]
+            [meta_client]
             timeout_millis = 3000
             connect_timeout_millis = 5000
             tcp_nodelay = true
@@ -386,9 +374,6 @@ mod tests {
             max_files_in_level0 = 7
             max_purge_tasks = 32
 
-            [storage.manifest]
-            checkpoint_on_startup = true
-
             [logging]
             level = "debug"
             dir = "/tmp/greptimedb/test/logs"
@@ -397,10 +382,10 @@ mod tests {
 
         let env_prefix = "DATANODE_UT";
         temp_env::with_vars(
-            vec![
+            [
                 (
                     // storage.manifest.gc_duration = 9s
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "storage".to_uppercase(),
                         "manifest".to_uppercase(),
@@ -411,7 +396,7 @@ mod tests {
                 ),
                 (
                     // storage.compaction.max_purge_tasks = 99
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "storage".to_uppercase(),
                         "compaction".to_uppercase(),
@@ -421,10 +406,10 @@ mod tests {
                     Some("99"),
                 ),
                 (
-                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
-                    vec![
+                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    [
                         env_prefix.to_string(),
-                        "meta_client_options".to_uppercase(),
+                        "meta_client".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -434,13 +419,15 @@ mod tests {
             || {
                 let command = StartCommand {
                     config_file: Some(file.path().to_str().unwrap().to_string()),
-                    wal_dir: Some("/other/wal/dir".to_string()),
                     env_prefix: env_prefix.to_string(),
                     ..Default::default()
                 };
 
                 let Options::Datanode(opts) =
-                    command.load_options(TopLevelOptions::default()).unwrap() else {unreachable!()};
+                    command.load_options(TopLevelOptions::default()).unwrap()
+                else {
+                    unreachable!()
+                };
 
                 // Should be read from env, env > default values.
                 assert_eq!(
@@ -448,7 +435,7 @@ mod tests {
                     Some(Duration::from_secs(9))
                 );
                 assert_eq!(
-                    opts.meta_client_options.unwrap().metasrv_addrs,
+                    opts.meta_client.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),
@@ -459,9 +446,6 @@ mod tests {
                 // Should be read from config file, config file > env > default values.
                 assert_eq!(opts.storage.compaction.max_purge_tasks, 32);
 
-                // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.wal.dir.unwrap(), "/other/wal/dir");
-
                 // Should be default value.
                 assert_eq!(
                     opts.storage.manifest.checkpoint_margin,

src/cmd/src/error.rs

@@ -23,43 +23,55 @@ use snafu::{Location, Snafu};
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
-    #[snafu(display("Failed to start datanode, source: {}", source))]
+    #[snafu(display("Failed to create default catalog and schema"))]
+    InitMetadata {
+        location: Location,
+        source: common_meta::error::Error,
+    },
+
+    #[snafu(display("Failed to iter stream"))]
+    IterStream {
+        location: Location,
+        source: common_meta::error::Error,
+    },
+
+    #[snafu(display("Failed to start datanode"))]
     StartDatanode {
         location: Location,
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown datanode, source: {}", source))]
+    #[snafu(display("Failed to shutdown datanode"))]
     ShutdownDatanode {
         location: Location,
         source: datanode::error::Error,
     },
 
-    #[snafu(display("Failed to start frontend, source: {}", source))]
+    #[snafu(display("Failed to start frontend"))]
     StartFrontend {
         location: Location,
         source: frontend::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown frontend, source: {}", source))]
+    #[snafu(display("Failed to shutdown frontend"))]
     ShutdownFrontend {
         location: Location,
         source: frontend::error::Error,
     },
 
-    #[snafu(display("Failed to build meta server, source: {}", source))]
+    #[snafu(display("Failed to build meta server"))]
     BuildMetaServer {
         location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to start meta server, source: {}", source))]
+    #[snafu(display("Failed to start meta server"))]
     StartMetaServer {
         location: Location,
         source: meta_srv::error::Error,
     },
 
-    #[snafu(display("Failed to shutdown meta server, source: {}", source))]
+    #[snafu(display("Failed to shutdown meta server"))]
     ShutdownMetaServer {
         location: Location,
         source: meta_srv::error::Error,
@@ -71,13 +83,13 @@ pub enum Error {
     #[snafu(display("Illegal config: {}", msg))]
     IllegalConfig { msg: String, location: Location },
 
-    #[snafu(display("Illegal auth config: {}", source))]
+    #[snafu(display("Illegal auth config"))]
     IllegalAuthConfig {
         location: Location,
-        source: servers::auth::Error,
+        source: auth::error::Error,
     },
 
-    #[snafu(display("Unsupported selector type, {} source: {}", selector_type, source))]
+    #[snafu(display("Unsupported selector type: {}", selector_type))]
     UnsupportedSelectorType {
         selector_type: String,
         location: Location,
@@ -87,75 +99,75 @@ pub enum Error {
     #[snafu(display("Invalid REPL command: {reason}"))]
     InvalidReplCommand { reason: String },
 
-    #[snafu(display("Cannot create REPL: {}", source))]
+    #[snafu(display("Cannot create REPL"))]
     ReplCreation {
         source: ReadlineError,
         location: Location,
     },
 
-    #[snafu(display("Error reading command: {}", source))]
+    #[snafu(display("Error reading command"))]
     Readline {
         source: ReadlineError,
         location: Location,
     },
 
-    #[snafu(display("Failed to request database, sql: {sql}, source: {source}"))]
+    #[snafu(display("Failed to request database, sql: {sql}"))]
     RequestDatabase {
         sql: String,
         location: Location,
         source: client::Error,
     },
 
-    #[snafu(display("Failed to collect RecordBatches, source: {source}"))]
+    #[snafu(display("Failed to collect RecordBatches"))]
     CollectRecordBatches {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to pretty print Recordbatches, source: {source}"))]
+    #[snafu(display("Failed to pretty print Recordbatches"))]
     PrettyPrintRecordBatches {
         location: Location,
         source: common_recordbatch::error::Error,
     },
 
-    #[snafu(display("Failed to start Meta client, source: {}", source))]
+    #[snafu(display("Failed to start Meta client"))]
     StartMetaClient {
         location: Location,
         source: meta_client::error::Error,
     },
 
-    #[snafu(display("Failed to parse SQL: {}, source: {}", sql, source))]
+    #[snafu(display("Failed to parse SQL: {}", sql))]
     ParseSql {
         sql: String,
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to plan statement, source: {}", source))]
+    #[snafu(display("Failed to plan statement"))]
     PlanStatement {
         location: Location,
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to encode logical plan in substrait, source: {}", source))]
+    #[snafu(display("Failed to encode logical plan in substrait"))]
     SubstraitEncodeLogicalPlan {
         location: Location,
         source: substrait::error::Error,
     },
 
-    #[snafu(display("Failed to load layered config, source: {}", source))]
+    #[snafu(display("Failed to load layered config"))]
     LoadLayeredConfig {
         source: ConfigError,
         location: Location,
     },
 
-    #[snafu(display("Failed to start catalog manager, source: {}", source))]
+    #[snafu(display("Failed to start catalog manager"))]
     StartCatalogManager {
         location: Location,
         source: catalog::error::Error,
     },
 
-    #[snafu(display("Failed to connect to Etcd at {etcd_addr}, source: {}", source))]
+    #[snafu(display("Failed to connect to Etcd at {etcd_addr}"))]
     ConnectEtcd {
         etcd_addr: String,
         source: etcd_client::Error,
@@ -176,6 +188,9 @@ impl ErrorExt for Error {
             Error::ShutdownMetaServer { source, .. } => source.status_code(),
             Error::BuildMetaServer { source, .. } => source.status_code(),
             Error::UnsupportedSelectorType { source, .. } => source.status_code(),
+            Error::IterStream { source, .. } | Error::InitMetadata { source, .. } => {
+                source.status_code()
+            }
             Error::MissingConfig { .. }
             | Error::LoadLayeredConfig { .. }
             | Error::IllegalConfig { .. }

src/cmd/src/frontend.rs

@@ -14,19 +14,18 @@
 
 use std::sync::Arc;
 
+use auth::UserProviderRef;
 use clap::Parser;
 use common_base::Plugins;
 use common_telemetry::logging;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
-use frontend::service_config::{InfluxdbOptions, PrometheusOptions};
 use meta_client::MetaClientOptions;
-use servers::auth::UserProviderRef;
 use servers::tls::{TlsMode, TlsOption};
-use servers::{auth, Mode};
+use servers::Mode;
 use snafu::ResultExt;
 
-use crate::error::{self, IllegalAuthConfigSnafu, Result, StartCatalogManagerSnafu};
+use crate::error::{self, IllegalAuthConfigSnafu, Result};
 use crate::options::{Options, TopLevelOptions};
 
 pub struct Instance {
@@ -35,12 +34,6 @@ pub struct Instance {
 
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
-        self.frontend
-            .catalog_manager()
-            .start()
-            .await
-            .context(StartCatalogManagerSnafu)?;
-
         self.frontend
             .start()
             .await
@@ -99,8 +92,6 @@ pub struct StartCommand {
     #[clap(long)]
     mysql_addr: Option<String>,
     #[clap(long)]
-    prom_addr: Option<String>,
-    #[clap(long)]
     postgres_addr: Option<String>,
     #[clap(long)]
     opentsdb_addr: Option<String>,
@@ -147,53 +138,40 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            if let Some(http_opts) = &mut opts.http_options {
-                http_opts.addr = addr.clone()
-            }
+            opts.http.addr = addr.clone()
         }
 
         if let Some(disable_dashboard) = self.disable_dashboard {
-            opts.http_options
-                .get_or_insert_with(Default::default)
-                .disable_dashboard = disable_dashboard;
+            opts.http.disable_dashboard = disable_dashboard;
         }
 
         if let Some(addr) = &self.grpc_addr {
-            if let Some(grpc_opts) = &mut opts.grpc_options {
-                grpc_opts.addr = addr.clone()
-            }
+            opts.grpc.addr = addr.clone()
         }
 
         if let Some(addr) = &self.mysql_addr {
-            if let Some(mysql_opts) = &mut opts.mysql_options {
-                mysql_opts.addr = addr.clone();
-                mysql_opts.tls = tls_opts.clone();
-            }
-        }
-
-        if let Some(addr) = &self.prom_addr {
-            opts.prometheus_options = Some(PrometheusOptions { addr: addr.clone() });
+            opts.mysql.enable = true;
+            opts.mysql.addr = addr.clone();
+            opts.mysql.tls = tls_opts.clone();
         }
 
         if let Some(addr) = &self.postgres_addr {
-            if let Some(postgres_opts) = &mut opts.postgres_options {
-                postgres_opts.addr = addr.clone();
-                postgres_opts.tls = tls_opts;
-            }
+            opts.postgres.enable = true;
+            opts.postgres.addr = addr.clone();
+            opts.postgres.tls = tls_opts;
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            if let Some(opentsdb_addr) = &mut opts.opentsdb_options {
-                opentsdb_addr.addr = addr.clone();
-            }
+            opts.opentsdb.enable = true;
+            opts.opentsdb.addr = addr.clone();
         }
 
         if let Some(enable) = self.influxdb_enable {
-            opts.influxdb_options = Some(InfluxdbOptions { enable });
+            opts.influxdb.enable = enable;
         }
 
         if let Some(metasrv_addrs) = &self.metasrv_addr {
-            opts.meta_client_options
+            opts.meta_client
                 .get_or_insert_with(MetaClientOptions::default)
                 .metasrv_addrs = metasrv_addrs.clone();
             opts.mode = Mode::Distributed;
@@ -236,10 +214,10 @@ mod tests {
     use std::io::Write;
     use std::time::Duration;
 
+    use auth::{Identity, Password, UserProviderRef};
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
     use frontend::service_config::GrpcOptions;
-    use servers::auth::{Identity, Password, UserProviderRef};
 
     use super::*;
     use crate::options::ENV_VAR_SEP;
@@ -248,7 +226,6 @@ mod tests {
     fn test_try_from_start_command() {
         let command = StartCommand {
             http_addr: Some("127.0.0.1:1234".to_string()),
-            prom_addr: Some("127.0.0.1:4444".to_string()),
             mysql_addr: Some("127.0.0.1:5678".to_string()),
             postgres_addr: Some("127.0.0.1:5432".to_string()),
             opentsdb_addr: Some("127.0.0.1:4321".to_string()),
@@ -257,47 +234,34 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Frontend(opts) =
-            command.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
+        let Options::Frontend(opts) = command.load_options(TopLevelOptions::default()).unwrap()
+        else {
+            unreachable!()
+        };
 
-        assert_eq!(opts.http_options.as_ref().unwrap().addr, "127.0.0.1:1234");
-        assert_eq!(
-            ReadableSize::mb(64),
-            opts.http_options.as_ref().unwrap().body_limit
-        );
-        assert_eq!(opts.mysql_options.as_ref().unwrap().addr, "127.0.0.1:5678");
-        assert_eq!(
-            opts.postgres_options.as_ref().unwrap().addr,
-            "127.0.0.1:5432"
-        );
-        assert_eq!(
-            opts.opentsdb_options.as_ref().unwrap().addr,
-            "127.0.0.1:4321"
-        );
-        assert_eq!(
-            opts.prometheus_options.as_ref().unwrap().addr,
-            "127.0.0.1:4444"
-        );
+        assert_eq!(opts.http.addr, "127.0.0.1:1234");
+        assert_eq!(ReadableSize::mb(64), opts.http.body_limit);
+        assert_eq!(opts.mysql.addr, "127.0.0.1:5678");
+        assert_eq!(opts.postgres.addr, "127.0.0.1:5432");
+        assert_eq!(opts.opentsdb.addr, "127.0.0.1:4321");
 
         let default_opts = FrontendOptions::default();
+
+        assert_eq!(opts.grpc.addr, default_opts.grpc.addr);
+        assert!(opts.mysql.enable);
+        assert_eq!(opts.mysql.runtime_size, default_opts.mysql.runtime_size);
+        assert!(opts.postgres.enable);
         assert_eq!(
-            opts.grpc_options.unwrap().addr,
-            default_opts.grpc_options.unwrap().addr
+            opts.postgres.runtime_size,
+            default_opts.postgres.runtime_size
         );
+        assert!(opts.opentsdb.enable);
         assert_eq!(
-            opts.mysql_options.as_ref().unwrap().runtime_size,
-            default_opts.mysql_options.as_ref().unwrap().runtime_size
-        );
-        assert_eq!(
-            opts.postgres_options.as_ref().unwrap().runtime_size,
-            default_opts.postgres_options.as_ref().unwrap().runtime_size
-        );
-        assert_eq!(
-            opts.opentsdb_options.as_ref().unwrap().runtime_size,
-            default_opts.opentsdb_options.as_ref().unwrap().runtime_size
+            opts.opentsdb.runtime_size,
+            default_opts.opentsdb.runtime_size
         );
 
-        assert!(!opts.influxdb_options.unwrap().enable);
+        assert!(!opts.influxdb.enable);
     }
 
     #[test]
@@ -306,7 +270,7 @@ mod tests {
         let toml_str = r#"
             mode = "distributed"
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
             timeout = "30s"
             body_limit = "2GB"
@@ -323,22 +287,15 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Frontend(fe_opts) =
-            command.load_options(TopLevelOptions::default()).unwrap() else {unreachable!()};
+        let Options::Frontend(fe_opts) = command.load_options(TopLevelOptions::default()).unwrap()
+        else {
+            unreachable!()
+        };
         assert_eq!(Mode::Distributed, fe_opts.mode);
-        assert_eq!(
-            "127.0.0.1:4000".to_string(),
-            fe_opts.http_options.as_ref().unwrap().addr
-        );
-        assert_eq!(
-            Duration::from_secs(30),
-            fe_opts.http_options.as_ref().unwrap().timeout
-        );
+        assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
+        assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
 
-        assert_eq!(
-            ReadableSize::gb(2),
-            fe_opts.http_options.as_ref().unwrap().body_limit
-        );
+        assert_eq!(ReadableSize::gb(2), fe_opts.http.body_limit);
 
         assert_eq!("debug", fe_opts.logging.level.as_ref().unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), fe_opts.logging.dir);
@@ -389,57 +346,57 @@ mod tests {
         let toml_str = r#"
             mode = "distributed"
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
 
-            [meta_client_options]
+            [meta_client]
             timeout_millis = 3000
             connect_timeout_millis = 5000
             tcp_nodelay = true
 
-            [mysql_options]
+            [mysql]
             addr = "127.0.0.1:4002"
         "#;
         write!(file, "{}", toml_str).unwrap();
 
         let env_prefix = "FRONTEND_UT";
         temp_env::with_vars(
-            vec![
+            [
                 (
-                    // mysql_options.addr = 127.0.0.1:14002
-                    vec![
+                    // mysql.addr = 127.0.0.1:14002
+                    [
                         env_prefix.to_string(),
-                        "mysql_options".to_uppercase(),
+                        "mysql".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("127.0.0.1:14002"),
                 ),
                 (
-                    // mysql_options.runtime_size = 11
-                    vec![
+                    // mysql.runtime_size = 11
+                    [
                         env_prefix.to_string(),
-                        "mysql_options".to_uppercase(),
+                        "mysql".to_uppercase(),
                         "runtime_size".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("11"),
                 ),
                 (
-                    // http_options.addr = 127.0.0.1:24000
-                    vec![
+                    // http.addr = 127.0.0.1:24000
+                    [
                         env_prefix.to_string(),
-                        "http_options".to_uppercase(),
+                        "http".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
                     Some("127.0.0.1:24000"),
                 ),
                 (
-                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
-                    vec![
+                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    [
                         env_prefix.to_string(),
-                        "meta_client_options".to_uppercase(),
+                        "meta_client".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -458,13 +415,15 @@ mod tests {
                     log_dir: None,
                     log_level: Some("error".to_string()),
                 };
-                let Options::Frontend(fe_opts) =
-                    command.load_options(top_level_opts).unwrap() else {unreachable!()};
+                let Options::Frontend(fe_opts) = command.load_options(top_level_opts).unwrap()
+                else {
+                    unreachable!()
+                };
 
                 // Should be read from env, env > default values.
-                assert_eq!(fe_opts.mysql_options.as_ref().unwrap().runtime_size, 11);
+                assert_eq!(fe_opts.mysql.runtime_size, 11);
                 assert_eq!(
-                    fe_opts.meta_client_options.unwrap().metasrv_addrs,
+                    fe_opts.meta_client.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),
@@ -473,22 +432,13 @@ mod tests {
                 );
 
                 // Should be read from config file, config file > env > default values.
-                assert_eq!(
-                    fe_opts.mysql_options.as_ref().unwrap().addr,
-                    "127.0.0.1:4002"
-                );
+                assert_eq!(fe_opts.mysql.addr, "127.0.0.1:4002");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(
-                    fe_opts.http_options.as_ref().unwrap().addr,
-                    "127.0.0.1:14000"
-                );
+                assert_eq!(fe_opts.http.addr, "127.0.0.1:14000");
 
                 // Should be default value.
-                assert_eq!(
-                    fe_opts.grpc_options.as_ref().unwrap().addr,
-                    GrpcOptions::default().addr
-                );
+                assert_eq!(fe_opts.grpc.addr, GrpcOptions::default().addr);
             },
         );
     }

src/cmd/src/metasrv.rs

@@ -91,9 +91,9 @@ struct StartCommand {
     #[clap(short, long)]
     selector: Option<String>,
     #[clap(long)]
-    use_memory_store: bool,
+    use_memory_store: Option<bool>,
     #[clap(long)]
-    disable_region_failover: bool,
+    enable_region_failover: Option<bool>,
     #[clap(long)]
     http_addr: Option<String>,
     #[clap(long)]
@@ -136,20 +136,24 @@ impl StartCommand {
                 .context(error::UnsupportedSelectorTypeSnafu { selector_type })?;
         }
 
-        opts.use_memory_store = self.use_memory_store;
+        if let Some(use_memory_store) = self.use_memory_store {
+            opts.use_memory_store = use_memory_store;
+        }
 
-        opts.disable_region_failover = self.disable_region_failover;
+        if let Some(enable_region_failover) = self.enable_region_failover {
+            opts.enable_region_failover = enable_region_failover;
+        }
 
         if let Some(http_addr) = &self.http_addr {
-            opts.http_opts.addr = http_addr.clone();
+            opts.http.addr = http_addr.clone();
         }
 
         if let Some(http_timeout) = self.http_timeout {
-            opts.http_opts.timeout = Duration::from_secs(http_timeout);
+            opts.http.timeout = Duration::from_secs(http_timeout);
         }
 
         // Disable dashboard in metasrv.
-        opts.http_opts.disable_dashboard = true;
+        opts.http.disable_dashboard = true;
 
         Ok(Options::Metasrv(Box::new(opts)))
     }
@@ -187,8 +191,10 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Metasrv(options) =
-        cmd.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
+        let Options::Metasrv(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
+        else {
+            unreachable!()
+        };
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!("127.0.0.1:2380".to_string(), options.store_addr);
         assert_eq!(SelectorType::LoadBased, options.selector);
@@ -201,7 +207,6 @@ mod tests {
             bind_addr = "127.0.0.1:3002"
             server_addr = "127.0.0.1:3002"
             store_addr = "127.0.0.1:2379"
-            datanode_lease_secs = 15
             selector = "LeaseBased"
             use_memory_store = false
 
@@ -216,12 +221,13 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Metasrv(options) =
-            cmd.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
+        let Options::Metasrv(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
+        else {
+            unreachable!()
+        };
         assert_eq!("127.0.0.1:3002".to_string(), options.bind_addr);
         assert_eq!("127.0.0.1:3002".to_string(), options.server_addr);
         assert_eq!("127.0.0.1:2379".to_string(), options.store_addr);
-        assert_eq!(15, options.datanode_lease_secs);
         assert_eq!(SelectorType::LeaseBased, options.selector);
         assert_eq!("debug", options.logging.level.as_ref().unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), options.logging.dir);
@@ -258,7 +264,7 @@ mod tests {
             selector = "LeaseBased"
             use_memory_store = false
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
 
             [logging]
@@ -269,22 +275,22 @@ mod tests {
 
         let env_prefix = "METASRV_UT";
         temp_env::with_vars(
-            vec![
+            [
                 (
                     // bind_addr = 127.0.0.1:14002
-                    vec![env_prefix.to_string(), "bind_addr".to_uppercase()].join(ENV_VAR_SEP),
+                    [env_prefix.to_string(), "bind_addr".to_uppercase()].join(ENV_VAR_SEP),
                     Some("127.0.0.1:14002"),
                 ),
                 (
                     // server_addr = 127.0.0.1:13002
-                    vec![env_prefix.to_string(), "server_addr".to_uppercase()].join(ENV_VAR_SEP),
+                    [env_prefix.to_string(), "server_addr".to_uppercase()].join(ENV_VAR_SEP),
                     Some("127.0.0.1:13002"),
                 ),
                 (
-                    // http_options.addr = 127.0.0.1:24000
-                    vec![
+                    // http.addr = 127.0.0.1:24000
+                    [
                         env_prefix.to_string(),
-                        "http_options".to_uppercase(),
+                        "http".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -300,7 +306,10 @@ mod tests {
                 };
 
                 let Options::Metasrv(opts) =
-                    command.load_options(TopLevelOptions::default()).unwrap() else {unreachable!()};
+                    command.load_options(TopLevelOptions::default()).unwrap()
+                else {
+                    unreachable!()
+                };
 
                 // Should be read from env, env > default values.
                 assert_eq!(opts.bind_addr, "127.0.0.1:14002");
@@ -309,7 +318,7 @@ mod tests {
                 assert_eq!(opts.server_addr, "127.0.0.1:3002");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.http_opts.addr, "127.0.0.1:14000");
+                assert_eq!(opts.http.addr, "127.0.0.1:14000");
 
                 // Should be default value.
                 assert_eq!(opts.store_addr, "127.0.0.1:2379");

src/cmd/src/options.rs

@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_config::KvStoreConfig;
 use common_telemetry::logging::LoggingOptions;
 use config::{Config, Environment, File, FileFormat};
-use datanode::datanode::DatanodeOptions;
+use datanode::config::{DatanodeOptions, ProcedureConfig};
 use frontend::frontend::FrontendOptions;
 use meta_srv::metasrv::MetaSrvOptions;
 use serde::{Deserialize, Serialize};
@@ -25,9 +26,13 @@ use crate::error::{LoadLayeredConfigSnafu, Result};
 pub const ENV_VAR_SEP: &str = "__";
 pub const ENV_LIST_SEP: &str = ",";
 
+/// Options mixed up from datanode, frontend and metasrv.
 pub struct MixOptions {
-    pub fe_opts: FrontendOptions,
-    pub dn_opts: DatanodeOptions,
+    pub data_home: String,
+    pub procedure: ProcedureConfig,
+    pub kv_store: KvStoreConfig,
+    pub frontend: FrontendOptions,
+    pub datanode: DatanodeOptions,
     pub logging: LoggingOptions,
 }
 
@@ -115,7 +120,7 @@ mod tests {
     use std::time::Duration;
 
     use common_test_util::temp_dir::create_named_temp_file;
-    use datanode::datanode::{DatanodeOptions, ObjectStoreConfig};
+    use datanode::config::{DatanodeOptions, ObjectStoreConfig};
 
     use super::*;
 
@@ -131,7 +136,7 @@ mod tests {
             mysql_addr = "127.0.0.1:4406"
             mysql_runtime_size = 2
 
-            [meta_client_options]
+            [meta_client]
             timeout_millis = 3000
             connect_timeout_millis = 5000
             tcp_nodelay = true
@@ -158,10 +163,10 @@ mod tests {
         let env_prefix = "DATANODE_UT";
         temp_env::with_vars(
             // The following environment variables will be used to override the values in the config file.
-            vec![
+            [
                 (
                     // storage.manifest.checkpoint_margin = 99
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "storage".to_uppercase(),
                         "manifest".to_uppercase(),
@@ -172,7 +177,7 @@ mod tests {
                 ),
                 (
                     // storage.type = S3
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "storage".to_uppercase(),
                         "type".to_uppercase(),
@@ -182,7 +187,7 @@ mod tests {
                 ),
                 (
                     // storage.bucket = mybucket
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "storage".to_uppercase(),
                         "bucket".to_uppercase(),
@@ -192,7 +197,7 @@ mod tests {
                 ),
                 (
                     // storage.manifest.gc_duration = 42s
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "storage".to_uppercase(),
                         "manifest".to_uppercase(),
@@ -201,20 +206,9 @@ mod tests {
                     .join(ENV_VAR_SEP),
                     Some("42s"),
                 ),
-                (
-                    // storage.manifest.checkpoint_on_startup = true
-                    vec![
-                        env_prefix.to_string(),
-                        "storage".to_uppercase(),
-                        "manifest".to_uppercase(),
-                        "checkpoint_on_startup".to_uppercase(),
-                    ]
-                    .join(ENV_VAR_SEP),
-                    Some("true"),
-                ),
                 (
                     // wal.dir = /other/wal/dir
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "wal".to_uppercase(),
                         "dir".to_uppercase(),
@@ -223,10 +217,10 @@ mod tests {
                     Some("/other/wal/dir"),
                 ),
                 (
-                    // meta_client_options.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
-                    vec![
+                    // meta_client.metasrv_addrs = 127.0.0.1:3001,127.0.0.1:3002,127.0.0.1:3003
+                    [
                         env_prefix.to_string(),
-                        "meta_client_options".to_uppercase(),
+                        "meta_client".to_uppercase(),
                         "metasrv_addrs".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -253,9 +247,8 @@ mod tests {
                     opts.storage.manifest.gc_duration,
                     Some(Duration::from_secs(42))
                 );
-                assert!(opts.storage.manifest.checkpoint_on_startup);
                 assert_eq!(
-                    opts.meta_client_options.unwrap().metasrv_addrs,
+                    opts.meta_client.unwrap().metasrv_addrs,
                     vec![
                         "127.0.0.1:3001".to_string(),
                         "127.0.0.1:3002".to_string(),
@@ -263,9 +256,6 @@ mod tests {
                     ]
                 );
 
-                // Should be the values from config file, not environment variables.
-                assert_eq!(opts.wal.dir.unwrap(), "/tmp/greptimedb/wal");
-
                 // Should be default values.
                 assert_eq!(opts.node_id, None);
             },

src/cmd/src/standalone.rs

@@ -14,18 +14,26 @@
 
 use std::sync::Arc;
 
+use catalog::kvbackend::KvBackendCatalogManager;
+use catalog::CatalogManagerRef;
 use clap::Parser;
 use common_base::Plugins;
+use common_config::{kv_store_dir, KvStoreConfig, WalConfig};
+use common_meta::cache_invalidator::DummyKvCacheInvalidator;
+use common_meta::kv_backend::KvBackendRef;
+use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
-use datanode::datanode::{Datanode, DatanodeOptions, ProcedureConfig, StorageConfig, WalConfig};
-use datanode::instance::InstanceRef;
+use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
+use datanode::datanode::{Datanode, DatanodeBuilder};
+use datanode::region_server::RegionServer;
+use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
-use frontend::instance::{FrontendInstance, Instance as FeInstance};
+use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
-    PrometheusOptions,
 };
+use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -33,8 +41,8 @@ use servers::Mode;
 use snafu::ResultExt;
 
 use crate::error::{
-    IllegalConfigSnafu, Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu,
-    StartFrontendSnafu,
+    IllegalConfigSnafu, InitMetadataSnafu, Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu,
+    StartDatanodeSnafu, StartFrontendSnafu,
 };
 use crate::frontend::load_frontend_plugins;
 use crate::options::{MixOptions, Options, TopLevelOptions};
@@ -46,12 +54,8 @@ pub struct Command {
 }
 
 impl Command {
-    pub async fn build(
-        self,
-        fe_opts: FrontendOptions,
-        dn_opts: DatanodeOptions,
-    ) -> Result<Instance> {
-        self.subcmd.build(fe_opts, dn_opts).await
+    pub async fn build(self, opts: MixOptions) -> Result<Instance> {
+        self.subcmd.build(opts).await
     }
 
     pub fn load_options(&self, top_level_options: TopLevelOptions) -> Result<Options> {
@@ -65,9 +69,9 @@ enum SubCommand {
 }
 
 impl SubCommand {
-    async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
+    async fn build(self, opts: MixOptions) -> Result<Instance> {
         match self {
-            SubCommand::Start(cmd) => cmd.build(fe_opts, dn_opts).await,
+            SubCommand::Start(cmd) => cmd.build(opts).await,
         }
     }
 
@@ -82,38 +86,44 @@ impl SubCommand {
 #[serde(default)]
 pub struct StandaloneOptions {
     pub mode: Mode,
-    pub enable_memory_catalog: bool,
-    pub http_options: Option<HttpOptions>,
-    pub grpc_options: Option<GrpcOptions>,
-    pub mysql_options: Option<MysqlOptions>,
-    pub postgres_options: Option<PostgresOptions>,
-    pub opentsdb_options: Option<OpentsdbOptions>,
-    pub influxdb_options: Option<InfluxdbOptions>,
-    pub prom_store_options: Option<PromStoreOptions>,
-    pub prometheus_options: Option<PrometheusOptions>,
+    pub enable_telemetry: bool,
+    pub http: HttpOptions,
+    pub grpc: GrpcOptions,
+    pub mysql: MysqlOptions,
+    pub postgres: PostgresOptions,
+    pub opentsdb: OpentsdbOptions,
+    pub influxdb: InfluxdbOptions,
+    pub prom_store: PromStoreOptions,
     pub wal: WalConfig,
     pub storage: StorageConfig,
+    pub kv_store: KvStoreConfig,
     pub procedure: ProcedureConfig,
     pub logging: LoggingOptions,
+    /// Options for different store engines.
+    pub region_engine: Vec<RegionEngineConfig>,
 }
 
 impl Default for StandaloneOptions {
     fn default() -> Self {
         Self {
             mode: Mode::Standalone,
-            enable_memory_catalog: false,
-            http_options: Some(HttpOptions::default()),
-            grpc_options: Some(GrpcOptions::default()),
-            mysql_options: Some(MysqlOptions::default()),
-            postgres_options: Some(PostgresOptions::default()),
-            opentsdb_options: Some(OpentsdbOptions::default()),
-            influxdb_options: Some(InfluxdbOptions::default()),
-            prom_store_options: Some(PromStoreOptions::default()),
-            prometheus_options: Some(PrometheusOptions::default()),
+            enable_telemetry: true,
+            http: HttpOptions::default(),
+            grpc: GrpcOptions::default(),
+            mysql: MysqlOptions::default(),
+            postgres: PostgresOptions::default(),
+            opentsdb: OpentsdbOptions::default(),
+            influxdb: InfluxdbOptions::default(),
+            prom_store: PromStoreOptions::default(),
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
+            kv_store: KvStoreConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
+            region_engine: vec![
+                RegionEngineConfig::Mito(MitoConfig::default()),
+                RegionEngineConfig::File(FileEngineConfig::default()),
+            ],
         }
     }
 }
@@ -122,15 +132,14 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
-            http_options: self.http_options,
-            grpc_options: self.grpc_options,
-            mysql_options: self.mysql_options,
-            postgres_options: self.postgres_options,
-            opentsdb_options: self.opentsdb_options,
-            influxdb_options: self.influxdb_options,
-            prom_store_options: self.prom_store_options,
-            prometheus_options: self.prometheus_options,
-            meta_client_options: None,
+            http: self.http,
+            grpc: self.grpc,
+            mysql: self.mysql,
+            postgres: self.postgres,
+            opentsdb: self.opentsdb,
+            influxdb: self.influxdb,
+            prom_store: self.prom_store,
+            meta_client: None,
             logging: self.logging,
             ..Default::default()
         }
@@ -138,10 +147,11 @@ impl StandaloneOptions {
 
     fn datanode_options(self) -> DatanodeOptions {
         DatanodeOptions {
-            enable_memory_catalog: self.enable_memory_catalog,
+            node_id: Some(0),
+            enable_telemetry: self.enable_telemetry,
             wal: self.wal,
             storage: self.storage,
-            procedure: self.procedure,
+            region_engine: self.region_engine,
             ..Default::default()
         }
     }
@@ -155,10 +165,7 @@ pub struct Instance {
 impl Instance {
     pub async fn start(&mut self) -> Result<()> {
         // Start datanode instance before starting services, to avoid requests come in before internal components are started.
-        self.datanode
-            .start_instance()
-            .await
-            .context(StartDatanodeSnafu)?;
+        self.datanode.start().await.context(StartDatanodeSnafu)?;
         info!("Datanode instance started");
 
         self.frontend.start().await.context(StartFrontendSnafu)?;
@@ -172,7 +179,7 @@ impl Instance {
             .context(ShutdownFrontendSnafu)?;
 
         self.datanode
-            .shutdown_instance()
+            .shutdown()
             .await
             .context(ShutdownDatanodeSnafu)?;
         info!("Datanode instance stopped.");
@@ -190,8 +197,6 @@ struct StartCommand {
     #[clap(long)]
     mysql_addr: Option<String>,
     #[clap(long)]
-    prom_addr: Option<String>,
-    #[clap(long)]
     postgres_addr: Option<String>,
     #[clap(long)]
     opentsdb_addr: Option<String>,
@@ -199,8 +204,6 @@ struct StartCommand {
     influxdb_enable: bool,
     #[clap(short, long)]
     config_file: Option<String>,
-    #[clap(short = 'm', long = "memory-catalog")]
-    enable_memory_catalog: bool,
     #[clap(long)]
     tls_mode: Option<TlsMode>,
     #[clap(long)]
@@ -221,8 +224,6 @@ impl StartCommand {
             None,
         )?;
 
-        opts.enable_memory_catalog = self.enable_memory_catalog;
-
         opts.mode = Mode::Standalone;
 
         if let Some(dir) = top_level_options.log_dir {
@@ -240,9 +241,7 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            if let Some(http_opts) = &mut opts.http_options {
-                http_opts.addr = addr.clone()
-            }
+            opts.http.addr = addr.clone()
         }
 
         if let Some(addr) = &self.rpc_addr {
@@ -256,52 +255,52 @@ impl StartCommand {
                 }
                 .fail();
             }
-            if let Some(grpc_opts) = &mut opts.grpc_options {
-                grpc_opts.addr = addr.clone()
-            }
+            opts.grpc.addr = addr.clone()
         }
 
         if let Some(addr) = &self.mysql_addr {
-            if let Some(mysql_opts) = &mut opts.mysql_options {
-                mysql_opts.addr = addr.clone();
-                mysql_opts.tls = tls_opts.clone();
-            }
-        }
-
-        if let Some(addr) = &self.prom_addr {
-            opts.prometheus_options = Some(PrometheusOptions { addr: addr.clone() })
+            opts.mysql.enable = true;
+            opts.mysql.addr = addr.clone();
+            opts.mysql.tls = tls_opts.clone();
         }
 
         if let Some(addr) = &self.postgres_addr {
-            if let Some(postgres_opts) = &mut opts.postgres_options {
-                postgres_opts.addr = addr.clone();
-                postgres_opts.tls = tls_opts;
-            }
+            opts.postgres.enable = true;
+            opts.postgres.addr = addr.clone();
+            opts.postgres.tls = tls_opts;
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            if let Some(opentsdb_addr) = &mut opts.opentsdb_options {
-                opentsdb_addr.addr = addr.clone();
-            }
+            opts.opentsdb.enable = true;
+            opts.opentsdb.addr = addr.clone();
         }
 
         if self.influxdb_enable {
-            opts.influxdb_options = Some(InfluxdbOptions { enable: true });
+            opts.influxdb.enable = self.influxdb_enable;
         }
-
-        let fe_opts = opts.clone().frontend_options();
+        let kv_store = opts.kv_store.clone();
+        let procedure = opts.procedure.clone();
+        let frontend = opts.clone().frontend_options();
         let logging = opts.logging.clone();
-        let dn_opts = opts.datanode_options();
+        let datanode = opts.datanode_options();
 
         Ok(Options::Standalone(Box::new(MixOptions {
-            fe_opts,
-            dn_opts,
+            procedure,
+            kv_store,
+            data_home: datanode.storage.data_home.to_string(),
+            frontend,
+            datanode,
             logging,
         })))
     }
 
-    async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
+    #[allow(unreachable_code)]
+    #[allow(unused_variables)]
+    #[allow(clippy::diverging_sub_expression)]
+    async fn build(self, opts: MixOptions) -> Result<Instance> {
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
+        let fe_opts = opts.frontend;
+        let dn_opts = opts.datanode;
 
         info!("Standalone start command: {:#?}", self);
         info!(
@@ -309,11 +308,40 @@ impl StartCommand {
             fe_opts, dn_opts
         );
 
-        let datanode = Datanode::new(dn_opts.clone(), Default::default())
-            .await
-            .context(StartDatanodeSnafu)?;
+        let kv_dir = kv_store_dir(&opts.data_home);
+        let (kv_store, procedure_manager) =
+            FeInstance::try_build_standalone_components(kv_dir, opts.kv_store, opts.procedure)
+                .await
+                .context(StartFrontendSnafu)?;
 
-        let mut frontend = build_frontend(plugins.clone(), datanode.get_instance()).await?;
+        let datanode =
+            DatanodeBuilder::new(dn_opts.clone(), Some(kv_store.clone()), plugins.clone())
+                .build()
+                .await
+                .context(StartDatanodeSnafu)?;
+        let region_server = datanode.region_server();
+
+        let catalog_manager = KvBackendCatalogManager::new(
+            kv_store.clone(),
+            Arc::new(DummyKvCacheInvalidator),
+            Arc::new(StandaloneDatanodeManager(region_server.clone())),
+        );
+
+        catalog_manager
+            .table_metadata_manager_ref()
+            .init()
+            .await
+            .context(InitMetadataSnafu)?;
+
+        // TODO: build frontend instance like in distributed mode
+        let mut frontend = build_frontend(
+            plugins,
+            kv_store,
+            procedure_manager,
+            catalog_manager,
+            region_server,
+        )
+        .await?;
 
         frontend
             .build_servers(&fe_opts)
@@ -327,12 +355,20 @@ impl StartCommand {
 /// Build frontend instance in standalone mode
 async fn build_frontend(
     plugins: Arc<Plugins>,
-    datanode_instance: InstanceRef,
+    kv_store: KvBackendRef,
+    procedure_manager: ProcedureManagerRef,
+    catalog_manager: CatalogManagerRef,
+    region_server: RegionServer,
 ) -> Result<FeInstance> {
-    let mut frontend_instance = FeInstance::try_new_standalone(datanode_instance.clone())
-        .await
-        .context(StartFrontendSnafu)?;
-    frontend_instance.set_plugins(plugins.clone());
+    let frontend_instance = FeInstance::try_new_standalone(
+        kv_store,
+        procedure_manager,
+        catalog_manager,
+        plugins,
+        region_server,
+    )
+    .await
+    .context(StartFrontendSnafu)?;
     Ok(frontend_instance)
 }
 
@@ -342,9 +378,9 @@ mod tests {
     use std::io::Write;
     use std::time::Duration;
 
+    use auth::{Identity, Password, UserProviderRef};
     use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
-    use servers::auth::{Identity, Password, UserProviderRef};
     use servers::Mode;
 
     use super::*;
@@ -405,11 +441,10 @@ mod tests {
             [storage.manifest]
             checkpoint_margin = 9
             gc_duration = '7s'
-            checkpoint_on_startup = true
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
-            timeout = "30s"
+            timeout = "33s"
             body_limit = "128MB"
 
             [logging]
@@ -423,41 +458,26 @@ mod tests {
             ..Default::default()
         };
 
-        let Options::Standalone(options) = cmd.load_options(TopLevelOptions::default()).unwrap() else {unreachable!()};
-        let fe_opts = options.fe_opts;
-        let dn_opts = options.dn_opts;
+        let Options::Standalone(options) = cmd.load_options(TopLevelOptions::default()).unwrap()
+        else {
+            unreachable!()
+        };
+        let fe_opts = options.frontend;
+        let dn_opts = options.datanode;
         let logging_opts = options.logging;
         assert_eq!(Mode::Standalone, fe_opts.mode);
-        assert_eq!(
-            "127.0.0.1:4000".to_string(),
-            fe_opts.http_options.as_ref().unwrap().addr
-        );
-        assert_eq!(
-            Duration::from_secs(30),
-            fe_opts.http_options.as_ref().unwrap().timeout
-        );
-        assert_eq!(
-            ReadableSize::mb(128),
-            fe_opts.http_options.as_ref().unwrap().body_limit
-        );
-        assert_eq!(
-            "127.0.0.1:4001".to_string(),
-            fe_opts.grpc_options.unwrap().addr
-        );
-        assert_eq!(
-            "127.0.0.1:4002",
-            fe_opts.mysql_options.as_ref().unwrap().addr
-        );
-        assert_eq!(2, fe_opts.mysql_options.as_ref().unwrap().runtime_size);
-        assert_eq!(
-            None,
-            fe_opts.mysql_options.as_ref().unwrap().reject_no_database
-        );
-        assert!(fe_opts.influxdb_options.as_ref().unwrap().enable);
+        assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
+        assert_eq!(Duration::from_secs(33), fe_opts.http.timeout);
+        assert_eq!(ReadableSize::mb(128), fe_opts.http.body_limit);
+        assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.addr);
+        assert!(fe_opts.mysql.enable);
+        assert_eq!("127.0.0.1:4002", fe_opts.mysql.addr);
+        assert_eq!(2, fe_opts.mysql.runtime_size);
+        assert_eq!(None, fe_opts.mysql.reject_no_database);
+        assert!(fe_opts.influxdb.enable);
 
-        assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap());
         match &dn_opts.storage.store {
-            datanode::datanode::ObjectStoreConfig::S3(s3_config) => {
+            datanode::config::ObjectStoreConfig::S3(s3_config) => {
                 assert_eq!(
                     "Secret([REDACTED alloc::string::String])".to_string(),
                     format!("{:?}", s3_config.access_key_id)
@@ -484,7 +504,8 @@ mod tests {
                 log_dir: Some("/tmp/greptimedb/test/logs".to_string()),
                 log_level: Some("debug".to_string()),
             })
-            .unwrap() else {
+            .unwrap()
+        else {
             unreachable!()
         };
 
@@ -498,7 +519,7 @@ mod tests {
         let toml_str = r#"
             mode = "standalone"
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
 
             [logging]
@@ -508,10 +529,10 @@ mod tests {
 
         let env_prefix = "STANDALONE_UT";
         temp_env::with_vars(
-            vec![
+            [
                 (
                     // logging.dir = /other/log/dir
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "logging".to_uppercase(),
                         "dir".to_uppercase(),
@@ -521,7 +542,7 @@ mod tests {
                 ),
                 (
                     // logging.level = info
-                    vec![
+                    [
                         env_prefix.to_string(),
                         "logging".to_uppercase(),
                         "level".to_uppercase(),
@@ -530,10 +551,10 @@ mod tests {
                     Some("info"),
                 ),
                 (
-                    // http_options.addr = 127.0.0.1:24000
-                    vec![
+                    // http.addr = 127.0.0.1:24000
+                    [
                         env_prefix.to_string(),
-                        "http_options".to_uppercase(),
+                        "http".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -552,8 +573,10 @@ mod tests {
                     log_dir: None,
                     log_level: None,
                 };
-                let Options::Standalone(opts) =
-                    command.load_options(top_level_opts).unwrap() else {unreachable!()};
+                let Options::Standalone(opts) = command.load_options(top_level_opts).unwrap()
+                else {
+                    unreachable!()
+                };
 
                 // Should be read from env, env > default values.
                 assert_eq!(opts.logging.dir, "/other/log/dir");
@@ -562,20 +585,11 @@ mod tests {
                 assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(
-                    opts.fe_opts.http_options.as_ref().unwrap().addr,
-                    "127.0.0.1:14000"
-                );
-                assert_eq!(
-                    ReadableSize::mb(64),
-                    opts.fe_opts.http_options.as_ref().unwrap().body_limit
-                );
+                assert_eq!(opts.frontend.http.addr, "127.0.0.1:14000");
+                assert_eq!(ReadableSize::mb(64), opts.frontend.http.body_limit);
 
                 // Should be default value.
-                assert_eq!(
-                    opts.fe_opts.grpc_options.unwrap().addr,
-                    GrpcOptions::default().addr
-                );
+                assert_eq!(opts.frontend.grpc.addr, GrpcOptions::default().addr);
             },
         );
     }

src/common/base/Cargo.toml

@@ -8,7 +8,7 @@ license.workspace = true
 anymap = "1.0.0-beta.2"
 bitvec = "1.0"
 bytes = { version = "1.1", features = ["serde"] }
-common-error = { path = "../error" }
+common-error = { workspace = true }
 paste = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 snafu.workspace = true

src/common/base/src/buffer.rs

@@ -37,7 +37,7 @@ pub enum Error {
     #[snafu(display("Buffer underflow"))]
     Underflow { location: Location },
 
-    #[snafu(display("IO operation reach EOF, source: {}", source))]
+    #[snafu(display("IO operation reach EOF"))]
     Eof {
         source: std::io::Error,
         location: Location,