test: more reduce tests

* feat: add integration-test for interval type.

* chore: add two cases.

* chore: cr

* chore: Field to Column

.cargo/config.toml

@@ -12,5 +12,9 @@ rustflags = [
     "-Wclippy::print_stdout",
     "-Wclippy::print_stderr",
     "-Wclippy::implicit_clone",
-    "-Aclippy::items_after_test_module",
+
+    # It seems clippy has made a false positive decision here when upgrading rust toolchain to
+    # nightly-2023-08-07, we do need it to be borrowed mutably.
+    # Allow it for now; try disallow it when the toolchain is upgraded in the future.
+    "-Aclippy::needless_pass_by_ref_mut",
 ]

.config/nextest.toml

@@ -1,2 +1,3 @@
 [profile.default]
 slow-timeout = { period = "60s", terminate-after = 3, grace-period = "30s" }
+retries = { backoff = "exponential", count = 3, delay = "10s", jitter = true }

.dockerignore

@@ -20,6 +20,3 @@ out/
 
 # Rust
 target/
-
-# Git
-.git

.env.example

@@ -14,4 +14,8 @@ GT_AZBLOB_CONTAINER=AZBLOB container
 GT_AZBLOB_ACCOUNT_NAME=AZBLOB account name
 GT_AZBLOB_ACCOUNT_KEY=AZBLOB account key
 GT_AZBLOB_ENDPOINT=AZBLOB endpoint
-
+# Settings for gcs test 
+GT_GCS_BUCKET = GCS bucket 
+GT_GCS_SCOPE  = GCS scope
+GT_GCS_CREDENTIAL_PATH = GCS credential path 
+GT_GCS_ENDPOINT = GCS end point

.github/actions/build-dev-builder-image/action.yml

@@ -0,0 +1,93 @@
+name: Build and push dev-builder image
+description: Build and push dev-builder image to DockerHub and ACR
+inputs:
+  dockerhub-image-registry:
+    description: The dockerhub image registry to store the images
+    required: false
+    default: docker.io
+  dockerhub-image-registry-username:
+    description: The dockerhub username to login to the image registry
+    required: true
+  dockerhub-image-registry-token:
+    description: The dockerhub token to login to the image registry
+    required: true
+  dockerhub-image-namespace:
+    description: The dockerhub namespace of the image registry to store the images
+    required: false
+    default: greptime
+  acr-image-registry:
+    description: The ACR image registry to store the images
+    required: true
+  acr-image-registry-username:
+    description: The ACR username to login to the image registry
+    required: true
+  acr-image-registry-password:
+    description: The ACR password to login to the image registry
+    required: true
+  acr-image-namespace:
+    description: The ACR namespace of the image registry to store the images
+    required: false
+    default: greptime
+  version:
+    description: Version of the dev-builder
+    required: false
+    default: latest
+runs:
+  using: composite
+  steps:
+    - name: Login to Dockerhub
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.dockerhub-image-registry }}
+        username: ${{ inputs.dockerhub-image-registry-username }}
+        password: ${{ inputs.dockerhub-image-registry-token }}
+
+    - name: Build and push ubuntu dev builder image to dockerhub
+      shell: bash
+      run:
+        make dev-builder \
+        BASE_IMAGE=ubuntu \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push centos dev builder image to dockerhub
+      shell: bash
+      run:
+        make dev-builder \
+        BASE_IMAGE=centos \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Login to ACR
+      uses: docker/login-action@v2
+      continue-on-error: true
+      with:
+        registry: ${{ inputs.acr-image-registry }}
+        username: ${{ inputs.acr-image-registry-username }}
+        password: ${{ inputs.acr-image-registry-password }}
+
+    - name: Build and push ubuntu dev builder image to ACR
+      shell: bash
+      continue-on-error: true
+      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
+        make dev-builder \
+        BASE_IMAGE=ubuntu \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}
+
+    - name: Build and push centos dev builder image to ACR
+      shell: bash
+      continue-on-error: true
+      run: # buildx will cache the images that already built, so it will not take long time to build the images again.
+        make dev-builder \
+        BASE_IMAGE=centos \
+        BUILDX_MULTI_PLATFORM_BUILD=true \
+        IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \
+        IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \
+        IMAGE_TAG=${{ inputs.version }}

.github/actions/build-greptime-binary/action.yml

@@ -0,0 +1,67 @@
+name: Build greptime binary
+description: Build and upload the single linux artifact
+inputs:
+  base-image:
+    description: Base image to build greptime
+    required: true
+  features:
+    description: Cargo features to build
+    required: true
+  cargo-profile:
+    description: Cargo profile to build
+    required: true
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
+  working-dir:
+    description: Working directory to build the artifacts
+    required: false
+    default: .
+runs:
+  using: composite
+  steps:
+    - name: Build greptime binary
+      shell: bash
+      run: |
+        cd ${{ inputs.working-dir }} && \
+        make build-by-dev-builder \
+          CARGO_PROFILE=${{ inputs.cargo-profile }} \
+          FEATURES=${{ inputs.features }} \
+          BASE_IMAGE=${{ inputs.base-image }}
+
+    - name: Upload artifacts
+      uses: ./.github/actions/upload-artifacts
+      with:
+        artifacts-dir: ${{ inputs.artifacts-dir }}
+        target-file: ./target/${{ inputs.cargo-profile }}/greptime
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}

.github/actions/build-greptime-images/action.yml

@@ -0,0 +1,104 @@
+name: Build greptime images
+description: Build and push greptime images
+inputs:
+  image-registry:
+    description: The image registry to store the images
+    required: true
+  image-registry-username:
+    description: The username to login to the image registry
+    required: true
+  image-registry-password:
+    description: The password to login to the image registry
+    required: true
+  amd64-artifact-name:
+    description: The name of the amd64 artifact for building images
+    required: true
+  arm64-artifact-name:
+    description: The name of the arm64 artifact for building images
+    required: false
+    default: ""
+  image-namespace:
+    description: The namespace of the image registry to store the images
+    required: true
+  image-name:
+    description: The name of the image to build
+    required: true
+  image-tag:
+    description: The tag of the image to build
+    required: true
+  docker-file:
+    description: The path to the Dockerfile to build
+    required: true
+  platforms:
+    description: The supported platforms to build the image
+    required: true
+  push-latest-tag:
+    description: Whether to push the latest tag
+    required: false
+    default: 'true'
+runs:
+  using: composite
+  steps:
+    - name: Login to image registry
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.image-registry }}
+        username: ${{ inputs.image-registry-username }}
+        password: ${{ inputs.image-registry-password }}
+
+    - name: Set up qemu for multi-platform builds
+      uses: docker/setup-qemu-action@v2
+
+    - name: Set up buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Download amd64 artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.amd64-artifact-name }}
+
+    - name: Unzip the amd64 artifacts
+      shell: bash
+      run: |
+        tar xvf ${{ inputs.amd64-artifact-name }}.tar.gz && \
+        rm ${{ inputs.amd64-artifact-name }}.tar.gz && \
+        rm -rf amd64 && \
+        mv ${{ inputs.amd64-artifact-name }} amd64
+
+    - name: Download arm64 artifacts
+      uses: actions/download-artifact@v3
+      if: ${{ inputs.arm64-artifact-name }}
+      with:
+        name: ${{ inputs.arm64-artifact-name }}
+
+    - name: Unzip the arm64 artifacts
+      shell: bash
+      if: ${{ inputs.arm64-artifact-name }}
+      run: |
+        tar xvf ${{ inputs.arm64-artifact-name }}.tar.gz && \
+        rm ${{ inputs.arm64-artifact-name }}.tar.gz && \
+        rm -rf arm64 && \
+        mv ${{ inputs.arm64-artifact-name }} arm64
+
+    - name: Build and push images(without latest) for amd64 and arm64
+      if: ${{ inputs.push-latest-tag == 'false' }}
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ${{ inputs.docker-file }}
+        push: true
+        platforms: ${{ inputs.platforms }}
+        tags: |
+          ${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:${{ inputs.image-tag }}
+
+    - name: Build and push images for amd64 and arm64
+      if: ${{ inputs.push-latest-tag == 'true' }}
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ${{ inputs.docker-file }}
+        push: true
+        platforms: ${{ inputs.platforms }}
+        tags: |
+          ${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:latest
+          ${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:${{ inputs.image-tag }}

.github/actions/build-images/action.yml

@@ -0,0 +1,62 @@
+name: Group for building greptimedb images
+description: Group for building greptimedb images
+inputs:
+  image-registry:
+    description: The image registry to store the images
+    required: true
+  image-namespace:
+    description: The namespace of the image registry to store the images
+    required: true
+  image-name:
+    description: The name of the image to build
+    required: false
+    default: greptimedb
+  image-registry-username:
+    description: The username to login to the image registry
+    required: true
+  image-registry-password:
+    description: The password to login to the image registry
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  push-latest-tag:
+    description: Whether to push the latest tag
+    required: false
+    default: 'true'
+  dev-mode:
+    description: Enable dev mode, only build standard greptime
+    required: false
+    default: 'false'
+runs:
+  using: composite
+  steps:
+    - name: Build and push standard images to dockerhub
+      uses: ./.github/actions/build-greptime-images
+      with: # The image will be used as '${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:${{ inputs.version }}'
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}
+        image-registry-username: ${{ inputs.image-registry-username }}
+        image-registry-password: ${{ inputs.image-registry-password }}
+        image-name: ${{ inputs.image-name }}
+        image-tag: ${{ inputs.version }}
+        docker-file: docker/ci/ubuntu/Dockerfile
+        amd64-artifact-name: greptime-linux-amd64-pyo3-${{ inputs.version }}
+        arm64-artifact-name: greptime-linux-arm64-pyo3-${{ inputs.version }}
+        platforms: linux/amd64,linux/arm64
+        push-latest-tag: ${{ inputs.push-latest-tag }}
+
+    - name: Build and push centos images to dockerhub
+      if: ${{ inputs.dev-mode == 'false' }}
+      uses: ./.github/actions/build-greptime-images
+      with:
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}
+        image-registry-username: ${{ inputs.image-registry-username }}
+        image-registry-password: ${{ inputs.image-registry-password }}
+        image-name: ${{ inputs.image-name }}-centos
+        image-tag: ${{ inputs.version }}
+        docker-file: docker/ci/centos/Dockerfile
+        amd64-artifact-name: greptime-linux-amd64-centos-${{ inputs.version }}
+        platforms: linux/amd64
+        push-latest-tag: ${{ inputs.push-latest-tag }}

.github/actions/build-linux-artifacts/action.yml

@@ -0,0 +1,116 @@
+name: Build linux artifacts
+description: Build linux artifacts
+inputs:
+  arch:
+    description: Architecture to build
+    required: true
+  cargo-profile:
+    description: Cargo profile to build
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  disable-run-tests:
+    description: Disable running integration tests
+    required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  dev-mode:
+    description: Enable dev mode, only build standard greptime
+    required: false
+    default: 'false'
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
+  working-dir:
+    description: Working directory to build the artifacts
+    required: false
+    default: .
+runs:
+  using: composite
+  steps:
+    - name: Run integration test
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      shell: bash
+      # NOTE: If the BUILD_JOBS > 4, it's always OOM in EC2 instance.
+      run: |
+        cd ${{ inputs.working-dir }} && \
+        make run-it-in-container BUILD_JOBS=4
+
+    - name: Upload sqlness logs
+      if: ${{ failure() && inputs.disable-run-tests == 'false' }} # Only upload logs when the integration tests failed.
+      uses: actions/upload-artifact@v3
+      with:
+        name: sqlness-logs
+        path: /tmp/greptime-*.log
+        retention-days: 3
+
+    - name: Build standard greptime
+      uses: ./.github/actions/build-greptime-binary
+      with:
+        base-image: ubuntu
+        features: pyo3_backend,servers/dashboard
+        cargo-profile: ${{ inputs.cargo-profile }}
+        artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }}
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}
+
+    - name: Build greptime without pyo3
+      if: ${{ inputs.dev-mode == 'false' }}
+      uses: ./.github/actions/build-greptime-binary
+      with:
+        base-image: ubuntu
+        features: servers/dashboard
+        cargo-profile: ${{ inputs.cargo-profile }}
+        artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}
+
+    - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build.
+      shell: bash
+      run: |
+        rm -rf ./target/
+
+    - name: Build greptime on centos base image
+      uses: ./.github/actions/build-greptime-binary
+      if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Only build centos7 base image for amd64.
+      with:
+        base-image: centos
+        features: servers/dashboard
+        cargo-profile: ${{ inputs.cargo-profile }}
+        artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        upload-to-s3: ${{ inputs.upload-to-s3 }}
+        upload-latest-artifacts: ${{ inputs.upload-latest-artifacts }}
+        working-dir: ${{ inputs.working-dir }}

.github/actions/build-macos-artifacts/action.yml

@@ -0,0 +1,105 @@
+name: Build macos artifacts
+description: Build macos artifacts
+inputs:
+  arch:
+    description: Architecture to build
+    required: true
+  rust-toolchain:
+    description: Rust toolchain to use
+    required: true
+  cargo-profile:
+    description: Cargo profile to build
+    required: true
+  features:
+    description: Cargo features to build
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  disable-run-tests:
+    description: Disable running integration tests
+    required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Cache cargo assets
+      id: cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          target/
+        key: ${{ inputs.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Install protoc
+      shell: bash
+      run: |
+        brew install protobuf
+
+    - name: Install rust toolchain
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ inputs.rust-toolchain }}
+        targets: ${{ inputs.arch }}
+
+    - name: Start etcd # For integration tests.
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      shell: bash
+      run: |
+        brew install etcd && \
+        brew services start etcd
+
+    - name: Install latest nextest release # For integration tests.
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      uses: taiki-e/install-action@nextest
+
+    - name: Run integration tests
+      if: ${{ inputs.disable-run-tests == 'false' }}
+      shell: bash
+      run: |
+        make test sqlness-test
+
+    - name: Upload sqlness logs
+      if: ${{ failure() }} # Only upload logs when the integration tests failed.
+      uses: actions/upload-artifact@v3
+      with:
+        name: sqlness-logs
+        path: /tmp/greptime-*.log
+        retention-days: 3
+
+    - name: Build greptime binary
+      shell: bash
+      run: |
+        make build \
+        CARGO_PROFILE=${{ inputs.cargo-profile }} \
+        FEATURES=${{ inputs.features }} \
+        TARGET=${{ inputs.arch }}
+
+    - name: Upload artifacts
+      uses: ./.github/actions/upload-artifacts
+      with:
+        artifacts-dir: ${{ inputs.artifacts-dir }}
+        target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime
+        version: ${{ inputs.version }}
+        release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}

.github/actions/release-artifacts/action.yml

@@ -0,0 +1,50 @@
+name: Release artifacts
+description: Release artifacts
+inputs:
+  version:
+    description: Version to release
+    required: true
+runs:
+  using: composite
+  steps:
+    # Download artifacts from previous jobs, the artifacts will be downloaded to:
+    # ${WORKING_DIR}
+    #   |- greptime-darwin-amd64-pyo3-v0.5.0/greptime-darwin-amd64-pyo3-v0.5.0.tar.gz
+    #   |- greptime-darwin-amd64-pyo3-v0.5.0.sha256sum/greptime-darwin-amd64-pyo3-v0.5.0.sha256sum
+    #   |- greptime-darwin-amd64-v0.5.0/greptime-darwin-amd64-v0.5.0.tar.gz
+    #   |- greptime-darwin-amd64-v0.5.0.sha256sum/greptime-darwin-amd64-v0.5.0.sha256sum
+    #   ...
+    - name: Download artifacts
+      uses: actions/download-artifact@v3
+
+    - name: Create git tag for release
+      if: ${{ github.event_name != 'push' }} # Meaning this is a scheduled or manual workflow.
+      shell: bash
+      run: |
+        git tag ${{ inputs.version }}
+
+    # Only publish release when the release tag is like v1.0.0, v1.0.1, v1.0.2, etc.
+    - name: Set release arguments
+      shell: bash
+      run: |
+        if [[ "${{ inputs.version }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          echo "prerelease=false" >> $GITHUB_ENV
+          echo "makeLatest=true" >> $GITHUB_ENV
+          echo "generateReleaseNotes=false" >> $GITHUB_ENV
+        else
+          echo "prerelease=true" >> $GITHUB_ENV
+          echo "makeLatest=false" >> $GITHUB_ENV
+          echo "generateReleaseNotes=true" >> $GITHUB_ENV
+        fi
+
+    - name: Publish release
+      uses: ncipollo/release-action@v1
+      with:
+        name: "Release ${{ inputs.version }}"
+        prerelease: ${{ env.prerelease }}
+        makeLatest: ${{ env.makeLatest }}
+        tag: ${{ inputs.version }}
+        generateReleaseNotes: ${{ env.generateReleaseNotes }}
+        allowUpdates: true
+        artifacts: |
+          **/greptime-*/*

.github/actions/start-runner/action.yml

@@ -0,0 +1,67 @@
+name: Start EC2 runner
+description: Start EC2 runner
+inputs:
+  runner:
+    description: The linux runner name
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  github-token:
+    description: The GitHub token to clone private repository
+    required: false
+    default: ""
+  image-id:
+    description: The EC2 image id
+    required: true
+  security-group-id:
+    description: The EC2 security group id
+    required: true
+  subnet-id:
+    description: The EC2 subnet id
+    required: true
+outputs:
+  label:
+    description: "label"
+    value: ${{ steps.start-linux-arm64-ec2-runner.outputs.label || inputs.runner }}
+  ec2-instance-id:
+    description: "ec2-instance-id"
+    value: ${{ steps.start-linux-arm64-ec2-runner.outputs.ec2-instance-id }}
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials
+      if: startsWith(inputs.runner, 'ec2')
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    # The EC2 runner will use the following format:
+    # <vm-type>-<instance-type>-<arch>
+    # like 'ec2-c6a.4xlarge-amd64'.
+    - name: Get EC2 instance type
+      if: startsWith(inputs.runner, 'ec2')
+      id: get-ec2-instance-type
+      shell: bash
+      run: |
+        echo "instance-type=$(echo ${{ inputs.runner }} | cut -d'-' -f2)" >> $GITHUB_OUTPUT
+
+    - name: Start EC2 runner
+      if: startsWith(inputs.runner, 'ec2')
+      uses: machulav/ec2-github-runner@v2
+      id: start-linux-arm64-ec2-runner
+      with:
+        mode: start
+        ec2-image-id: ${{ inputs.image-id }}
+        ec2-instance-type: ${{ steps.get-ec2-instance-type.outputs.instance-type }}
+        subnet-id: ${{ inputs.subnet-id }}
+        security-group-id: ${{ inputs.security-group-id }}
+        github-token: ${{ inputs.github-token }}

.github/actions/stop-runner/action.yml

@@ -0,0 +1,41 @@
+name: Stop EC2 runner
+description: Stop EC2 runner
+inputs:
+  label:
+    description: The linux runner name
+    required: true
+  ec2-instance-id:
+    description: The EC2 instance id
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  github-token:
+    description: The GitHub token to clone private repository
+    required: false
+    default: ""
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials
+      if: ${{ inputs.label && inputs.ec2-instance-id }}
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Stop EC2 runner
+      if: ${{ inputs.label && inputs.ec2-instance-id }}
+      uses: machulav/ec2-github-runner@v2
+      with:
+        mode: stop
+        label: ${{ inputs.label }}
+        ec2-instance-id: ${{ inputs.ec2-instance-id }}
+        github-token: ${{ inputs.github-token }}

.github/actions/upload-artifacts/action.yml

@@ -0,0 +1,125 @@
+name: Upload artifacts
+description: Upload artifacts
+inputs:
+  artifacts-dir:
+    description: Directory to store artifacts
+    required: true
+  target-file:
+    description: The path of the target artifact
+    required: true
+  version:
+    description: Version of the artifact
+    required: true
+  release-to-s3-bucket:
+    description: S3 bucket to store released artifacts
+    required: true
+  aws-access-key-id:
+    description: AWS access key id
+    required: true
+  aws-secret-access-key:
+    description: AWS secret access key
+    required: true
+  aws-region:
+    description: AWS region
+    required: true
+  upload-to-s3:
+    description: Upload to S3
+    required: false
+    default: 'true'
+  upload-latest-artifacts:
+    description: Upload the latest artifacts to S3
+    required: false
+    default: 'true'
+  upload-max-retry-times:
+    description: Max retry times for uploading artifacts to S3
+    required: false
+    default: "20"
+  upload-retry-timeout:
+    description: Timeout for uploading artifacts to S3
+    required: false
+    default: "10" # minutes
+  working-dir:
+    description: Working directory to upload the artifacts
+    required: false
+    default: .
+runs:
+  using: composite
+  steps:
+    - name: Create artifacts directory
+      working-directory: ${{ inputs.working-dir }}
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.artifacts-dir }} && \
+        mv ${{ inputs.target-file }} ${{ inputs.artifacts-dir }}
+
+    # The compressed artifacts will use the following layout:
+    # greptime-linux-amd64-pyo3-v0.3.0sha256sum
+    # greptime-linux-amd64-pyo3-v0.3.0.tar.gz
+    #   greptime-linux-amd64-pyo3-v0.3.0
+    #   └── greptime
+    - name: Compress artifacts and calculate checksum
+      working-directory: ${{ inputs.working-dir }}
+      shell: bash
+      run: |
+        tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }} && \
+        echo $(shasum -a 256 ${{ inputs.artifacts-dir }}.tar.gz | cut -f1 -d' ') > ${{ inputs.artifacts-dir }}.sha256sum
+
+    # Note: The artifacts will be double zip compressed(related issue: https://github.com/actions/upload-artifact/issues/39).
+    # However, when we use 'actions/download-artifact@v3' to download the artifacts, it will be automatically unzipped.
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifacts-dir }}
+        path: ${{ inputs.working-dir }}/${{ inputs.artifacts-dir }}.tar.gz
+
+    - name: Upload checksum
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifacts-dir }}.sha256sum
+        path: ${{ inputs.working-dir }}/${{ inputs.artifacts-dir }}.sha256sum
+
+    - name: Upload artifacts to S3
+      if: ${{ inputs.upload-to-s3 == 'true' }}
+      uses: nick-invision/retry@v2
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
+      with:
+        max_attempts: ${{ inputs.upload-max-retry-times }}
+        timeout_minutes: ${{ inputs.upload-retry-timeout }}
+        # The bucket layout will be:
+        # releases/greptimedb
+        # ├── v0.1.0
+        # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
+        # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
+        # └── v0.2.0
+        #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
+        #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
+        command: |
+          cd ${{ inputs.working-dir }} && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.tar.gz \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.tar.gz && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.sha256sum \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/${{ inputs.version }}/${{ inputs.artifacts-dir }}.sha256sum
+
+    - name: Upload latest artifacts to S3
+      if: ${{ inputs.upload-to-s3 == 'true' && inputs.upload-latest-artifacts == 'true' }} # We'll also upload the latest artifacts to S3 in the scheduled and formal release.
+      uses: nick-invision/retry@v2
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+        AWS_DEFAULT_REGION: ${{ inputs.aws-region }}
+      with:
+        max_attempts: ${{ inputs.upload-max-retry-times }}
+        timeout_minutes: ${{ inputs.upload-retry-timeout }}
+        command: |
+          cd ${{ inputs.working-dir }} && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.tar.gz \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/latest/${{ inputs.artifacts-dir }}.tar.gz && \
+          aws s3 cp \
+            ${{ inputs.artifacts-dir }}.sha256sum \
+            s3://${{ inputs.release-to-s3-bucket }}/releases/greptimedb/latest/${{ inputs.artifacts-dir }}.sha256sum

.github/scripts/create-version.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -e
+
+# - If it's a tag push release, the version is the tag name(${{ github.ref_name }});
+# - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like 'v0.2.0-nightly-20230313';
+# - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-$(git rev-parse --short HEAD)-YYYYMMDDSS', like 'v0.2.0-e5b243c-2023071245';
+# - If it's a nightly build, the version is 'nightly-YYYYMMDD-$(git rev-parse --short HEAD)', like 'nightly-20230712-e5b243c'.
+# create_version ${GIHUB_EVENT_NAME} ${NEXT_RELEASE_VERSION} ${NIGHTLY_RELEASE_PREFIX}
+function create_version() {
+  # Read from envrionment variables.
+  if [ -z "$GITHUB_EVENT_NAME" ]; then
+      echo "GITHUB_EVENT_NAME is empty"
+      exit 1
+  fi
+
+  if [ -z "$NEXT_RELEASE_VERSION" ]; then
+      echo "NEXT_RELEASE_VERSION is empty"
+      exit 1
+  fi
+
+  if [ -z "$NIGHTLY_RELEASE_PREFIX" ]; then
+      echo "NIGHTLY_RELEASE_PREFIX is empty"
+      exit 1
+  fi
+
+  # Reuse $NEXT_RELEASE_VERSION to identify whether it's a nightly build.
+  # It will be like 'nigtly-20230808-7d0d8dc6'.
+  if [ "$NEXT_RELEASE_VERSION" = nightly ]; then
+    echo "$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")-$(git rev-parse --short HEAD)"
+    exit 0
+  fi
+
+  # Reuse $NEXT_RELEASE_VERSION to identify whether it's a dev build.
+  # It will be like 'dev-2023080819-f0e7216c'.
+  if [ "$NEXT_RELEASE_VERSION" = dev ]; then
+    if [ -z "$COMMIT_SHA" ]; then
+      echo "COMMIT_SHA is empty in dev build"
+      exit 1
+    fi
+    echo "dev-$(date "+%Y%m%d-%s")-$(echo "$COMMIT_SHA" | cut -c1-8)"
+    exit 0
+  fi
+
+  # Note: Only output 'version=xxx' to stdout when everything is ok, so that it can be used in GitHub Actions Outputs.
+  if [ "$GITHUB_EVENT_NAME" = push ]; then
+    if [ -z "$GITHUB_REF_NAME" ]; then
+      echo "GITHUB_REF_NAME is empty in push event"
+      exit 1
+    fi
+    echo "$GITHUB_REF_NAME"
+  elif [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
+    echo "$NEXT_RELEASE_VERSION-$(git rev-parse --short HEAD)-$(date "+%Y%m%d-%s")"
+  elif [ "$GITHUB_EVENT_NAME" = schedule ]; then
+    echo "$NEXT_RELEASE_VERSION-$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")"
+  else
+    echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME"
+    exit 1
+  fi
+}
+
+# You can run as following examples:
+#  GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly GITHUB_REF_NAME=v0.3.0 ./create-version.sh
+#  GITHUB_EVENT_NAME=workflow_dispatch NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=nightly NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+#  GITHUB_EVENT_NAME=workflow_dispatch COMMIT_SHA=f0e7216c4bb6acce9b29a21ec2d683be2e3f984a NEXT_RELEASE_VERSION=dev NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh
+create_version

.github/workflows/apidoc.yml

@@ -13,7 +13,7 @@ on:
 name: Build API docs
 
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-03
+  RUST_TOOLCHAIN: nightly-2023-08-07
 
 jobs:
   apidoc:

.github/workflows/dev-build.yml

@@ -0,0 +1,344 @@
+# Development build only build the debug version of the artifacts manually.
+name: GreptimeDB Development Build
+
+on:
+  workflow_dispatch: # Allows you to run this workflow manually.
+    inputs:
+      repository:
+        description: The public repository to build
+        required: false
+        default: GreptimeTeam/greptimedb
+      commit: # Note: We only pull the source code and use the current workflow to build the artifacts.
+        description: The commit to build
+        required: true
+      linux_amd64_runner:
+        type: choice
+        description: The runner uses to build linux-amd64 artifacts
+        default: ec2-c6i.4xlarge-amd64
+        options:
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
+          - ec2-c6i.xlarge-amd64 # 4C8G
+          - ec2-c6i.2xlarge-amd64 # 8C16G
+          - ec2-c6i.4xlarge-amd64 # 16C32G
+          - ec2-c6i.8xlarge-amd64 # 32C64G
+          - ec2-c6i.16xlarge-amd64 # 64C128G
+      linux_arm64_runner:
+        type: choice
+        description: The runner uses to build linux-arm64 artifacts
+        default: ec2-c6g.4xlarge-arm64
+        options:
+          - ec2-c6g.xlarge-arm64 # 4C8G
+          - ec2-c6g.2xlarge-arm64 # 8C16G
+          - ec2-c6g.4xlarge-arm64 # 16C32G
+          - ec2-c6g.8xlarge-arm64 # 32C64G
+          - ec2-c6g.16xlarge-arm64 # 64C128G
+      skip_test:
+        description: Do not run integration tests during the build
+        type: boolean
+        default: true
+      build_linux_amd64_artifacts:
+        type: boolean
+        description: Build linux-amd64 artifacts
+        required: false
+        default: true
+      build_linux_arm64_artifacts:
+        type: boolean
+        description: Build linux-arm64 artifacts
+        required: false
+        default: true
+      release_images:
+        type: boolean
+        description: Build and push images to DockerHub and ACR
+        required: false
+        default: true
+
+# Use env variables to control all the release process.
+env:
+  CARGO_PROFILE: nightly
+
+  # Controls whether to run tests, include unit-test, integration-test and sqlness.
+  DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}
+
+  # Always use 'dev' to indicate it's the dev build.
+  NEXT_RELEASE_VERSION: dev
+
+  NIGHTLY_RELEASE_PREFIX: nightly
+
+  # Use the different image name to avoid conflict with the release images.
+  IMAGE_NAME: greptimedb-dev
+
+  # The source code will check out in the following path: '${WORKING_DIR}/dev/greptime'.
+  CHECKOUT_GREPTIMEDB_PATH: dev/greptimedb
+
+jobs:
+  allocate-runners:
+    name: Allocate runners
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-latest
+    outputs:
+      linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
+
+      # The following EC2 resource id will be used for resource releasing.
+      linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }}
+      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}
+
+      # The 'version' use as the global tag name of the release workflow.
+      version: ${{ steps.create-version.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create version
+        id: create-version
+        run: |
+          version=$(./.github/scripts/create-version.sh) && \
+          echo $version && \
+          echo "version=$version" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+          COMMIT_SHA: ${{ inputs.commit }}
+          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
+          NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
+
+      - name: Allocate linux-amd64 runner
+        if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-amd64-runner
+        with:
+          runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+      - name: Allocate linux-arm64 runner
+        if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-arm64-runner
+        with:
+          runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+  build-linux-amd64-artifacts:
+    name: Build linux-amd64 artifacts
+    if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Checkout greptimedb
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.repository }}
+          ref: ${{ inputs.commit }}
+          path: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: amd64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: true # Only build the standard greptime binary.
+          upload-to-s3: false # No need to upload to S3.
+          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+
+  build-linux-arm64-artifacts:
+    name: Build linux-arm64 artifacts
+    if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Checkout greptimedb
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.repository }}
+          ref: ${{ inputs.commit }}
+          path: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: arm64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          dev-mode: true # Only build the standard greptime binary.
+          upload-to-s3: false # No need to upload to S3.
+          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+
+  release-images-to-dockerhub:
+    name: Build and push images to DockerHub
+    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-latest
+    outputs:
+      build-result: ${{ steps.set-build-result.outputs.build-result }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push images to dockerhub
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: docker.io
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-name: ${{ env.IMAGE_NAME }}
+          image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          push-latest-tag: false # Don't push the latest tag to registry.
+          dev-mode: true # Only build the standard images.
+
+      - name: Set build result
+        id: set-build-result
+        run: |
+          echo "build-result=success" >> $GITHUB_OUTPUT
+
+  release-images-to-acr:
+    name: Build and push images to ACR
+    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-latest
+    # When we push to ACR, it's easy to fail due to some unknown network issues.
+    # However, we don't want to fail the whole workflow because of this.
+    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-name: ${{ env.IMAGE_NAME }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          push-latest-tag: false # Don't push the latest tag to registry.
+          dev-mode: true # Only build the standard images.
+
+  stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-amd64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-arm64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-arm64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  notification:
+    if: ${{ always() }} # Not requiring successful dependent jobs, always run.
+    name: Send notification to Greptime team
+    needs: [
+      release-images-to-dockerhub
+    ]
+    runs-on: ubuntu-latest
+    env:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
+    steps:
+      - name: Notifiy nightly build successful result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }}
+        with:
+          payload: |
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}
+
+      - name: Notifiy nightly build failed result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.release-images-to-dockerhub.outputs.build-result != 'success' }}
+        with:
+          payload: |
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."}

.github/workflows/develop.yml

@@ -1,4 +1,5 @@
 on:
+  merge_group:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
     paths-ignore:
@@ -23,8 +24,12 @@ on:
 
 name: CI
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-03
+  RUST_TOOLCHAIN: nightly-2023-08-07
 
 jobs:
   typos:
@@ -50,7 +55,7 @@ jobs:
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
       - name: Run cargo check
-        run: cargo check --workspace --all-targets
+        run: cargo check --locked --workspace --all-targets
 
   toml:
     name: Toml Check
@@ -61,62 +66,22 @@ jobs:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          toolchain: stable
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
       - name: Install taplo
-        run: cargo install taplo-cli --version ^0.8 --locked
+        run: cargo +stable install taplo-cli --version ^0.8 --locked
       - name: Run taplo
-        run: taplo format --check --option "indent_string=    "
-
-  # Use coverage to run test.
-  # test:
-  #   name: Test Suite
-  #   if: github.event.pull_request.draft == false
-  #   runs-on: ubuntu-latest
-  #   timeout-minutes: 60
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - name: Cache LLVM and Clang
-  #       id: cache-llvm
-  #       uses: actions/cache@v3
-  #       with:
-  #         path: ./llvm
-  #         key: llvm
-  #     - uses: arduino/setup-protoc@v1
-  #       with:
-  #         repo-token: ${{ secrets.GITHUB_TOKEN }}
-  #     - uses: KyleMayes/install-llvm-action@v1
-  #       with:
-  #         version: "14.0"
-  #         cached: ${{ steps.cache-llvm.outputs.cache-hit }}
-  #     - uses: dtolnay/rust-toolchain@master
-  #       with:
-  #         toolchain: ${{ env.RUST_TOOLCHAIN }}
-  #     - name: Rust Cache
-  #       uses: Swatinem/rust-cache@v2
-  #     - name: Cleanup disk
-  #       uses: curoky/cleanup-disk-action@v2.0
-  #       with:
-  #         retain: 'rust,llvm'
-  #     - name: Install latest nextest release
-  #       uses: taiki-e/install-action@nextest
-  #     - name: Run tests
-  #       run: cargo nextest run
-  #       env:
-  #         CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
-  #         RUST_BACKTRACE: 1
-  #         GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
-  #         GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-  #         GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-  #         UNITTEST_LOG_DIR: "__unittest_logs"
+        run: taplo format --check
 
   sqlness:
     name: Sqlness Test
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest-8-cores, windows-latest-8-cores ]
     timeout-minutes: 60
-    needs: [clippy]
     steps:
       - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
@@ -127,25 +92,14 @@ jobs:
           toolchain: ${{ env.RUST_TOOLCHAIN }}
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-      - name: Run etcd
-        run: |
-          ETCD_VER=v3.5.7
-          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
-          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-          mkdir -p /tmp/etcd-download
-          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
-          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-
-          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
-          nohup etcd >/tmp/etcd.log 2>&1 &
       - name: Run sqlness
-        run: cargo sqlness && ls /tmp
+        run: cargo sqlness
       - name: Upload sqlness logs
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: sqlness-logs
-          path: /tmp/greptime-*.log
+          path: ${{ runner.temp }}/greptime-*.log
           retention-days: 3
 
   fmt:
@@ -190,7 +144,6 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest-8-cores
     timeout-minutes: 60
-    needs: [clippy]
     steps:
       - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
@@ -235,3 +188,43 @@ jobs:
           flags: rust
           fail_ci_if_error: false
           verbose: true
+
+  test-on-windows:
+    if: github.event.pull_request.draft == false
+    runs-on: windows-latest-8-cores
+    timeout-minutes: 60
+    steps:
+      - run: git config --global core.autocrlf false
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: llvm-tools-preview
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install PyArrow Package
+        run: pip install pyarrow
+      - name: Install WSL distribution
+        uses: Vampire/setup-wsl@v2
+        with:
+          distribution: Ubuntu-22.04
+      - name: Running tests
+        run: cargo nextest run -F pyo3_backend,dashboard
+        env:
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          GT_S3_REGION: ${{ secrets.S3_REGION }}
+          UNITTEST_LOG_DIR: "__unittest_logs"

.github/workflows/docs.yml

@@ -1,4 +1,5 @@
 on:
+  merge_group:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
     paths:
@@ -27,6 +28,13 @@ name: CI
 # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
 
 jobs:
+  typos:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: crate-ci/typos@v1.13.10
+
   check:
     name: Check
     if: github.event.pull_request.draft == false
@@ -53,3 +61,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - run: 'echo "No action required"'
+
+  sqlness:
+    name: Sqlness Test
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No action required"'

.github/workflows/nightly-build.yml

@@ -0,0 +1,311 @@
+# Nightly build only do the following things:
+# 1. Run integration tests;
+# 2. Build binaries and images for linux-amd64 and linux-arm64 platform;
+name: GreptimeDB Nightly Build
+
+on:
+  schedule:
+    # Trigger at 00:00(UTC) on every day-of-week from Monday through Friday.
+    - cron: '0 0 * * 1-5'
+  workflow_dispatch: # Allows you to run this workflow manually.
+    inputs:
+      linux_amd64_runner:
+        type: choice
+        description: The runner uses to build linux-amd64 artifacts
+        default: ec2-c6i.2xlarge-amd64
+        options:
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
+          - ec2-c6i.xlarge-amd64 # 4C8G
+          - ec2-c6i.2xlarge-amd64 # 8C16G
+          - ec2-c6i.4xlarge-amd64 # 16C32G
+          - ec2-c6i.8xlarge-amd64 # 32C64G
+          - ec2-c6i.16xlarge-amd64 # 64C128G
+      linux_arm64_runner:
+        type: choice
+        description: The runner uses to build linux-arm64 artifacts
+        default: ec2-c6g.2xlarge-arm64
+        options:
+          - ec2-c6g.xlarge-arm64 # 4C8G
+          - ec2-c6g.2xlarge-arm64 # 8C16G
+          - ec2-c6g.4xlarge-arm64 # 16C32G
+          - ec2-c6g.8xlarge-arm64 # 32C64G
+          - ec2-c6g.16xlarge-arm64 # 64C128G
+      skip_test:
+        description: Do not run integration tests during the build
+        type: boolean
+        default: true
+      build_linux_amd64_artifacts:
+        type: boolean
+        description: Build linux-amd64 artifacts
+        required: false
+        default: false
+      build_linux_arm64_artifacts:
+        type: boolean
+        description: Build linux-arm64 artifacts
+        required: false
+        default: false
+      release_images:
+        type: boolean
+        description: Build and push images to DockerHub and ACR
+        required: false
+        default: false
+
+# Use env variables to control all the release process.
+env:
+  CARGO_PROFILE: nightly
+
+  # Controls whether to run tests, include unit-test, integration-test and sqlness.
+  DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}
+
+  # Always use 'nightly' to indicate it's the nightly build.
+  NEXT_RELEASE_VERSION: nightly
+
+  NIGHTLY_RELEASE_PREFIX: nightly
+
+jobs:
+  allocate-runners:
+    name: Allocate runners
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-latest
+    outputs:
+      linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
+
+      # The following EC2 resource id will be used for resource releasing.
+      linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }}
+      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}
+
+      # The 'version' use as the global tag name of the release workflow.
+      version: ${{ steps.create-version.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create version
+        id: create-version
+        run: |
+          version=$(./.github/scripts/create-version.sh) && \
+          echo $version && \
+          echo "version=$version" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
+          NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
+
+      - name: Allocate linux-amd64 runner
+        if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-amd64-runner
+        with:
+          runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+      - name: Allocate linux-arm64 runner
+        if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-arm64-runner
+        with:
+          runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+  build-linux-amd64-artifacts:
+    name: Build linux-amd64 artifacts
+    if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: amd64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-latest-artifacts: false
+
+  build-linux-arm64-artifacts:
+    name: Build linux-arm64 artifacts
+    if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: arm64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+          upload-latest-artifacts: false
+
+  release-images-to-dockerhub:
+    name: Build and push images to DockerHub
+    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-latest
+    outputs:
+      nightly-build-result: ${{ steps.set-nightly-build-result.outputs.nightly-build-result }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push images to dockerhub
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: docker.io
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          push-latest-tag: false # Don't push the latest tag to registry.
+
+      - name: Set nightly build result
+        id: set-nightly-build-result
+        run: |
+          echo "nightly-build-result=success" >> $GITHUB_OUTPUT
+
+  release-images-to-acr:
+    name: Build and push images to ACR
+    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-latest
+    # When we push to ACR, it's easy to fail due to some unknown network issues.
+    # However, we don't want to fail the whole workflow because of this.
+    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          push-latest-tag: false # Don't push the latest tag to registry.
+
+  stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-amd64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-arm64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-arm64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  notification:
+    if: ${{ always() }} # Not requiring successful dependent jobs, always run.
+    name: Send notification to Greptime team
+    needs: [
+      release-images-to-dockerhub
+    ]
+    runs-on: ubuntu-latest
+    env:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
+    steps:
+      - name: Notifiy nightly build successful result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result == 'success' }}
+        with:
+          payload: |
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}
+
+      - name: Notifiy nightly build failed result
+        uses: slackapi/slack-github-action@v1.23.0
+        if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result != 'success' }}
+        with:
+          payload: |
+            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."}

.github/workflows/release.yml

@@ -1,3 +1,8 @@
+name: Release
+
+# There are two kinds of formal release:
+# 1. The tag('v*.*.*') push release: the release workflow will be triggered by the tag push event.
+# 2. The scheduled release(the version will be '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD'): the release workflow will be triggered by the schedule event.
 on:
   push:
     tags:
@@ -5,464 +10,386 @@ on:
   schedule:
     # At 00:00 on Monday.
     - cron: '0 0 * * 1'
-  # Mannually trigger only builds binaries.
-  workflow_dispatch:
-
-name: Release
+  workflow_dispatch: # Allows you to run this workflow manually.
+    # Notes: The GitHub Actions ONLY support 10 inputs, and it's already used up.
+    inputs:
+      linux_amd64_runner:
+        type: choice
+        description: The runner uses to build linux-amd64 artifacts
+        default: ec2-c6i.4xlarge-amd64
+        options:
+          - ubuntu-latest
+          - ubuntu-latest-8-cores
+          - ubuntu-latest-16-cores
+          - ubuntu-latest-32-cores
+          - ubuntu-latest-64-cores
+          - ec2-c6i.xlarge-amd64 # 4C8G
+          - ec2-c6i.2xlarge-amd64 # 8C16G
+          - ec2-c6i.4xlarge-amd64 # 16C32G
+          - ec2-c6i.8xlarge-amd64 # 32C64G
+          - ec2-c6i.16xlarge-amd64 # 64C128G
+      linux_arm64_runner:
+        type: choice
+        description: The runner uses to build linux-arm64 artifacts
+        default: ec2-c6g.4xlarge-arm64
+        options:
+          - ec2-c6g.xlarge-arm64 # 4C8G
+          - ec2-c6g.2xlarge-arm64 # 8C16G
+          - ec2-c6g.4xlarge-arm64 # 16C32G
+          - ec2-c6g.8xlarge-arm64 # 32C64G
+          - ec2-c6g.16xlarge-arm64 # 64C128G
+      macos_runner:
+        type: choice
+        description: The runner uses to build macOS artifacts
+        default: macos-latest
+        options:
+          - macos-latest
+      skip_test:
+        description: Do not run integration tests during the build
+        type: boolean
+        default: true
+      build_linux_amd64_artifacts:
+        type: boolean
+        description: Build linux-amd64 artifacts
+        required: false
+        default: false
+      build_linux_arm64_artifacts:
+        type: boolean
+        description: Build linux-arm64 artifacts
+        required: false
+        default: false
+      build_macos_artifacts:
+        type: boolean
+        description: Build macos artifacts
+        required: false
+        default: false
+      release_artifacts:
+        type: boolean
+        description: Create GitHub release and upload artifacts
+        required: false
+        default: false
+      release_images:
+        type: boolean
+        description: Build and push images to DockerHub and ACR
+        required: false
+        default: false
+      release_dev_builder_image:
+        type: boolean
+        description: Release dev-builder image
+        required: false
+        default: false
 
+# Use env variables to control all the release process.
 env:
-  RUST_TOOLCHAIN: nightly-2023-05-03
-
-  SCHEDULED_BUILD_VERSION_PREFIX: v0.3.0
-
-  SCHEDULED_PERIOD: nightly
-
+  # The arguments of building greptime.
+  RUST_TOOLCHAIN: nightly-2023-08-07
   CARGO_PROFILE: nightly
 
   # Controls whether to run tests, include unit-test, integration-test and sqlness.
-  DISABLE_RUN_TESTS: false
+  DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}
+
+  # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
+  NIGHTLY_RELEASE_PREFIX: nightly
+  # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
+  NEXT_RELEASE_VERSION: v0.4.0
 
 jobs:
-  build-macos:
-    name: Build macOS binary
+  allocate-runners:
+    name: Allocate runners
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-latest
+    outputs:
+      linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      macos-runner: ${{ inputs.macos_runner || vars.DEFAULT_MACOS_RUNNER }}
+
+      # The following EC2 resource id will be used for resource releasing.
+      linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
+      linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }}
+      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}
+
+      # The 'version' use as the global tag name of the release workflow.
+      version: ${{ steps.create-version.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # The create-version will create a global variable named 'version' in the global workflows.
+      # - If it's a tag push release, the version is the tag name(${{ github.ref_name }});
+      # - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nigthly-20230313;
+      # - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-<short-git-sha>-YYYYMMDDSS', like v0.2.0-e5b243c-2023071245;
+      - name: Create version
+        id: create-version
+        run: |
+          echo "version=$(./.github/scripts/create-version.sh)" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
+          NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}
+
+      - name: Allocate linux-amd64 runner
+        if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-amd64-runner
+        with:
+          runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+      - name: Allocate linux-arm64 runner
+        if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+        uses: ./.github/actions/start-runner
+        id: start-linux-arm64-runner
+        with:
+          runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+  build-linux-amd64-artifacts:
+    name: Build linux-amd64 artifacts
+    if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: amd64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+
+  build-linux-arm64-artifacts:
+    name: Build linux-arm64 artifacts
+    if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+    ]
+    runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/build-linux-artifacts
+        with:
+          arch: arm64
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
+
+  build-macos-artifacts:
+    name: Build macOS artifacts
     strategy:
+      fail-fast: false
       matrix:
-        # The file format is greptime-<os>-<arch>
         include:
-          - arch: aarch64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-arm64
-            continue-on-error: false
-            opts: "-F servers/dashboard"
-          - arch: x86_64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-amd64
-            continue-on-error: false
-            opts: "-F servers/dashboard"
-          - arch: aarch64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-arm64-pyo3
-            continue-on-error: false
-            opts: "-F pyo3_backend,servers/dashboard"
-          - arch: x86_64-apple-darwin
-            os: macos-latest
-            file: greptime-darwin-amd64-pyo3
-            continue-on-error: false
-            opts: "-F pyo3_backend,servers/dashboard"
+          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
+            arch: aarch64-apple-darwin
+            features: servers/dashboard
+            artifacts-dir-prefix: greptime-darwin-arm64
+          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
+            arch: aarch64-apple-darwin
+            features: pyo3_backend,servers/dashboard
+            artifacts-dir-prefix: greptime-darwin-arm64-pyo3
+          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
+            features: servers/dashboard
+            arch: x86_64-apple-darwin
+            artifacts-dir-prefix: greptime-darwin-amd64
+          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
+            features: pyo3_backend,servers/dashboard
+            arch: x86_64-apple-darwin
+            artifacts-dir-prefix: greptime-darwin-amd64-pyo3
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.continue-on-error }}
-    if: github.repository == 'GreptimeTeam/greptimedb'
+    needs: [
+      allocate-runners,
+    ]
+    if: ${{ inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Cache cargo assets
-        id: cache
-        uses: actions/cache@v3
+      - uses: actions/checkout@v3
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+          fetch-depth: 0
 
-      - name: Install Protoc for macos
-        if: contains(matrix.arch, 'darwin')
-        run: |
-          brew install protobuf
-
-      - name: Install etcd for macos
-        if: contains(matrix.arch, 'darwin')
-        run: |
-          brew install etcd
-          brew services start etcd
-
-      - name: Install rust toolchain
-        uses: dtolnay/rust-toolchain@master
+      - uses: ./.github/actions/build-macos-artifacts
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          targets: ${{ matrix.arch }}
-      - name: Install latest nextest release
-        uses: taiki-e/install-action@nextest
-      - name: Output package versions
-        run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version
+          arch: ${{ matrix.arch }}
+          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
+          cargo-profile: ${{ env.CARGO_PROFILE }}
+          features: ${{ matrix.features }}
+          version: ${{ needs.allocate-runners.outputs.version }}
+          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
+          artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
+          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
 
-      - name: Run tests
-        if: env.DISABLE_RUN_TESTS == 'false'
-        run: make test sqlness-test
-
-      - name: Run cargo build
-        if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false
-        run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
-
-      - name: Calculate checksum and rename binary
-        shell: bash
-        run: |
-          cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}
-          chmod +x greptime
-          tar -zcvf ${{ matrix.file }}.tgz greptime
-          echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.file }}
-          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz
-
-      - name: Upload checksum of artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.file }}.sha256sum
-          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum
-
-  build-linux:
-    name: Build linux binary
-    strategy:
-      matrix:
-        # The file format is greptime-<os>-<arch>
-        include:
-          - arch: x86_64-unknown-linux-gnu
-            os: ubuntu-2004-16-cores
-            file: greptime-linux-amd64
-            continue-on-error: false
-            opts: "-F servers/dashboard"
-          - arch: aarch64-unknown-linux-gnu
-            os: ubuntu-2004-16-cores
-            file: greptime-linux-arm64
-            continue-on-error: false
-            opts: "-F servers/dashboard"
-          - arch: x86_64-unknown-linux-gnu
-            os: ubuntu-2004-16-cores
-            file: greptime-linux-amd64-pyo3
-            continue-on-error: false
-            opts: "-F pyo3_backend,servers/dashboard"
-          - arch: aarch64-unknown-linux-gnu
-            os: ubuntu-2004-16-cores
-            file: greptime-linux-arm64-pyo3
-            continue-on-error: false
-            opts: "-F pyo3_backend,servers/dashboard"
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.continue-on-error }}
-    if: github.repository == 'GreptimeTeam/greptimedb'
+  release-images-to-dockerhub:
+    name: Build and push images to DockerHub
+    if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-2004-16-cores
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Cache cargo assets
-        id: cache
-        uses: actions/cache@v3
+      - uses: actions/checkout@v3
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}
+          fetch-depth: 0
 
-      - name: Install Protoc for linux
-        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
-        run: | # Make sure the protoc is >= 3.15
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip
-          unzip protoc-21.9-linux-x86_64.zip -d protoc
-          sudo cp protoc/bin/protoc /usr/local/bin/
-          sudo cp -r protoc/include/google /usr/local/include/
-
-      - name: Install etcd for linux
-        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
-        run: |
-          ETCD_VER=v3.5.7
-          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
-          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-          mkdir -p /tmp/etcd-download
-          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
-          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
-
-          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
-          nohup etcd >/tmp/etcd.log 2>&1 &
-
-      - name: Install dependencies for linux
-        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
-        run: |
-          sudo apt-get -y update
-          sudo apt-get -y install libssl-dev pkg-config g++-aarch64-linux-gnu gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu wget
-
-      # FIXME(zyy17): Should we specify the version of python when building binary for darwin?
-      - name: Compile Python 3.10.10 from source for linux
-        if: contains(matrix.arch, 'linux') && contains(matrix.opts, 'pyo3_backend')
-        run: |
-          sudo chmod +x ./docker/aarch64/compile-python.sh
-          sudo ./docker/aarch64/compile-python.sh ${{ matrix.arch }}
-
-      - name: Install rust toolchain
-        uses: dtolnay/rust-toolchain@master
+      - name: Build and push images to dockerhub
+        uses: ./.github/actions/build-images
         with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-          targets: ${{ matrix.arch }}
-      - name: Install latest nextest release
-        uses: taiki-e/install-action@nextest
-      - name: Output package versions
-        run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version
+          image-registry: docker.io
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+          version: ${{ needs.allocate-runners.outputs.version }}
 
-      - name: Run tests
-        if: env.DISABLE_RUN_TESTS == 'false'
-        run: make test sqlness-test
-
-      - name: Run cargo build
-        if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false
-        run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
-
-      - name: Run cargo build with pyo3 for aarch64-linux
-        if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
-        run: |
-          # TODO(zyy17): We should make PYO3_CROSS_LIB_DIR configurable.
-          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
-          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
-          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
-          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
-
-          export PYO3_CROSS_LIB_DIR=${PWD}/python-3.10.10/aarch64
-          echo "PYO3_CROSS_LIB_DIR: $PYO3_CROSS_LIB_DIR"
-          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
-          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip
-
-          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
-
-      - name: Run cargo build with pyo3 for amd64-linux
-        if: contains(matrix.arch, 'x86_64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
-        run: |
-          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
-          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
-          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
-          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
-
-          echo "implementation=CPython" >> pyo3.config
-          echo "version=3.10" >> pyo3.config
-          echo "implementation=CPython" >> pyo3.config
-          echo "shared=true" >> pyo3.config
-          echo "abi3=true" >> pyo3.config
-          echo "lib_name=python3.10" >> pyo3.config
-          echo "lib_dir=$PYTHON_INSTALL_PATH_AMD64/lib" >> pyo3.config
-          echo "executable=$PYTHON_INSTALL_PATH_AMD64/bin/python3" >> pyo3.config
-          echo "pointer_width=64" >> pyo3.config
-          echo "build_flags=" >> pyo3.config
-          echo "suppress_build_script_link_lines=false" >> pyo3.config
-
-          cat pyo3.config
-          export PYO3_CONFIG_FILE=${PWD}/pyo3.config
-          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
-          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip
-
-          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}
-
-      - name: Calculate checksum and rename binary
-        shell: bash
-        run: |
-          cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}
-          chmod +x greptime
-          tar -zcvf ${{ matrix.file }}.tgz greptime
-          echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.file }}
-          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz
-
-      - name: Upload checksum of artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.file }}.sha256sum
-          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum
-
-  docker:
-    name: Build docker image
-    needs: [build-linux, build-macos]
-    runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Login to Dockerhub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
-        shell: bash
-        if: github.event_name == 'schedule'
-        run: |
-          buildTime=`date "+%Y%m%d"`
-          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
-          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV
-
-      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
-        shell: bash
-        if: github.event_name != 'schedule'
-        run: |
-          VERSION=${{ github.ref_name }}
-          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Download amd64 binary
-        uses: actions/download-artifact@v3
-        with:
-          name: greptime-linux-amd64-pyo3
-          path: amd64
-
-      - name: Unzip the amd64 artifacts
-        run: |
-          tar xvf amd64/greptime-linux-amd64-pyo3.tgz -C amd64/ && rm amd64/greptime-linux-amd64-pyo3.tgz
-          cp -r amd64 docker/ci
-
-      - name: Download arm64 binary
-        id: download-arm64
-        uses: actions/download-artifact@v3
-        with:
-          name: greptime-linux-arm64-pyo3
-          path: arm64
-
-      - name: Unzip the arm64 artifacts
-        id: unzip-arm64
-        if: success() || steps.download-arm64.conclusion == 'success'
-        run: |
-          tar xvf arm64/greptime-linux-arm64-pyo3.tgz -C arm64/ && rm arm64/greptime-linux-arm64-pyo3.tgz
-          cp -r arm64 docker/ci
-
-      - name: Build and push all
-        uses: docker/build-push-action@v3
-        if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds
-        with:
-          context: ./docker/ci/
-          file: ./docker/ci/Dockerfile
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            greptime/greptimedb:latest
-            greptime/greptimedb:${{ env.IMAGE_TAG }}
-
-      - name: Build and push amd64 only
-        uses: docker/build-push-action@v3
-        if: success() || steps.download-arm64.conclusion == 'failure' # Only build and push amd64 platform if download-arm64 fails
-        with:
-          context: ./docker/ci/
-          file: ./docker/ci/Dockerfile
-          push: true
-          platforms: linux/amd64
-          tags: |
-            greptime/greptimedb:latest
-            greptime/greptimedb:${{ env.IMAGE_TAG }}
-
-  release:
-    name: Release artifacts
-    # Release artifacts only when all the artifacts are built successfully.
-    needs: [build-linux, build-macos, docker]
-    runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Configure scheduled build version # the version would be ${SCHEDULED_BUILD_VERSION_PREFIX}-${SCHEDULED_PERIOD}-YYYYMMDD, like v0.2.0-nigthly-20230313.
-        shell: bash
-        if: github.event_name == 'schedule'
-        run: |
-          buildTime=`date "+%Y%m%d"`
-          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-${{ env.SCHEDULED_PERIOD }}-$buildTime
-          echo "SCHEDULED_BUILD_VERSION=${SCHEDULED_BUILD_VERSION}" >> $GITHUB_ENV
-
-      # Only publish release when the release tag is like v1.0.0, v1.0.1, v1.0.2, etc.
-      - name: Set whether it is the latest release
-        run: |
-          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "prerelease=false" >> $GITHUB_ENV
-            echo "makeLatest=true" >> $GITHUB_ENV
-          else
-            echo "prerelease=true" >> $GITHUB_ENV
-            echo "makeLatest=false" >> $GITHUB_ENV
-          fi
-
-      - name: Create scheduled build git tag
-        if: github.event_name == 'schedule'
-        run: |
-          git tag ${{ env.SCHEDULED_BUILD_VERSION }}
-
-      - name: Publish scheduled release # configure the different release title and tags.
-        uses: ncipollo/release-action@v1
-        if: github.event_name == 'schedule'
-        with:
-          name: "Release ${{ env.SCHEDULED_BUILD_VERSION }}"
-          prerelease: ${{ env.prerelease }}
-          makeLatest: ${{ env.makeLatest }}
-          tag: ${{ env.SCHEDULED_BUILD_VERSION }}
-          generateReleaseNotes: true
-          artifacts: |
-            **/greptime-*
-
-      - name: Publish release
-        uses: ncipollo/release-action@v1
-        if: github.event_name != 'schedule'
-        with:
-          name: "${{ github.ref_name }}"
-          prerelease: ${{ env.prerelease }}
-          makeLatest: ${{ env.makeLatest }}
-          generateReleaseNotes: true
-          artifacts: |
-            **/greptime-*
-
-  docker-push-acr:
-    name: Push docker image to alibaba cloud container registry
-    needs: [docker]
-    runs-on: ubuntu-latest
-    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
+  release-images-to-acr:
+    name: Build and push images to ACR
+    if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+    ]
+    runs-on: ubuntu-2004-16-cores
+    # When we push to ACR, it's easy to fail due to some unknown network issues.
+    # However, we don't want to fail the whole workflow because of this.
+    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
     continue-on-error: true
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to alibaba cloud container registry
-        uses: docker/login-action@v2
+      - uses: actions/checkout@v3
         with:
-          registry: registry.cn-hangzhou.aliyuncs.com
-          username: ${{ secrets.ALICLOUD_USERNAME }}
-          password: ${{ secrets.ALICLOUD_PASSWORD }}
+          fetch-depth: 0
 
-      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
-        shell: bash
-        if: github.event_name == 'schedule'
-        run: |
-          buildTime=`date "+%Y%m%d"`
-          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
-          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV
+      - name: Build and push images to ACR
+        uses: ./.github/actions/build-images
+        with:
+          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
+          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+          version: ${{ needs.allocate-runners.outputs.version }}
 
-      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
-        shell: bash
-        if: github.event_name != 'schedule'
-        run: |
-          VERSION=${{ github.ref_name }}
-          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV
+  release-artifacts:
+    name: Create GitHub release and upload artifacts
+    if: ${{ inputs.release_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+      build-linux-arm64-artifacts,
+      build-macos-artifacts,
+      release-images-to-dockerhub,
+    ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
-      - name: Push image to alibaba cloud container registry # Use 'docker buildx imagetools create' to create a new image base on source image.
-        run: |
-          docker buildx imagetools create \
-            --tag registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:latest \
-            --tag registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:${{ env.IMAGE_TAG }} \
-            greptime/greptimedb:${{ env.IMAGE_TAG }}
+      - name: Release artifacts
+        uses: ./.github/actions/release-artifacts
+        with:
+          version: ${{ needs.allocate-runners.outputs.version }}
+
+  release-dev-builder-image:
+    name: Release dev builder image
+    if: ${{ inputs.release_dev_builder_image }} # Only manually trigger this job.
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build and push dev builder image
+        uses: ./.github/actions/build-dev-builder-image
+        with:
+          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          acr-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          acr-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
+          acr-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
+
+  ### Stop runners ###
+  # It's very necessary to split the job of releasing runners into 'stop-linux-amd64-runner' and 'stop-linux-arm64-runner'.
+  # Because we can terminate the specified EC2 instance immediately after the job is finished without uncessary waiting.
+  stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-amd64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-amd64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+  stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
+    name: Stop linux-arm64 runner
+    # Only run this job when the runner is allocated.
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runners,
+      build-linux-arm64-artifacts,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Stop EC2 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

.gitignore

@@ -44,3 +44,5 @@ benchmarks/data
 
 # Vscode workspace
 *.code-workspace
+
+venv/

Cargo.toml

@@ -2,6 +2,7 @@
 members = [
     "benchmarks",
     "src/api",
+    "src/auth",
     "src/catalog",
     "src/client",
     "src/cmd",
@@ -11,6 +12,7 @@ members = [
     "src/common/error",
     "src/common/function",
     "src/common/function-macro",
+    "src/common/greptimedb-telemetry",
     "src/common/grpc",
     "src/common/grpc-expr",
     "src/common/mem-prof",
@@ -24,6 +26,7 @@ members = [
     "src/common/telemetry",
     "src/common/test-util",
     "src/common/time",
+    "src/common/version",
     "src/datanode",
     "src/datatypes",
     "src/file-table-engine",
@@ -32,6 +35,7 @@ members = [
     "src/meta-client",
     "src/meta-srv",
     "src/mito",
+    "src/mito2",
     "src/object-store",
     "src/partition",
     "src/promql",
@@ -42,56 +46,117 @@ members = [
     "src/sql",
     "src/storage",
     "src/store-api",
+    "src/flow",
     "src/table",
     "src/table-procedure",
     "tests-integration",
     "tests/runner",
 ]
+resolver = "2"
 
 [workspace.package]
-version = "0.2.0"
+version = "0.4.0-nightly"
 edition = "2021"
 license = "Apache-2.0"
 
 [workspace.dependencies]
-arrow = { version = "40.0" }
-arrow-array = "40.0"
-arrow-flight = "40.0"
-arrow-schema = { version = "40.0", features = ["serde"] }
+arrow = { version = "43.0" }
+etcd-client = "0.11"
+arrow-array = "43.0"
+arrow-flight = "43.0"
+arrow-schema = { version = "43.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
-# TODO(ruihang): use arrow-datafusion when it contains https://github.com/apache/arrow-datafusion/pull/6032
-datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
-datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "63e52dde9e44cac4b1f6c6e6b6bf6368ba3bd323" }
+datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "c0b0fca548e99d020c76e1a1cd7132aab26000e1" }
+derive_builder = "0.12"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "4398d20c56d5f7939cc2960789cb1fa7dd18e6fe" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "4a277f27caa035a801d5b9c020a0449777736614" }
+humantime-serde = "1.1"
 itertools = "0.10"
-parquet = "40.0"
+lazy_static = "1.4"
+once_cell = "1.18"
+opentelemetry-proto = { version = "0.2", features = ["gen-tonic", "metrics"] }
+parquet = "43.0"
 paste = "1.0"
 prost = "0.11"
 rand = "0.8"
+regex = "1.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 snafu = { version = "0.7", features = ["backtraces"] }
-sqlparser = "0.34"
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4", features = [
+    "visitor",
+] }
+strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
 tokio = { version = "1.28", features = ["full"] }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
+toml = "0.7"
 tonic = { version = "0.9", features = ["tls"] }
 uuid = { version = "1", features = ["serde", "v4", "fast-rng"] }
 metrics = "0.20"
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "f0798c4c648d89f51abe63e870919c75dd463199" }
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "abbd357c1e193cd270ea65ee7652334a150b628f" }
+## workspaces members
+api = { path = "src/api" }
+auth = { path = "src/auth" }
+catalog = { path = "src/catalog" }
+client = { path = "src/client" }
+cmd = { path = "src/cmd" }
+common-base = { path = "src/common/base" }
+common-catalog = { path = "src/common/catalog" }
+common-datasource = { path = "src/common/datasource" }
+common-error = { path = "src/common/error" }
+common-function = { path = "src/common/function" }
+common-function-macro = { path = "src/common/function-macro" }
+common-greptimedb-telemetry = { path = "src/common/greptimedb-telemetry" }
+common-grpc = { path = "src/common/grpc" }
+common-grpc-expr = { path = "src/common/grpc-expr" }
+common-mem-prof = { path = "src/common/mem-prof" }
+common-meta = { path = "src/common/meta" }
+common-procedure = { path = "src/common/procedure" }
+common-procedure-test = { path = "src/common/procedure-test" }
+common-pprof = { path = "src/common/pprof" }
+common-query = { path = "src/common/query" }
+common-recordbatch = { path = "src/common/recordbatch" }
+common-runtime = { path = "src/common/runtime" }
+substrait = { path = "src/common/substrait" }
+common-telemetry = { path = "src/common/telemetry" }
+common-test-util = { path = "src/common/test-util" }
+common-time = { path = "src/common/time" }
+common-version = { path = "src/common/version" }
+datanode = { path = "src/datanode" }
+datatypes = { path = "src/datatypes" }
+file-table-engine = { path = "src/file-table-engine" }
+frontend = { path = "src/frontend" }
+log-store = { path = "src/log-store" }
+meta-client = { path = "src/meta-client" }
+meta-srv = { path = "src/meta-srv" }
+mito = { path = "src/mito" }
+mito2 = { path = "src/mito2" }
+object-store = { path = "src/object-store" }
+partition = { path = "src/partition" }
+promql = { path = "src/promql" }
+query = { path = "src/query" }
+script = { path = "src/script" }
+servers = { path = "src/servers" }
+session = { path = "src/session" }
+sql = { path = "src/sql" }
+storage = { path = "src/storage" }
+store-api = { path = "src/store-api" }
+table = { path = "src/table" }
+table-procedure = { path = "src/table-procedure" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
-rev = "f0798c4c648d89f51abe63e870919c75dd463199"
+rev = "abbd357c1e193cd270ea65ee7652334a150b628f"
 
 [profile.release]
 debug = true

Cross.toml

@@ -0,0 +1,7 @@
+[build]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt update && apt install -y unzip zlib1g-dev zlib1g-dev:$CROSS_DEB_ARCH",
+    "curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip && unzip protoc-3.15.8-linux-x86_64.zip -d /usr/",
+    "chmod a+x /usr/bin/protoc && chmod -R a+rx /usr/include/google",
+]

Makefile

@@ -1,15 +1,77 @@
-IMAGE_REGISTRY ?= greptimedb
+# The arguments for building images.
+CARGO_PROFILE ?=
+FEATURES ?=
+TARGET_DIR ?=
+TARGET ?=
+CARGO_BUILD_OPTS := --locked
+IMAGE_REGISTRY ?= docker.io
+IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
+BUILDX_MULTI_PLATFORM_BUILD ?= false
+BUILDX_BUILDER_NAME ?= gtbuilder
+BASE_IMAGE ?= ubuntu
+RUST_TOOLCHAIN ?= $(shell cat rust-toolchain.toml | grep channel | cut -d'"' -f2)
+CARGO_REGISTRY_CACHE ?= ${HOME}/.cargo/registry
+ARCH := $(shell uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
+OUTPUT_DIR := $(shell if [ "$(RELEASE)" = "true" ]; then echo "release"; elif [ ! -z "$(CARGO_PROFILE)" ]; then echo "$(CARGO_PROFILE)" ; else echo "debug"; fi)
+
+# The arguments for running integration tests.
+ETCD_VERSION ?= v3.5.9
+ETCD_IMAGE ?= quay.io/coreos/etcd:${ETCD_VERSION}
+RETRY_COUNT ?= 3
+NEXTEST_OPTS := --retries ${RETRY_COUNT}
+BUILD_JOBS ?= $(shell which nproc 1>/dev/null && expr $$(nproc) / 2) # If nproc is not available, we don't set the build jobs.
+ifeq ($(BUILD_JOBS), 0) # If the number of cores is less than 2, set the build jobs to 1.
+  BUILD_JOBS := 1
+endif
+
+ifneq ($(strip $(BUILD_JOBS)),)
+	NEXTEST_OPTS += --build-jobs=${BUILD_JOBS}
+endif
+
+ifneq ($(strip $(CARGO_PROFILE)),)
+	CARGO_BUILD_OPTS += --profile ${CARGO_PROFILE}
+endif
+
+ifneq ($(strip $(FEATURES)),)
+	CARGO_BUILD_OPTS += --features ${FEATURES}
+endif
+
+ifneq ($(strip $(TARGET_DIR)),)
+	CARGO_BUILD_OPTS += --target-dir ${TARGET_DIR}
+endif
+
+ifneq ($(strip $(TARGET)),)
+	CARGO_BUILD_OPTS += --target ${TARGET}
+endif
+
+ifneq ($(strip $(RELEASE)),)
+	CARGO_BUILD_OPTS += --release
+endif
+
+ifeq ($(BUILDX_MULTI_PLATFORM_BUILD), true)
+	BUILDX_MULTI_PLATFORM_BUILD_OPTS := --platform linux/amd64,linux/arm64 --push
+else
+	BUILDX_MULTI_PLATFORM_BUILD_OPTS := -o type=docker
+endif
 
 ##@ Build
 
 .PHONY: build
 build: ## Build debug version greptime.
-	cargo build
+	cargo build ${CARGO_BUILD_OPTS}
 
-.PHONY: release
-release:  ## Build release version greptime.
-	cargo build --release
+.POHNY: build-by-dev-builder
+build-by-dev-builder: ## Build greptime by dev-builder.
+	docker run --network=host \
+	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
+	make build \
+	CARGO_PROFILE=${CARGO_PROFILE} \
+	FEATURES=${FEATURES} \
+	TARGET_DIR=${TARGET_DIR} \
+	TARGET=${TARGET} \
+	RELEASE=${RELEASE}
 
 .PHONY: clean
 clean: ## Clean the project.
@@ -21,20 +83,42 @@ fmt: ## Format all the Rust code.
 
 .PHONY: fmt-toml
 fmt-toml: ## Format all TOML files.
-	taplo format --option "indent_string=    "
+	taplo format
 
 .PHONY: check-toml
 check-toml: ## Check all TOML files.
-	taplo format --check --option "indent_string=    "
+	taplo format --check
 
 .PHONY: docker-image
-docker-image: ## Build docker image.
-	docker build --network host -f docker/Dockerfile -t ${IMAGE_REGISTRY}:${IMAGE_TAG} .
+docker-image: build-by-dev-builder ## Build docker image.
+	mkdir -p ${ARCH} && \
+	cp ./target/${OUTPUT_DIR}/greptime ${ARCH}/greptime && \
+	docker build -f docker/ci/${BASE_IMAGE}/Dockerfile -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb:${IMAGE_TAG} . && \
+	rm -r ${ARCH}
+
+.PHONY: docker-image-buildx
+docker-image-buildx: multi-platform-buildx ## Build docker image by buildx.
+	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
+	  --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" \
+	  --build-arg="FEATURES=${FEATURES}" \
+	  --build-arg="OUTPUT_DIR=${OUTPUT_DIR}" \
+	  -f docker/buildx/${BASE_IMAGE}/Dockerfile \
+	  -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
+
+.PHONY: dev-builder
+dev-builder: multi-platform-buildx ## Build dev-builder image.
+	docker buildx build --builder ${BUILDX_BUILDER_NAME} \
+	--build-arg="RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" \
+	-f docker/dev-builder/${BASE_IMAGE}/Dockerfile \
+	-t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} .
+
+.PHONY: multi-platform-buildx
+multi-platform-buildx: ## Create buildx multi-platform builder.
+	docker buildx inspect ${BUILDX_BUILDER_NAME} || docker buildx create --name ${BUILDX_BUILDER_NAME} --driver docker-container --bootstrap --use
 
 ##@ Test
-
 test: nextest ## Run unit and integration tests.
-	cargo nextest run
+	cargo nextest run ${NEXTEST_OPTS}
 
 .PHONY: nextest ## Install nextest tools.
 nextest:
@@ -50,12 +134,27 @@ check: ## Cargo check all the targets.
 
 .PHONY: clippy
 clippy: ## Check clippy rules.
-	cargo clippy --workspace --all-targets -- -D warnings
+	cargo clippy --workspace --all-targets -F pyo3_backend -- -D warnings
 
 .PHONY: fmt-check
 fmt-check: ## Check code format.
 	cargo fmt --all -- --check
 
+.PHONY: start-etcd
+start-etcd: ## Start single node etcd for testing purpose.
+	docker run --rm -d --network=host -p 2379-2380:2379-2380 ${ETCD_IMAGE}
+
+.PHONY: stop-etcd
+stop-etcd: ## Stop single node etcd for testing purpose.
+	docker stop $$(docker ps -q --filter ancestor=${ETCD_IMAGE})
+
+.PHONY: run-it-in-container
+run-it-in-container: start-etcd ## Run integration tests in dev-builder.
+	docker run --network=host \
+	-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v /tmp:/tmp \
+	-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:latest \
+	make test sqlness-test BUILD_JOBS=${BUILD_JOBS}
+
 ##@ General
 
 # The help target prints out all targets with their descriptions organized
@@ -71,4 +170,4 @@ fmt-check: ## Check code format.
 
 .PHONY: help
 help: ## Display help messages.
-	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-30s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

README.md

@@ -47,14 +47,10 @@ for years. Based on their best-practices, GreptimeDB is born to give you:
 
 ## Quick Start
 
-### GreptimePlay
+### [GreptimePlay](https://greptime.com/playground)
 
 Try out the features of GreptimeDB right from your browser.
 
-<a href="https://greptime.com/playground" target="_blank"><img
-src="https://www.greptime.com/assets/greptime_play_button_colorful.1bbe2746.png"
-alt="GreptimePlay" width="200px" /></a>
-
 ### Build
 
 #### Build from Source
@@ -106,7 +102,7 @@ Please see [the online document site](https://docs.greptime.com/getting-started/
 
 Read the [complete getting started guide](https://docs.greptime.com/getting-started/overview#connect) on our [official document site](https://docs.greptime.com/).
 
-To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients).
+To write and query data, GreptimeDB is compatible with multiple [protocols and clients](https://docs.greptime.com/user-guide/clients/overview).
 
 ## Resources
 
@@ -123,7 +119,7 @@ To write and query data, GreptimeDB is compatible with multiple [protocols and c
 
 ### Documentation
 
-- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts.html)
+- GreptimeDB [User Guide](https://docs.greptime.com/user-guide/concepts/overview)
 - GreptimeDB [Developer
   Guide](https://docs.greptime.com/developer-guide/overview.html)
 - GreptimeDB [internal code document](https://greptimedb.rs)
@@ -133,8 +129,12 @@ To write and query data, GreptimeDB is compatible with multiple [protocols and c
 
 ### SDK
 
-- [GreptimeDB Java
-  Client](https://github.com/GreptimeTeam/greptimedb-client-java)
+- [GreptimeDB C++ Client](https://github.com/GreptimeTeam/greptimedb-client-cpp)
+- [GreptimeDB Erlang Client](https://github.com/GreptimeTeam/greptimedb-client-erl)
+- [GreptimeDB Go Client](https://github.com/GreptimeTeam/greptimedb-client-go)
+- [GreptimeDB Java Client](https://github.com/GreptimeTeam/greptimedb-client-java)
+- [GreptimeDB Python Client](https://github.com/GreptimeTeam/greptimedb-client-py) (WIP)
+- [GreptimeDB Rust Client](https://github.com/GreptimeTeam/greptimedb-client-rust)
 
 ## Project Status
 

benchmarks/Cargo.toml

@@ -7,7 +7,7 @@ license.workspace = true
 [dependencies]
 arrow.workspace = true
 clap = { version = "4.0", features = ["derive"] }
-client = { path = "../src/client" }
+client = { workspace = true }
 indicatif = "0.17.1"
 itertools.workspace = true
 parquet.workspace = true

benchmarks/src/bin/nyc-taxi.rs

@@ -114,7 +114,7 @@ async fn write_data(
         };
 
         let now = Instant::now();
-        db.insert(requests).await.unwrap();
+        let _ = db.insert(requests).await.unwrap();
         let elapsed = now.elapsed();
         total_rpc_elapsed_ms += elapsed.as_millis();
         progress_bar.inc(row_count as _);
@@ -377,19 +377,16 @@ fn create_table_expr() -> CreateTableExpr {
 }
 
 fn query_set() -> HashMap<String, String> {
-    let mut ret = HashMap::new();
-
-    ret.insert(
-        "count_all".to_string(),
-        format!("SELECT COUNT(*) FROM {TABLE_NAME};"),
-    );
-
-    ret.insert(
-        "fare_amt_by_passenger".to_string(),
-        format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {TABLE_NAME} GROUP BY passenger_count")
-    );
-
-    ret
+    HashMap::from([
+        (
+            "count_all".to_string(), 
+            format!("SELECT COUNT(*) FROM {TABLE_NAME};"),
+        ),
+        (
+            "fare_amt_by_passenger".to_string(),
+            format!("SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), SUM(fare_amount) FROM {TABLE_NAME} GROUP BY passenger_count"),
+        )
+    ])
 }
 
 async fn do_write(args: &Args, db: &Database) {
@@ -414,7 +411,8 @@ async fn do_write(args: &Args, db: &Database) {
             let db = db.clone();
             let mpb = multi_progress_bar.clone();
             let pb_style = progress_bar_style.clone();
-            write_jobs.spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+            let _ = write_jobs
+                .spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
         }
     }
     while write_jobs.join_next().await.is_some() {
@@ -423,7 +421,8 @@ async fn do_write(args: &Args, db: &Database) {
             let db = db.clone();
             let mpb = multi_progress_bar.clone();
             let pb_style = progress_bar_style.clone();
-            write_jobs.spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
+            let _ = write_jobs
+                .spawn(async move { write_data(batch_size, &db, path, mpb, pb_style).await });
         }
     }
 }

config/datanode.example.toml

@@ -11,6 +11,10 @@ rpc_hostname = "127.0.0.1"
 # The number of gRPC server worker threads, 8 by default.
 rpc_runtime_size = 8
 
+[heartbeat]
+# Interval for sending heartbeat messages to the Metasrv in milliseconds, 5000 by default.
+interval_millis = 5000
+
 # Metasrv client options.
 [meta_client_options]
 # Metasrv address list.
@@ -26,16 +30,17 @@ tcp_nodelay = true
 [wal]
 # WAL data directory
 # dir = "/tmp/greptimedb/wal"
-file_size = "1GB"
-purge_threshold = "50GB"
+file_size = "256MB"
+purge_threshold = "4GB"
 purge_interval = "10m"
 read_batch_size = 128
 sync_write = false
 
 # Storage options, see `standalone.example.toml`.
 [storage]
-type = "File"
+# The working home directory.
 data_home = "/tmp/greptimedb/"
+type = "File"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
 
@@ -51,9 +56,7 @@ max_purge_tasks = 32
 # Create a checkpoint every <checkpoint_margin> actions.
 checkpoint_margin = 10
 # Region manifest logs and checkpoints gc execution duration
-gc_duration = '30s'
-# Whether to try creating a manifest checkpoint on region opening
-checkpoint_on_startup = false
+gc_duration = '10m'
 
 # Storage flush options
 [storage.flush]

config/frontend.example.toml

@@ -1,10 +1,17 @@
 # Node running mode, see `standalone.example.toml`.
 mode = "distributed"
 
+[heartbeat]
+# Interval for sending heartbeat task to the Metasrv in milliseconds, 5000 by default.
+interval_millis = 5000
+# Interval for retry sending heartbeat task in milliseconds, 5000 by default.
+retry_interval_millis = 5000
+
 # HTTP server options, see `standalone.example.toml`.
 [http_options]
 addr = "127.0.0.1:4000"
 timeout = "30s"
+body_limit = "64MB"
 
 # gRPC server options, see `standalone.example.toml`.
 [grpc_options]
@@ -42,18 +49,16 @@ runtime_size = 2
 [influxdb_options]
 enable = true
 
-# Prometheus protocol options, see `standalone.example.toml`.
-[prometheus_options]
+# Prometheus remote storage options, see `standalone.example.toml`.
+[prom_store_options]
 enable = true
 
-# Prometheus protocol options, see `standalone.example.toml`.
-[prom_options]
-addr = "127.0.0.1:4004"
-
 # Metasrv client options, see `datanode.example.toml`.
 [meta_client_options]
 metasrv_addrs = ["127.0.0.1:3002"]
 timeout_millis = 3000
+# DDL timeouts options.
+ddl_timeout_millis = 10000
 connect_timeout_millis = 5000
 tcp_nodelay = true
 
@@ -61,3 +66,11 @@ tcp_nodelay = true
 # [logging]
 # dir = "/tmp/greptimedb/logs"
 # level = "info"
+
+# Datanode options.
+[datanode]
+# Datanode client options.
+[datanode.client]
+timeout = "10s"
+connect_timeout = "10s"
+tcp_nodelay = true

config/metasrv.example.toml

@@ -1,3 +1,5 @@
+# The working home directory.
+data_home = "/tmp/metasrv/"
 # The bind address of metasrv, "127.0.0.1:3002" by default.
 bind_addr = "127.0.0.1:3002"
 # The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost.
@@ -13,8 +15,25 @@ datanode_lease_secs = 15
 selector = "LeaseBased"
 # Store data in memory, false by default.
 use_memory_store = false
+# Whether to enable greptimedb telemetry, true by default.
+enable_telemetry = true
 
 # Log options, see `standalone.example.toml`
 # [logging]
 # dir = "/tmp/greptimedb/logs"
 # level = "info"
+
+# Procedure storage options.
+[procedure]
+# Procedure max retry time.
+max_retry_times = 12
+# Initial retry delay of procedures, increases exponentially
+retry_delay = "500ms"
+
+# # Datanode options.
+# [datanode]
+# # Datanode client options.
+# [datanode.client_options]
+# timeout_millis = 10000
+# connect_timeout_millis = 10000
+# tcp_nodelay = true

config/standalone.example.toml

@@ -2,6 +2,8 @@
 mode = "standalone"
 # Whether to use in-memory catalog, `false` by default.
 enable_memory_catalog = false
+# Whether to enable greptimedb telemetry, true by default.
+enable_telemetry = true
 
 # HTTP server options.
 [http_options]
@@ -9,6 +11,9 @@ enable_memory_catalog = false
 addr = "127.0.0.1:4000"
 # HTTP request timeout, 30s by default.
 timeout = "30s"
+# HTTP request body limit, 64Mb by default.
+# the following units are supported: B, KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, PiB
+body_limit = "64MB"
 
 # gRPC server options.
 [grpc_options]
@@ -66,24 +71,19 @@ runtime_size = 2
 # Whether to enable InfluxDB protocol in HTTP API, true by default.
 enable = true
 
-# Prometheus protocol options.
-[prometheus_options]
+# Prometheus remote storage options
+[prom_store_options]
 # Whether to enable Prometheus remote write and read in HTTP API, true by default.
 enable = true
 
-# Prom protocol options.
-[prom_options]
-# Prometheus API server address, "127.0.0.1:4004" by default.
-addr = "127.0.0.1:4004"
-
 # WAL options.
 [wal]
 # WAL data directory
 # dir = "/tmp/greptimedb/wal"
 # WAL file size in bytes.
-file_size = "1GB"
-# WAL purge threshold in bytes.
-purge_threshold = "50GB"
+file_size = "256MB"
+# WAL purge threshold.
+purge_threshold = "4GB"
 # WAL purge interval in seconds.
 purge_interval = "10m"
 # WAL read batch size.
@@ -93,10 +93,10 @@ sync_write = false
 
 # Storage options.
 [storage]
+# The working home directory.
+data_home = "/tmp/greptimedb/"
 # Storage type.
 type = "File"
-# Data directory, "/tmp/greptimedb/data" by default.
-data_home = "/tmp/greptimedb/"
 # TTL for all tables. Disabled by default.
 # global_ttl = "7d"
 
@@ -115,9 +115,7 @@ max_purge_tasks = 32
 # Create a checkpoint every <checkpoint_margin> actions.
 checkpoint_margin = 10
 # Region manifest logs and checkpoints gc execution duration
-gc_duration = '30s'
-# Whether to try creating a manifest checkpoint on region opening
-checkpoint_on_startup = false
+gc_duration = '10m'
 
 # Storage flush options
 [storage.flush]

docker/Dockerfile

@@ -1,38 +0,0 @@
-FROM ubuntu:22.04 as builder
-
-ENV LANG en_US.utf8
-WORKDIR /greptimedb
-
-# Install dependencies.
-RUN apt-get update && apt-get install -y \
-    libssl-dev \
-    protobuf-compiler \
-    curl \
-    build-essential \
-    pkg-config \
-    python3 \
-    python3-dev \
-    python3-pip \
-    && pip3 install --upgrade pip \
-    && pip3 install pyarrow
-
-# Install Rust.
-SHELL ["/bin/bash", "-c"]
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
-ENV PATH /root/.cargo/bin/:$PATH
-
-# Build the project in release mode.
-COPY . .
-RUN cargo build --release
-
-# Export the binary to the clean image.
-# TODO(zyy17): Maybe should use the more secure container image.
-FROM ubuntu:22.04 as base
-
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates
-
-WORKDIR /greptime
-COPY --from=builder /greptimedb/target/release/greptime /greptime/bin/
-ENV PATH /greptime/bin/:$PATH
-
-ENTRYPOINT ["greptime"]

docker/aarch64/Dockerfile

@@ -1,57 +0,0 @@
-FROM ubuntu:22.04 as builder
-
-ENV LANG en_US.utf8
-WORKDIR /greptimedb
-
-# Install dependencies.
-RUN apt-get update && apt-get install -y \
-    libssl-dev \
-    protobuf-compiler \
-    curl \
-    build-essential \
-    pkg-config \
-    wget
-
-# Install Rust.
-SHELL ["/bin/bash", "-c"]
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
-ENV PATH /root/.cargo/bin/:$PATH
-
-# Install cross platform toolchain
-RUN apt-get -y update && \
-    apt-get -y install g++-aarch64-linux-gnu gcc-aarch64-linux-gnu && \
-    apt-get install binutils-aarch64-linux-gnu
-
-COPY ./docker/aarch64/compile-python.sh ./docker/aarch64/
-RUN chmod +x ./docker/aarch64/compile-python.sh && \
-    ./docker/aarch64/compile-python.sh
-
-COPY ./rust-toolchain.toml .
-# Install rustup target for cross compiling.
-RUN rustup target add aarch64-unknown-linux-gnu
-COPY . .
-# Update dependency, using separate `RUN` to separate cache
-RUN cargo fetch
-
-# This three env var is set in script, so I set it manually in dockerfile.
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
-ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib/
-ENV PY_INSTALL_PATH=/greptimedb/python_arm64_build
-
-# Set the environment variable for cross compiling and compile it
-# cross compiled python is `python3` in path, but pyo3 need `python` in path so alias it
-# Build the project in release mode.
-RUN export PYO3_CROSS_LIB_DIR=$PY_INSTALL_PATH/lib && \ 
-    alias python=python3 && \
-    cargo build --target aarch64-unknown-linux-gnu --release -F pyo3_backend
-
-# Exporting the binary to the clean image
-FROM ubuntu:22.04 as base
-
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates
-
-WORKDIR /greptime
-COPY --from=builder /greptimedb/target/aarch64-unknown-linux-gnu/release/greptime /greptime/bin/
-ENV PATH /greptime/bin/:$PATH
-
-ENTRYPOINT ["greptime"]

docker/aarch64/compile-python.sh

@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-# this script will download Python source code, compile it, and install it to /usr/local/lib
-# then use this python to compile cross-compiled python for aarch64
-ARCH=$1
-PYTHON_VERSION=3.10.10
-PYTHON_SOURCE_DIR=Python-${PYTHON_VERSION}
-PYTHON_INSTALL_PATH_AMD64=${PWD}/python-${PYTHON_VERSION}/amd64
-PYTHON_INSTALL_PATH_AARCH64=${PWD}/python-${PYTHON_VERSION}/aarch64
-
-function download_python_source_code() {
-  wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz
-  tar -xvf Python-$PYTHON_VERSION.tgz
-}
-
-function compile_for_amd64_platform() {
-  mkdir -p "$PYTHON_INSTALL_PATH_AMD64"
-
-  echo "Compiling for amd64 platform..."
-
-  ./configure \
-    --prefix="$PYTHON_INSTALL_PATH_AMD64" \
-    --enable-shared \
-    ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
-    ac_cv_have_long_long_format=yes \
-    --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no
-
-  make
-  make install
-}
-
-# explain Python compile options here a bit:s
-# --enable-shared: enable building a shared Python library (default is no) but we do need it for calling from rust
-# CC, CXX, AR, LD, RANLIB: set the compiler, archiver, linker, and ranlib programs to use
-# build: the machine you are building on, host: the machine you will run the compiled program on
-# --with-system-ffi: build _ctypes module using an installed ffi library, see Doc/library/ctypes.rst, not used in here TODO: could remove
-# ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes:
-# allow cross-compiled python to have -pthread set for CXX, see https://github.com/python/cpython/pull/22525
-# ac_cv_have_long_long_format=yes: target platform supports long long type
-# disable-ipv6: disable ipv6 support, we don't need it in here
-# ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no: disable pty support, we don't need it in here
-function compile_for_aarch64_platform() {
-  export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
-  export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
-  export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH
-
-  mkdir -p "$PYTHON_INSTALL_PATH_AARCH64"
-
-  echo "Compiling for aarch64 platform..."
-  echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
-  echo "LIBRARY_PATH: $LIBRARY_PATH"
-  echo "PATH: $PATH"
-
-  ./configure --build=x86_64-linux-gnu --host=aarch64-linux-gnu \
-    --prefix="$PYTHON_INSTALL_PATH_AARCH64" --enable-optimizations \
-    CC=aarch64-linux-gnu-gcc \
-    CXX=aarch64-linux-gnu-g++ \
-    AR=aarch64-linux-gnu-ar \
-    LD=aarch64-linux-gnu-ld \
-    RANLIB=aarch64-linux-gnu-ranlib \
-    --enable-shared \
-    ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \
-    ac_cv_have_long_long_format=yes \
-    --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no
-
-  make
-  make altinstall
-}
-
-# Main script starts here.
-download_python_source_code
-
-# Enter the python source code directory.
-cd $PYTHON_SOURCE_DIR || exit 1
-
-# Build local python first, then build cross-compiled python.
-compile_for_amd64_platform
-
-# Clean the build directory.
-make clean && make distclean
-
-# Cross compile python for aarch64.
-if [ "$ARCH" = "aarch64-unknown-linux-gnu" ]; then
-  compile_for_aarch64_platform
-fi

docker/buildx/centos/Dockerfile

@@ -0,0 +1,54 @@
+FROM centos:7 as builder
+
+ARG CARGO_PROFILE
+ARG FEATURES
+ARG OUTPUT_DIR
+
+ENV LANG en_US.utf8
+WORKDIR /greptimedb
+
+# Install dependencies
+RUN ulimit -n 1024000 && yum groupinstall -y 'Development Tools'
+RUN yum install -y epel-release  \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel \
+    which
+
+# Install protoc
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
+RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
+
+# Install Rust
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH
+
+# Build the project in release mode.
+RUN --mount=target=.,rw \
+    --mount=type=cache,target=/root/.cargo/registry \
+    make build \
+    CARGO_PROFILE=${CARGO_PROFILE} \
+    FEATURES=${FEATURES} \
+    TARGET_DIR=/out/target
+
+# Export the binary to the clean image.
+FROM centos:7 as base
+
+ARG OUTPUT_DIR
+
+RUN yum install -y epel-release \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel \
+    which
+
+WORKDIR /greptime
+COPY --from=builder /out/target/${OUTPUT_DIR}/greptime /greptime/bin/
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/buildx/ubuntu/Dockerfile

@@ -0,0 +1,57 @@
+FROM ubuntu:22.04 as builder
+
+ARG CARGO_PROFILE
+ARG FEATURES
+ARG OUTPUT_DIR
+
+ENV LANG en_US.utf8
+WORKDIR /greptimedb
+
+# Install dependencies.
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get update && apt-get install -y \
+    libssl-dev \
+    protobuf-compiler \
+    curl \
+    git \
+    build-essential \
+    pkg-config \
+    python3.10 \
+    python3.10-dev \
+    python3-pip
+
+# Install Rust.
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+
+# Build the project in release mode.
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cargo/registry \
+    make build \
+    CARGO_PROFILE=${CARGO_PROFILE} \
+    FEATURES=${FEATURES} \
+    TARGET_DIR=/out/target
+
+# Export the binary to the clean image.
+# TODO(zyy17): Maybe should use the more secure container image.
+FROM ubuntu:22.04 as base
+
+ARG OUTPUT_DIR
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get \
+    -y install ca-certificates \
+    python3.10 \
+    python3.10-dev \
+    python3-pip \
+    curl
+
+COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt
+
+RUN python3 -m pip install -r /etc/greptime/requirements.txt
+
+WORKDIR /greptime
+COPY --from=builder /out/target/${OUTPUT_DIR}/greptime /greptime/bin/
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/ci/centos/Dockerfile

@@ -0,0 +1,16 @@
+FROM centos:7
+
+RUN yum install -y epel-release \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel
+
+ARG TARGETARCH
+
+ADD $TARGETARCH/greptime /greptime/bin/
+
+ENV PATH /greptime/bin/:$PATH
+
+ENTRYPOINT ["greptime"]

docker/ci/ubuntu/Dockerfile

@@ -4,9 +4,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     ca-certificates \
     python3.10 \
     python3.10-dev \
-    python3-pip
+    python3-pip \
+    curl
 
-COPY requirements.txt /etc/greptime/requirements.txt
+COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt
 
 RUN python3 -m pip install -r /etc/greptime/requirements.txt
 

docker/dev-builder/centos/Dockerfile

@@ -0,0 +1,29 @@
+FROM centos:7 as builder
+
+ENV LANG en_US.utf8
+
+# Install dependencies
+RUN ulimit -n 1024000 && yum groupinstall -y 'Development Tools'
+RUN yum install -y epel-release  \
+    openssl \
+    openssl-devel  \
+    centos-release-scl  \
+    rh-python38  \
+    rh-python38-python-devel \
+    which
+
+# Install protoc
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
+RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
+
+# Install Rust
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH
+
+# Install Rust toolchains.
+ARG RUST_TOOLCHAIN
+RUN rustup toolchain install ${RUST_TOOLCHAIN}
+
+# Install nextest.
+RUN cargo install cargo-nextest --locked

docker/dev-builder/ubuntu/Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:22.04
+
+ENV LANG en_US.utf8
+WORKDIR /greptimedb
+
+# Install dependencies.
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    libssl-dev \
+    tzdata \
+    protobuf-compiler \
+    curl \
+    ca-certificates \
+    git \
+    build-essential \
+    pkg-config \
+    python3.10 \
+    python3.10-dev \
+    python3-pip
+
+RUN git config --global --add safe.directory /greptimedb
+
+# Install Python dependencies.
+COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt
+RUN python3 -m pip install -r /etc/greptime/requirements.txt
+
+# Install Rust.
+SHELL ["/bin/bash", "-c"]
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+
+# Install Rust toolchains.
+ARG RUST_TOOLCHAIN
+RUN rustup toolchain install ${RUST_TOOLCHAIN}
+
+# Install nextest.
+RUN cargo install cargo-nextest --locked

docs/benchmarks/tsbs/v0.3.2.md

@@ -0,0 +1,39 @@
+# TSBS benchmark - v0.3.2
+
+## Environment
+
+|     |     |
+| --- | --- |
+| CPU | AMD Ryzen 7 7735HS (8 core 3.2GHz) |
+| Memory | 32GB |
+| Disk | SOLIDIGM SSDPFKNU010TZ |
+| OS | Ubuntu 22.04.2 LTS |
+
+
+## Write performance
+
+| Write buffer size | Ingest rate（rows/s） |
+| --- | --- |
+| 512M | 139583.04 |
+| 32M | 279250.52 |
+
+
+## Query performance
+
+| Query type  | v0.3.2 write buffer 32M (ms) | v0.3.2 write buffer 512M (ms) | v0.3.1 write buffer 32M (ms) |
+| --- | --- | --- | --- |
+| cpu-max-all-1 | 921.12 | 241.23 | 553.63 |
+| cpu-max-all-8 | 2657.66 | 502.78 | 3308.41 |
+| double-groupby-1 | 28238.85 | 27367.42 | 52148.22 |
+| double-groupby-5 | 33094.65 | 32421.89 | 56762.37 |
+| double-groupby-all | 38565.89 | 38635.52 | 59596.80 |
+| groupby-orderby-limit | 23321.60 | 22423.55 | 53983.23 |
+| high-cpu-1 | 1167.04 | 254.15 | 832.41 |
+| high-cpu-all | 32814.08 | 29906.94 | 62853.12 |
+| lastpoint | 192045.05 | 153575.42 | NA   |
+| single-groupby-1-1-1 | 63.97 | 87.35 | 92.66 |
+| single-groupby-1-1-12 | 666.24 | 326.98 | 781.50 |
+| single-groupby-1-8-1 | 225.29 | 137.97 |281.95 |
+| single-groupby-5-1-1 | 70.40 | 81.64 | 86.15 |
+| single-groupby-5-1-12 | 722.75 | 356.01 | 805.18 |
+| single-groupby-5-8-1 | 285.60 | 115.88 | 326.29 |

docs/rfcs/2023-07-06-table-engine-refactor.md

@@ -0,0 +1,303 @@
+---
+Feature Name: table-engine-refactor
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/1869
+Date: 2023-07-06
+Author: "Yingwen <realevenyag@gmail.com>"
+---
+
+Refactor Table Engine
+----------------------
+
+# Summary
+Refactor table engines to address several historical tech debts.
+
+# Motivation
+Both `Frontend` and `Datanode` have to deal with multiple regions in a table. This results in code duplication and additional burden to the `Datanode`.
+
+Before:
+
+
+```mermaid
+graph TB
+
+subgraph Frontend["Frontend"]
+    subgraph MyTable
+        A("region 0, 2 -> Datanode0")
+        B("region 1, 3 -> Datanode1")
+    end
+end
+
+MyTable --> MetaSrv
+MetaSrv --> ETCD
+
+MyTable-->TableEngine0
+MyTable-->TableEngine1
+
+subgraph Datanode0
+    Procedure0("procedure")
+    TableEngine0("table engine")
+    region0
+    region2
+    mytable0("my_table")
+
+    Procedure0-->mytable0
+    TableEngine0-->mytable0
+    mytable0-->region0
+    mytable0-->region2
+end
+
+
+subgraph Datanode1
+    Procedure1("procedure")
+    TableEngine1("table engine")
+    region1
+    region3
+    mytable1("my_table")
+
+    Procedure1-->mytable1
+    TableEngine1-->mytable1
+    mytable1-->region1
+    mytable1-->region3
+end
+
+
+subgraph manifest["table manifest"]
+    M0("my_table")
+    M1("regions: [0, 1, 2, 3]")
+end
+
+mytable1-->manifest
+mytable0-->manifest
+
+RegionManifest0("region manifest 0")
+RegionManifest1("region manifest 1")
+RegionManifest2("region manifest 2")
+RegionManifest3("region manifest 3")
+region0-->RegionManifest0
+region1-->RegionManifest1
+region2-->RegionManifest2
+region3-->RegionManifest3
+```
+
+`Datanodes` can update the same manifest file for a table as regions are assigned to different nodes in the cluster. We also have to run procedures on `Datanode` to ensure the table manifest is consistent with region manifests. "Table" in a `Datanode` is a subset of the table's regions. The `Datanode` is much closer to `RegionServer` in `HBase` which only deals with regions.
+
+In cluster mode, we store table metadata in etcd and table manifest. The table manifest becomes redundant. We can remove the table manifest if we refactor the table engines to region engines that only care about regions. What's more, we don't need to run those procedures on `Datanode`.
+
+After:
+```mermaid
+graph TB
+
+subgraph Frontend["Frontend"]
+    direction LR
+    subgraph MyTable
+        A("region 0, 2 -> Datanode0")
+        B("region 1, 3 -> Datanode1")
+    end
+end
+
+MyTable --> MetaSrv
+MetaSrv --> ETCD
+
+MyTable-->RegionEngine
+MyTable-->RegionEngine1
+
+subgraph Datanode0
+    RegionEngine("region engine")
+    region0
+    region2
+    RegionEngine-->region0
+    RegionEngine-->region2
+end
+
+
+subgraph Datanode1
+    RegionEngine1("region engine")
+    region1
+    region3
+    RegionEngine1-->region1
+    RegionEngine1-->region3
+end
+
+RegionManifest0("region manifest 0")
+RegionManifest1("region manifest 1")
+RegionManifest2("region manifest 2")
+RegionManifest3("region manifest 3")
+region0-->RegionManifest0
+region1-->RegionManifest1
+region2-->RegionManifest2
+region3-->RegionManifest3
+```
+This RFC proposes to refactor table engines into region engines as a first step to make the `Datanode` acts like a `RegionServer`.
+
+
+# Details
+## Overview
+
+We plan to refactor the `TableEngine` trait into `RegionEngine` gradually. This RFC focuses on the `mito` engine as it is the default table engine and the most complicated engine.
+
+Currently, we built `MitoEngine` upon `StorageEngine` that manages regions of the `mito` engine. Since `MitoEngine` becomes a region engine, we could combine `StorageEngine` with `MitoEngine` to simplify our code structure.
+
+The chart below shows the overall architecture of the `MitoEngine`.
+
+```mermaid
+classDiagram
+class MitoEngine~LogStore~ {
+    -WorkerGroup workers
+}
+class MitoRegion {
+    +VersionControlRef version_control
+    -RegionId region_id
+    -String manifest_dir
+    -AtomicI64 last_flush_millis
+    +region_id() RegionId
+    +scan() ChunkReaderImpl
+}
+class RegionMap {
+    -HashMap&lt;RegionId, MitoRegionRef&gt; regions
+}
+class ChunkReaderImpl
+
+class WorkerGroup {
+    -Vec~RegionWorker~ workers
+}
+class RegionWorker {
+    -RegionMap regions
+    -Sender sender
+    -JoinHandle handle
+}
+class RegionWorkerThread~LogStore~ {
+    -RegionMap regions
+    -Receiver receiver
+    -Wal~LogStore~ wal
+    -ObjectStore object_store
+    -MemtableBuilderRef memtable_builder
+    -FlushSchedulerRef~LogStore~ flush_scheduler
+    -FlushStrategy flush_strategy
+    -CompactionSchedulerRef~LogStore~ compaction_scheduler
+    -FilePurgerRef file_purger
+}
+class Wal~LogStore~ {
+    -LogStore log_store
+}
+class MitoConfig
+
+MitoEngine~LogStore~ o-- MitoConfig
+MitoEngine~LogStore~ o-- MitoRegion
+MitoEngine~LogStore~ o-- WorkerGroup
+MitoRegion o-- VersionControl
+MitoRegion -- ChunkReaderImpl
+WorkerGroup o-- RegionWorker
+RegionWorker o-- RegionMap
+RegionWorker -- RegionWorkerThread~LogStore~
+RegionWorkerThread~LogStore~ o-- RegionMap
+RegionWorkerThread~LogStore~ o-- Wal~LogStore~
+```
+
+We replace the `RegionWriter` with `RegionWorker` to process write requests and DDL requests.
+
+
+## Metadata
+We also merge region's metadata with table's metadata. It should make metadata much easier to maintain.
+```mermaid
+classDiagram
+class VersionControl {
+    -CowCell~Version~ version
+    -AtomicU64 committed_sequence
+}
+class Version {
+    -RegionMetadataRef metadata
+    -MemtableVersionRef memtables
+    -LevelMetasRef ssts
+    -SequenceNumber flushed_sequence
+    -ManifestVersion manifest_version
+}
+class MemtableVersion {
+    -MemtableRef mutable
+    -Vec~MemtableRef~ immutables
+    +mutable_memtable() MemtableRef
+    +immutable_memtables() &[MemtableRef]
+    +freeze_mutable(MemtableRef new_mutable) MemtableVersion
+}
+class LevelMetas {
+    -LevelMetaVec levels
+    -AccessLayerRef sst_layer
+    -FilePurgerRef file_purger
+    -Option~i64~ compaction_time_window
+}
+class LevelMeta {
+    -Level level
+    -HashMap&lt;FileId, FileHandle&gt; files
+}
+class FileHandle {
+    -FileMeta meta
+    -bool compacting
+    -AtomicBool deleted
+    -AccessLayerRef sst_layer
+    -FilePurgerRef file_purger
+}
+class FileMeta {
+    +RegionId region_id
+    +FileId file_id
+    +Option&lt;Timestamp, Timestamp&gt; time_range
+    +Level level
+    +u64 file_size
+}
+
+VersionControl o-- Version
+Version o-- RegionMetadata
+Version o-- MemtableVersion
+Version o-- LevelMetas
+LevelMetas o-- LevelMeta
+LevelMeta o-- FileHandle
+FileHandle o-- FileMeta
+
+class RegionMetadata {
+    +RegionId region_id
+    +VersionNumber version
+    +SchemaRef table_schema
+    +Vec~usize~ primary_key_indices
+    +Vec~usize~ value_indices
+    +ColumnId next_column_id
+    +TableOptions region_options
+    +DateTime~Utc~ created_on
+    +RegionSchemaRef region_schema
+}
+class RegionSchema {
+    -SchemaRef user_schema
+    -StoreSchemaRef store_schema
+    -ColumnsMetadataRef columns
+}
+class Schema
+class StoreSchema {
+    -Vec~ColumnMetadata~ columns
+    -SchemaRef schema
+    -usize row_key_end
+    -usize user_column_end
+}
+class ColumnsMetadata {
+    -Vec~ColumnMetadata~ columns
+    -HashMap&lt;String, usize&gt; name_to_col_index
+    -usize row_key_end
+    -usize timestamp_key_index
+    -usize user_column_end
+}
+class ColumnMetadata
+
+RegionMetadata o-- RegionSchema
+RegionMetadata o-- Schema
+RegionSchema o-- StoreSchema
+RegionSchema o-- Schema
+RegionSchema o-- ColumnsMetadata
+StoreSchema o-- ColumnsMetadata
+StoreSchema o-- Schema
+StoreSchema o-- ColumnMetadata
+ColumnsMetadata o-- ColumnMetadata
+```
+
+# Drawback
+This is a breaking change.
+
+# Future Work
+- Rename `TableEngine` to `RegionEngine`
+- Simplify schema relationship in the `mito` engine
+- Refactor the `Datanode` into a `RegionServer`.

docs/rfcs/2023-07-10-metric-engine.md

@@ -0,0 +1,202 @@
+---
+Feature Name: metric-engine
+Tracking Issue: TBD
+Date: 2023-07-10
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+# Summary
+
+A new metric engine that can significantly enhance our ability to handle the tremendous number of small tables in scenarios like Prometheus metrics, by leveraging a synthetic wide table that offers storage and metadata multiplexing capabilities over the existing engine.
+
+# Motivation
+
+The concept "Table" in GreptimeDB is a bit "heavy" compared to other time-series storage like Prometheus or VictoriaMetrics. This has lots of disadvantages in aspects from performance, footprint, and storage to cost.
+
+# Details
+
+## Top level description
+
+- User Interface
+
+    This feature will add a new type of storage engine. It might be available to be an option like `with ENGINE=mito` or an internal interface like auto create table on Prometheus remote write. From the user side, there is no difference from tables in mito engine. All the DDL like `CREATE`, `ALTER` and DML like `SELECT` should be supported.
+
+- Implementation Overlook
+
+    This new engine doesn't re-implement low level components like file R/W etc. It's a wrapper layer over the existing mito engine, with extra storage and metadata multiplexing capabilities. I.e., it expose multiple table based on one mito engine table like this:
+	``` plaintext
+	   ┌───────────────┐ ┌───────────────┐ ┌───────────────┐
+	   │ Metric Engine │ │ Metric Engine │ │ Metric Engine │
+	   │   Table 1     │ │   Table 2     │ │   Table 3     │
+	   └───────────────┘ └───────────────┘ └───────────────┘
+	           ▲               ▲                   ▲
+	           │               │                   │
+	           └───────────────┼───────────────────┘
+	                           │
+	                 ┌─────────┴────────┐
+	                 │ Metric Region    │
+	                 │   Engine         │
+	                 │    ┌─────────────┤
+	                 │    │ Mito Region │
+	                 │    │   Engine    │
+	                 └────▲─────────────┘
+	                      │
+	                      │
+	                ┌─────┴───────────────┐
+	                │                     │
+	                │  Mito Engine Table  │
+	                │                     │
+	                └─────────────────────┘
+	```
+
+The following parts will describe these implementation details:
+    - How to route these metric region tables and how those table are distributed
+    - How to maintain the schema and other metadata of the underlying mito engine table
+    - How to maintain the schema of metric engine table
+    - How the query goes
+
+## Routing
+
+Before this change, the region route rule was based on a group of partition keys. Relation of physical table to region is one-to-many.
+
+``` rust
+  pub struct PartitionDef {
+      partition_columns: Vec<String>,
+      partition_bounds: Vec<PartitionBound>,
+  }
+```
+
+And for metric engine tables, the key difference is we split the concept of "physical table" and "logical table". Like the previous ASCII chart, multiple logical tables are based on one physical table. The relationship of logical table to region becomes many-to-many. Thus, we must include the table name (of logical table) into partition rules.
+
+Consider the partition/route interface is a generic map of string array to region id, all we need to do is to insert logical table name into the request:
+
+``` rust
+  fn route(request: Vec<String>) -> RegionId;
+```
+
+The next question is, where to do this conversion? The basic idea is to dispatch different routing behavior based on the engine type. Since we have all the necessary information in frontend, it's a good place to do that. And can leave meta server untouched. The essential change is to associate engine type with route rule.
+
+## Physical Region Schema
+
+The idea "physical wide table" is to perform column-level multiplexing. I.e., map all logical columns to physical columns by their names.
+
+```
+   ┌────────────┐      ┌────────────┐         ┌────────────┐
+   │   Table 1  │      │   Table 2  │         │   Table 3  │
+   ├───┬────┬───┤      ├───┬────┬───┤         ├───┬────┬───┤
+   │C1 │ C2 │ C3│      │C1 │ C3 │ C5├──────┐  │C2 │ C4 │ C6│
+   └─┬─┴──┬─┴─┬─┘ ┌────┴───┴──┬─┴───┘      │  └─┬─┴──┬─┴─┬─┘
+     │    │   │   │           │            │    │    │   │
+     │    │   │   │           └──────────┐ │    │    │   │
+     │    │   │   │                      │ │    │    │   │
+     │    │   │   │  ┌─────────────────┐ │ │    │    │   │
+     │    │   │   │  │ Physical Table  │ │ │    │    │   │
+     │    │   │   │  ├──┬──┬──┬──┬──┬──┘ │ │    │    │   │
+     └────x───x───┴─►│C1│C2│C3│C4│C5│C6◄─┼─x────x────x───┘
+          │   │      └──┘▲─┘▲─┴─▲└─▲└──┘ │ │    │    │
+          │   │          │  │   │  │     │ │    │    │
+          ├───x──────────┘  ├───x──x─────┘ │    │    │
+          │   │             │   │  │       │    │    │
+          │   └─────────────┘   │  └───────┘    │    │
+          │                     │               │    │
+          └─────────────────────x───────────────┘    │
+                                │                    │
+                                └────────────────────┘
+```
+
+This approach is very straightforward but has one problem. It only works when two columns have different semantic type (time index, tag or field) or data types but with the same name. E.g., `CREATE TABLE t1 (c1 timestamp(3) TIME INDEX)` and `CREATE TABLE t2 (c1 STRING PRIMARY KEY)`.
+
+One possible workaround is to prefix each column with its data type and semantic type, like `_STRING_PK_c1`. However, considering the primary goal at present is to support data from monitoring metrics like Prometheus remote write, it's acceptable not to support this at first because data types are often simple and limited here.
+
+
+The next point is changing the physical table's schema. This is only needed when creating a new logical table or altering the existing table. Typically speaking, table creating and altering are explicit. We only need to emit an add column request to underlying physical table on processing logical table's DDL. GreptimeDB can create or alter table automatically on some protocols, but the internal logic is the same.
+
+Also for simplicity, we don't support shrinking the underlying table at first. This can be achieved by introducing mechanism on the physical column.
+
+Frontend needs not to keep physical table's schema.
+
+## Metadata of physical regions
+
+Those metric engine regions need to store extra metadata like the schema of logical table or all logical table's name. That information is relatively simple and can be stored in a format like key-value pair. For now, we have to use another physical mito region for metadata. This involves an issue with region scheduling. Since we don't have the ability to perform affinity scheduling, the initial version will just assume the data region and metadata region are in the same instance. See alternatives - other storage for physical region's metadata for possible future improvement.
+
+Here is the schema of metadata region and how we would use it. The `CREATE TABLE` clause of metadata region looks like the following. Notice that it wouldn't be actually created by SQL.
+
+``` sql
+  CREATE TABLE metadata(
+  	ts timestamp time index,
+    	key string primary key,
+    	value string
+  );
+```
+
+The `ts` field is just a placeholder -- for the constraints that a mito region must contain a time index field. It will always be `0`. The other two fields `key` and `value` will be used as a k-v storage. It contains two group of key
+    - `__table_<TABLE_NAME>` is used for marking table existence. It doesn't have value.
+    - `__column_<TABLE_NAME>_<COLUMN_NAME>` is used for marking table existence, the value is column's semantic type.
+
+## Physical region implementation
+
+This RFC proposes to add a new region implementation named "MetricRegion". As showed in the first chart, it's wrapped over the existing mito region. This section will describe the implementation details. Firstly, here is a chart shows how the region hierarchy looks like:
+
+```plaintext
+ ┌───────────────────────┐
+ │ Metric Region         │
+ │                       │
+ │   ┌────────┬──────────┤
+ │   │ Mito   │ Mito     │
+ │   │ Region │ Region   │
+ │   │ for    │ for      │
+ │   │ Data   │ Metadata │
+ └───┴────────┴──────────┘
+```
+
+All upper levels only see the Metric Region. E.g., Meta Server schedules on this region, or Frontend routes requests to this Metrics Region's id. To be scheduled (open or close etc.), Metric Region needs to implement its own procedures. Most of those procedures can be simply assembled from underlying Mito Regions', but those related to data like alter or drop will have its own new logic.
+
+Another point is region id. Since the region id is used widely from meta server to persisted state, it's better to keep it unchanged. This means we can't use the same id for two regions, but one for each. To achieve this, this RFC proposes a concept named "region id group". A region id group is a group of region ids that are bound for different purposes. Like the two underlying regions here. 
+
+This preserves the first 8 bits of the `u32` region number for grouping. Each group has one main id (the first one) and other sub ids (the rest non-zero ids). All components other than the region implementation itself doesn't aware of the existence of region id group. They only see the main id. The region implementation is in response of managing and using the region id group.
+
+```plaintext
+63                                  31         23                  0
+┌────────────────────────────────────┬──────────┬──────────────────┐
+│          Table Id(32)              │ Group(8) │ Region Number(24)│
+└────────────────────────────────────┴──────────┴──────────────────┘
+                                            Region Id(32)
+```
+
+## Routing in meta server
+
+From previous sections, we can conclude the following points about routing:
+- Each "logical table" has its own, universe unique table id.
+- Logical table doesn't have physical region, they share the same physical region with other logical tables.
+- Route rule of logical table's is a strict subset of physical table's.
+
+To associate the logical table with physical region, we need to specify necessary information in the create table request. Specifically, the table type and its parent table. This require to change our gRPC proto's definition. And once meta recognize the table to create is a logical table, it will use the parent table's region to create route entry.
+
+And to reduce the consumption of region failover (which need to update the physical table route info), we'd better to split the current route table structure into two parts:
+
+```rust
+region_route: Map<TableName, [RegionId]>,
+node_route: Map<RegionId, NodeId>,
+```
+
+By doing this on each failover the meta server only needs to update the second `node_route` map and leave the first one untouched.
+
+## Query
+
+Like other existing components, a user query always starts in the frontend. In the planning phase, frontend needs to fetch related schemas of the queried table. This part is the same. I.e., changes in this RFC don't affect components above the `Table` abstraction.
+
+# Alternatives
+
+## Other routing method
+
+We can also do this "special" route rule in the meta server. But there is no difference with the proposed method.
+
+## Other storage for physical region's metadata
+
+Once we have implemented the "region family" that allows multiple physical schemas exist in one region, we can store the metadata and table data into one region.
+
+Before that, we can also let the `MetricRegion` holds a `KvBackend` to access the storage layer directly. But this breaks the abstraction in some way.
+
+# Drawbacks
+
+Since the physical storage is mixed together. It's hard to do fine-grained operations in table level. Like configuring TTL, memtable size or compaction strategy in table level. Or define different partition rules for different tables. For scenarios like this, it's better to move the table out of metrics engine and "upgrade" it to a normal mito engine table. This requires a migration process in a low cost. And we have to ensure data consistency during the migration, which may require a out-of-service period.

docs/rfcs/2023-08-04-tabie-trait-refactor.md

@@ -0,0 +1,175 @@
+---
+Feature Name: table-trait-refactor
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/2065
+Date: 2023-08-04
+Author: "Ruihang Xia <waynestxia@gmail.com>"
+---
+
+Refactor Table Trait
+--------------------
+
+# Summary
+Refactor `Table` trait to adapt the new region server architecture and make code more straightforward.
+
+# Motivation
+The `Table` is designed in the background of both frontend and datanode keeping the same concepts. And all the operations are served by a `Table`. However, in our practice, we found that not all the operations are suitable to be served by a `Table`. For example, the `Table` doesn't hold actual physical data itself, thus operations like write or alter are simply a proxy over underlying regions. And in the recent refactor to datanode ([rfc table-engine-refactor](./2023-07-06-table-engine-refactor.md)), we are changing datanode to region server that is only aware of `Region` things. This also calls for a refactor to the `Table` trait.
+
+# Details
+
+## Definitions
+
+The current `Table` trait contains the following methods:
+```rust
+pub trait Table {
+    /// Get a reference to the schema for this table
+    fn schema(&self) -> SchemaRef;
+
+    /// Get a reference to the table info.
+    fn table_info(&self) -> TableInfoRef;
+
+    /// Get the type of this table for metadata/catalog purposes.
+    fn table_type(&self) -> TableType;
+
+    /// Insert values into table.
+    ///
+    /// Returns number of inserted rows.
+    async fn insert(&self, _request: InsertRequest) -> Result<usize>;
+
+    /// Generate a record batch stream for querying.
+    async fn scan_to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream>;
+
+    /// Tests whether the table provider can make use of any or all filter expressions
+    /// to optimise data retrieval.
+    fn supports_filters_pushdown(&self, filters: &[&Expr]) -> Result<Vec<FilterPushDownType>>;
+
+    /// Alter table.
+    async fn alter(&self, _context: AlterContext, _request: &AlterTableRequest) -> Result<()>;
+
+    /// Delete rows in the table.
+    ///
+    /// Returns number of deleted rows.
+    async fn delete(&self, _request: DeleteRequest) -> Result<usize>;
+
+    /// Flush table.
+    ///
+    /// Options:
+    /// - region_number: specify region to flush.
+    /// - wait: Whether to wait until flush is done.
+    async fn flush(&self, region_number: Option<RegionNumber>, wait: Option<bool>) -> Result<()>;
+
+    /// Close the table.
+    async fn close(&self, _regions: &[RegionNumber]) -> Result<()>;
+
+    /// Get region stats in this table.
+    fn region_stats(&self) -> Result<Vec<RegionStat>>;
+
+    /// Return true if contains the region
+    fn contains_region(&self, _region: RegionNumber) -> Result<bool>;
+
+    /// Get statistics for this table, if available
+    fn statistics(&self) -> Option<TableStatistics>;
+
+    async fn compact(&self, region_number: Option<RegionNumber>, wait: Option<bool>) -> Result<()>;
+}
+```
+
+We can divide those methods into three categories from the perspective of functionality:
+
+|     Retrieve Metadata      | Manipulate Data |    Read Data     |
+| :------------------------: | :-------------: | :--------------: |
+|          `schema`          |    `insert`     | `scan_to_stream` |
+|        `table_info`        |     `alter`     |                  |
+|        `table_type`        |    `delete`     |                  |
+| `supports_filter_pushdown` |     `flush`     |                  |
+|       `region_stats`       |     `close`     |                  |
+|     `contains_region`      |    `compact`    |                  |
+|        `statistics`        |                 |                  |
+
+And considering most of the access to metadata happens in frontend, like route or query; and all the persisted data are stored in regions; while only the query engine needs to read data. We can divide the `Table` trait into three concepts:
+
+- struct `Table` provides metadata:
+    ```rust
+    impl Table {
+        /// Get a reference to the schema for this table
+        fn schema(&self) -> SchemaRef;
+
+        /// Get a reference to the table info.
+        fn table_info(&self) -> TableInfoRef;
+
+        /// Get the type of this table for metadata/catalog purposes.
+        fn table_type(&self) -> TableType;
+
+        /// Get statistics for this table, if available
+        fn statistics(&self) -> Option<TableStatistics>;
+
+        fn to_data_source(&self) -> DataSourceRef;
+    }
+    ```
+- Requests to region server
+  - `InsertRequest`
+  - `AlterRequest`
+  - `DeleteRequest`
+  - `FlushRequest`
+  - `CompactRequest`
+  - `CloseRequest`
+
+- trait `DataSource` provides data (`RecordBatch`)
+    ```rust
+    trait DataSource {
+        fn get_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream>;
+    }
+    ```
+
+## Use `Table`
+
+`Table` will only be used in frontend. It's constructed from the `OpenTableRequest` or `CreateTableRequest`.
+
+`Table` also provides a method `to_data_source` to generate a `DataSource` from itself. But this method is only for non-`TableType::Base` tables (i.e., `TableType::View` and `TableType::Temporary`) because `TableType::Base` table doesn't hold actual data itself. Its `DataSource` should be constructed from the `Region` directly (in other words, it's a remote query).
+
+And it requires some extra information to construct a `DataSource`, named `TableSourceProvider`:
+
+```rust
+type TableFactory = Arc<dyn Fn() -> DataSourceRef>;
+
+pub enum TableSourceProvider {
+    Base,
+    View(LogicalPlan),
+    Temporary(TableFactory),
+}
+```
+
+## Use `DataSource`
+
+`DataSource` will be adapted to the `TableProvider` from DataFusion that can be `scan()`ed in a `TableScan` plan.
+
+In frontend this is done in the planning phase. And datanode will have one implementation for `Region` to generate record batch stream.
+
+## Interact with RegionServer
+
+Previously, persisted state change operations were through the old `Table` trait, like said before. Now they will come from the action source, like the procedure or protocol handler directly to the region server. E.g., on alter table, the corresponding procedure will generate its `AlterRequest` and send it to regions. Or write request will be split in frontend handler, and sent to regions. `Table` only provides necessary metadata like route information if needed, but not the necessary part anymore.
+
+## Implement temporary table
+
+Temporary table is a special table that doesn't revolves to any persistent physical region. Examples are:
+- the `Numbers` table for testing, which produces a record batch that contains 0-100 integers.
+- tables in information schema. It is an interface for querying catalog's metadata. The contents are generated on the fly with information from `CatalogManager`. The `CatalogManager` can be held in `TableFactory`.
+- Function table that produces data generated by a formula or a function. Like something that always `sin(current_timestamp())`.
+
+## Relationship among those components
+
+Here is a diagram to show the relationship among those components, and how they interact with each other.
+
+```mermaid
+erDiagram
+    CatalogManager ||--|{ Table : manages
+    Table ||--|{ DataStream : generates
+    Table ||--|{ Region : routes
+    Region ||--|{ DataStream : implements
+    DataStream }|..|| QueryEngine : adapts-to
+    Procedure ||--|{ Region : requests
+    Protocol ||--|{ Region : writes
+    Protocol ||--|{ QueryEngine : queries
+```
+
+# Drawback
+This is a breaking change.

docs/rfcs/2023-08-13-metadata-txn.md

@@ -0,0 +1,90 @@
+---
+Feature Name: Update Metadata in single transaction
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/1715
+Date: 2023-08-13
+Author: "Feng Yangsen <fengys1996@gmail.com>, Xu Wenkang <wenymedia@gmail.com>"
+---
+
+# Summary
+Update Metadata in single transaction.
+
+# Motivation
+Currently, multiple transactions are involved during the procedure. This implementation is inefficient, and it's hard to make data consistent. Therefore, We can update multiple metadata in a single transaction.
+
+# Details 
+Now we have the following table metadata keys:
+
+**TableInfo** 
+```rust
+// __table_info/{table_id}
+pub struct TableInfoKey {
+    table_id: TableId,
+}
+
+pub struct TableInfoValue {
+    pub table_info: RawTableInfo,
+    version: u64,
+}
+```
+
+**TableRoute** 
+```rust
+// __table_route/{table_id}
+pub struct NextTableRouteKey {
+    table_id: TableId,
+}
+
+pub struct TableRoute {
+    pub region_routes: Vec<RegionRoute>,
+}
+```
+**DatanodeTable**
+```rust
+// __table_route/{datanode_id}/{table_id}
+pub struct DatanodeTableKey {
+    datanode_id: DatanodeId,
+    table_id: TableId,
+}
+
+pub struct DatanodeTableValue {
+    pub table_id: TableId,
+    pub regions: Vec<RegionNumber>,
+    version: u64,
+}
+```
+
+**TableNameKey**
+```rust
+// __table_name/{CatalogName}/{SchemaName}/{TableName}
+pub struct TableNameKey<'a> {
+    pub catalog: &'a str,
+    pub schema: &'a str,
+    pub table: &'a str,
+}
+
+pub struct TableNameValue {
+    table_id: TableId,
+}
+```
+
+These table metadata only updates in the following operations.
+
+## Region Failover
+It needs to update `TableRoute` key and `DatanodeTable` keys. If the `TableRoute` equals the Snapshot of `TableRoute` submitting the Failover task, then we can safely update these keys.
+
+After submitting Failover tasks to acquire locks for execution, the `TableRoute` may be updated by another task. After acquiring the lock, we can get the latest `TableRoute` again and then execute it if needed.
+
+## Create Table DDL
+Creates all of the above keys. `TableRoute`, `TableInfo`, should be empty.
+
+The **TableNameKey**'s lock will be held by the procedure framework.
+## Drop Table DDL
+
+`TableInfoKey` and `NextTableRouteKey` will be added with  `__removed-` prefix, and the other above keys will be deleted.  The transaction will not compare any keys.
+## Alter Table DDL
+
+1. Rename table, updates `TableInfo` and `TableName`. Compares `TableInfo`, and the new `TableNameKey` should be empty, and TableInfo should equal the Snapshot when submitting DDL.
+
+The old and new **TableNameKey**'s lock will be held by the procedure framework.
+
+2. Alter table, updates `TableInfo`. `TableInfo` should equal the Snapshot when submitting DDL.

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-05-03"
+channel = "nightly-2023-08-07"

scripts/fetch-dashboard-assets.sh

@@ -2,14 +2,14 @@
 
 # This script is used to download built dashboard assets from the "GreptimeTeam/dashboard" repository.
 
-set -e
+set -e -x
 
 declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
 declare -r ROOT_DIR=$(dirname ${SCRIPT_DIR})
 declare -r STATIC_DIR="$ROOT_DIR/src/servers/dashboard"
 OUT_DIR="${1:-$SCRIPT_DIR}"
 
-RELEASE_VERSION="$(cat $STATIC_DIR/VERSION)"
+RELEASE_VERSION="$(cat $STATIC_DIR/VERSION | tr -d '\t\r\n ')"
 
 echo "Downloading assets to dir: $OUT_DIR"
 cd $OUT_DIR

scripts/install.sh

@@ -61,7 +61,16 @@ if [ -n "${OS_TYPE}" ] && [ -n "${ARCH_TYPE}" ]; then
     fi
 
     echo "Downloading ${BIN}, OS: ${OS_TYPE}, Arch: ${ARCH_TYPE}, Version: ${VERSION}"
+    PACKAGE_NAME="${BIN}-${OS_TYPE}-${ARCH_TYPE}-${VERSION}.tar.gz"
 
-    wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz"
-    tar xvf ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && rm ${BIN}-${OS_TYPE}-${ARCH_TYPE}.tgz && echo "Run './${BIN} --help' to get started"
+    if [ -n "${PACKAGE_NAME}" ]; then
+      wget "https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${VERSION}/${PACKAGE_NAME}"
+
+      # Extract the binary and clean the rest.
+      tar xvf "${PACKAGE_NAME}" && \
+      mv "${PACKAGE_NAME%.tar.gz}/${BIN}" "${PWD}" && \
+      rm -r "${PACKAGE_NAME}" && \
+      rm -r "${PACKAGE_NAME%.tar.gz}" && \
+      echo "Run './${BIN} --help' to get started"
+    fi
 fi

src/api/Cargo.toml

@@ -5,11 +5,10 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-arrow-flight.workspace = true
-common-base = { path = "../common/base" }
-common-error = { path = "../common/error" }
-common-time = { path = "../common/time" }
-datatypes = { path = "../datatypes" }
+common-base = { workspace = true }
+common-error = { workspace = true }
+common-time = { workspace = true }
+datatypes = { workspace = true }
 greptime-proto.workspace = true
 prost.workspace = true
 snafu = { version = "0.7", features = ["backtraces"] }
@@ -17,3 +16,6 @@ tonic.workspace = true
 
 [build-dependencies]
 tonic-build = "0.9"
+
+[dev-dependencies]
+paste = "1.0"

src/api/src/error.rs

@@ -15,7 +15,7 @@
 use std::any::Any;
 
 use common_error::ext::ErrorExt;
-use common_error::prelude::StatusCode;
+use common_error::status_code::StatusCode;
 use datatypes::prelude::ConcreteDataType;
 use snafu::prelude::*;
 use snafu::Location;
@@ -41,7 +41,7 @@ pub enum Error {
     ))]
     ConvertColumnDefaultConstraint {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -52,7 +52,7 @@ pub enum Error {
     ))]
     InvalidColumnDefaultConstraint {
         column: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 }

src/api/src/lib.rs

@@ -15,7 +15,7 @@
 pub mod error;
 pub mod helper;
 
-pub mod prometheus {
+pub mod prom_store {
     pub mod remote {
         pub use greptime_proto::prometheus::remote::*;
     }

src/auth/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "auth"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+default = []
+testing = []
+
+[dependencies]
+api.workspace = true
+async-trait.workspace = true
+common-error.workspace = true
+digest = "0.10"
+hex = { version = "0.4" }
+secrecy = { version = "0.8", features = ["serde", "alloc"] }
+sha1 = "0.10"
+snafu.workspace = true
+sql.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+common-test-util.workspace = true

src/auth/src/common.rs

@@ -0,0 +1,147 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use digest::Digest;
+use secrecy::SecretString;
+use sha1::Sha1;
+use snafu::{ensure, OptionExt};
+
+use crate::error::{IllegalParamSnafu, InvalidConfigSnafu, Result, UserPasswordMismatchSnafu};
+use crate::user_info::DefaultUserInfo;
+use crate::user_provider::static_user_provider::{StaticUserProvider, STATIC_USER_PROVIDER};
+use crate::{UserInfoRef, UserProviderRef};
+
+pub(crate) const DEFAULT_USERNAME: &str = "greptime";
+
+/// construct a [`UserInfo`] impl with name
+/// use default username `greptime` if None is provided
+pub fn userinfo_by_name(username: Option<String>) -> UserInfoRef {
+    DefaultUserInfo::with_name(username.unwrap_or_else(|| DEFAULT_USERNAME.to_string()))
+}
+
+pub fn user_provider_from_option(opt: &String) -> Result<UserProviderRef> {
+    let (name, content) = opt.split_once(':').context(InvalidConfigSnafu {
+        value: opt.to_string(),
+        msg: "UserProviderOption must be in format `<option>:<value>`",
+    })?;
+    match name {
+        STATIC_USER_PROVIDER => {
+            let provider =
+                StaticUserProvider::try_from(content).map(|p| Arc::new(p) as UserProviderRef)?;
+            Ok(provider)
+        }
+        _ => InvalidConfigSnafu {
+            value: name.to_string(),
+            msg: "Invalid UserProviderOption",
+        }
+        .fail(),
+    }
+}
+
+type Username<'a> = &'a str;
+type HostOrIp<'a> = &'a str;
+
+#[derive(Debug, Clone)]
+pub enum Identity<'a> {
+    UserId(Username<'a>, Option<HostOrIp<'a>>),
+}
+
+pub type HashedPassword<'a> = &'a [u8];
+pub type Salt<'a> = &'a [u8];
+
+/// Authentication information sent by the client.
+pub enum Password<'a> {
+    PlainText(SecretString),
+    MysqlNativePassword(HashedPassword<'a>, Salt<'a>),
+    PgMD5(HashedPassword<'a>, Salt<'a>),
+}
+
+pub fn auth_mysql(
+    auth_data: HashedPassword,
+    salt: Salt,
+    username: &str,
+    save_pwd: &[u8],
+) -> Result<()> {
+    ensure!(
+        auth_data.len() == 20,
+        IllegalParamSnafu {
+            msg: "Illegal mysql password length"
+        }
+    );
+    // ref: https://github.com/mysql/mysql-server/blob/a246bad76b9271cb4333634e954040a970222e0a/sql/auth/password.cc#L62
+    let hash_stage_2 = double_sha1(save_pwd);
+    let tmp = sha1_two(salt, &hash_stage_2);
+    // xor auth_data and tmp
+    let mut xor_result = [0u8; 20];
+    for i in 0..20 {
+        xor_result[i] = auth_data[i] ^ tmp[i];
+    }
+    let candidate_stage_2 = sha1_one(&xor_result);
+    if candidate_stage_2 == hash_stage_2 {
+        Ok(())
+    } else {
+        UserPasswordMismatchSnafu {
+            username: username.to_string(),
+        }
+        .fail()
+    }
+}
+
+fn sha1_two(input_1: &[u8], input_2: &[u8]) -> Vec<u8> {
+    let mut hasher = Sha1::new();
+    hasher.update(input_1);
+    hasher.update(input_2);
+    hasher.finalize().to_vec()
+}
+
+fn sha1_one(data: &[u8]) -> Vec<u8> {
+    let mut hasher = Sha1::new();
+    hasher.update(data);
+    hasher.finalize().to_vec()
+}
+
+fn double_sha1(data: &[u8]) -> Vec<u8> {
+    sha1_one(&sha1_one(data))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_sha() {
+        let sha_1_answer: Vec<u8> = vec![
+            124, 74, 141, 9, 202, 55, 98, 175, 97, 229, 149, 32, 148, 61, 194, 100, 148, 248, 148,
+            27,
+        ];
+        let sha_1 = sha1_one("123456".as_bytes());
+        assert_eq!(sha_1, sha_1_answer);
+
+        let double_sha1_answer: Vec<u8> = vec![
+            107, 180, 131, 126, 183, 67, 41, 16, 94, 228, 86, 141, 218, 125, 198, 126, 210, 202,
+            42, 217,
+        ];
+        let double_sha1 = double_sha1("123456".as_bytes());
+        assert_eq!(double_sha1, double_sha1_answer);
+
+        let sha1_2_answer: Vec<u8> = vec![
+            132, 115, 215, 211, 99, 186, 164, 206, 168, 152, 217, 192, 117, 47, 240, 252, 142, 244,
+            37, 204,
+        ];
+        let sha1_2 = sha1_two("123456".as_bytes(), "654321".as_bytes());
+        assert_eq!(sha1_2, sha1_2_answer);
+    }
+}

src/auth/src/error.rs

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,84 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
-
-use common_error::ext::BoxedError;
-use common_error::prelude::ErrorExt;
+use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
-use secrecy::SecretString;
-use session::context::UserInfo;
-use snafu::{Location, OptionExt, Snafu};
-
-use crate::auth::user_provider::StaticUserProvider;
-
-pub mod user_provider;
-
-#[async_trait::async_trait]
-pub trait UserProvider: Send + Sync {
-    fn name(&self) -> &str;
-
-    /// [`authenticate`] checks whether a user is valid and allowed to access the database.
-    async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfo>;
-
-    /// [`authorize`] checks whether a connection request
-    /// from a certain user to a certain catalog/schema is legal.
-    /// This method should be called after [`authenticate`].
-    async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfo) -> Result<()>;
-
-    /// [`auth`] is a combination of [`authenticate`] and [`authorize`].
-    /// In most cases it's preferred for both convenience and performance.
-    async fn auth(
-        &self,
-        id: Identity<'_>,
-        password: Password<'_>,
-        catalog: &str,
-        schema: &str,
-    ) -> Result<UserInfo> {
-        let user_info = self.authenticate(id, password).await?;
-        self.authorize(catalog, schema, &user_info).await?;
-        Ok(user_info)
-    }
-}
-
-pub type UserProviderRef = Arc<dyn UserProvider>;
-
-type Username<'a> = &'a str;
-type HostOrIp<'a> = &'a str;
-
-#[derive(Debug, Clone)]
-pub enum Identity<'a> {
-    UserId(Username<'a>, Option<HostOrIp<'a>>),
-}
-
-pub type HashedPassword<'a> = &'a [u8];
-pub type Salt<'a> = &'a [u8];
-
-/// Authentication information sent by the client.
-pub enum Password<'a> {
-    PlainText(SecretString),
-    MysqlNativePassword(HashedPassword<'a>, Salt<'a>),
-    PgMD5(HashedPassword<'a>, Salt<'a>),
-}
-
-pub fn user_provider_from_option(opt: &String) -> Result<UserProviderRef> {
-    let (name, content) = opt.split_once(':').context(InvalidConfigSnafu {
-        value: opt.to_string(),
-        msg: "UserProviderOption must be in format `<option>:<value>`",
-    })?;
-    match name {
-        user_provider::STATIC_USER_PROVIDER => {
-            let provider =
-                StaticUserProvider::try_from(content).map(|p| Arc::new(p) as UserProviderRef)?;
-            Ok(provider)
-        }
-        _ => InvalidConfigSnafu {
-            value: name.to_string(),
-            msg: "Invalid UserProviderOption",
-        }
-        .fail(),
-    }
-}
+use snafu::{Location, Snafu};
 
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
@@ -111,7 +36,7 @@ pub enum Error {
 
     #[snafu(display("Auth failed, source: {}", source))]
     AuthBackend {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
@@ -135,6 +60,9 @@ pub enum Error {
         schema: String,
         username: String,
     },
+
+    #[snafu(display("User is not authorized to perform this action"))]
+    PermissionDenied { location: Location },
 }
 
 impl ErrorExt for Error {
@@ -150,6 +78,7 @@ impl ErrorExt for Error {
             Error::UnsupportedPasswordType { .. } => StatusCode::UnsupportedPasswordType,
             Error::UserPasswordMismatch { .. } => StatusCode::UserPasswordMismatch,
             Error::AccessDenied { .. } => StatusCode::AccessDenied,
+            Error::PermissionDenied { .. } => StatusCode::PermissionDenied,
         }
     }
 

src/auth/src/lib.rs

@@ -0,0 +1,34 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod common;
+pub mod error;
+mod permission;
+mod user_info;
+mod user_provider;
+
+#[cfg(feature = "testing")]
+pub mod tests;
+
+pub use common::{
+    auth_mysql, user_provider_from_option, userinfo_by_name, HashedPassword, Identity, Password,
+};
+pub use permission::{PermissionChecker, PermissionReq, PermissionResp};
+pub use user_info::UserInfo;
+pub use user_provider::UserProvider;
+
+/// pub type alias
+pub type UserInfoRef = std::sync::Arc<dyn UserInfo>;
+pub type UserProviderRef = std::sync::Arc<dyn UserProvider>;
+pub type PermissionCheckerRef = std::sync::Arc<dyn PermissionChecker>;

src/auth/src/permission.rs

@@ -0,0 +1,64 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+
+use api::v1::greptime_request::Request;
+use sql::statements::statement::Statement;
+
+use crate::error::{PermissionDeniedSnafu, Result};
+use crate::{PermissionCheckerRef, UserInfoRef};
+
+#[derive(Debug, Clone)]
+pub enum PermissionReq<'a> {
+    GrpcRequest(&'a Request),
+    SqlStatement(&'a Statement),
+    PromQuery,
+    Opentsdb,
+    LineProtocol,
+    PromStoreWrite,
+    PromStoreRead,
+    Otlp,
+}
+
+#[derive(Debug)]
+pub enum PermissionResp {
+    Allow,
+    Reject,
+}
+
+pub trait PermissionChecker: Send + Sync {
+    fn check_permission(
+        &self,
+        user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> Result<PermissionResp>;
+}
+
+impl PermissionChecker for Option<&PermissionCheckerRef> {
+    fn check_permission(
+        &self,
+        user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> Result<PermissionResp> {
+        match self {
+            Some(checker) => match checker.check_permission(user_info, req) {
+                Ok(PermissionResp::Reject) => PermissionDeniedSnafu.fail(),
+                Ok(PermissionResp::Allow) => Ok(PermissionResp::Allow),
+                Err(e) => Err(e),
+            },
+            None => Ok(PermissionResp::Allow),
+        }
+    }
+}

src/auth/src/tests.rs

@@ -11,14 +11,14 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 use secrecy::ExposeSecret;
-use servers::auth::user_provider::auth_mysql;
-use servers::auth::{
-    AccessDeniedSnafu, Identity, Password, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu,
-    UserPasswordMismatchSnafu, UserProvider,
+
+use crate::error::{
+    AccessDeniedSnafu, Result, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu,
+    UserPasswordMismatchSnafu,
 };
-use session::context::UserInfo;
+use crate::user_info::DefaultUserInfo;
+use crate::{auth_mysql, Identity, Password, UserInfoRef, UserProvider};
 
 pub struct DatabaseAuthInfo<'a> {
     pub catalog: &'a str,
@@ -56,17 +56,13 @@ impl UserProvider for MockUserProvider {
         "mock_user_provider"
     }
 
-    async fn authenticate(
-        &self,
-        id: Identity<'_>,
-        password: Password<'_>,
-    ) -> servers::auth::Result<UserInfo> {
+    async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfoRef> {
         match id {
             Identity::UserId(username, _host) => match password {
                 Password::PlainText(password) => {
                     if username == "greptime" {
                         if password.expose_secret() == "greptime" {
-                            Ok(UserInfo::new("greptime"))
+                            Ok(DefaultUserInfo::with_name("greptime"))
                         } else {
                             UserPasswordMismatchSnafu {
                                 username: username.to_string(),
@@ -82,7 +78,7 @@ impl UserProvider for MockUserProvider {
                 }
                 Password::MysqlNativePassword(auth_data, salt) => {
                     auth_mysql(auth_data, salt, username, "greptime".as_bytes())
-                        .map(|_| UserInfo::new(username))
+                        .map(|_| DefaultUserInfo::with_name(username))
                 }
                 _ => UnsupportedPasswordTypeSnafu {
                     password_type: "mysql_native_password",
@@ -92,12 +88,7 @@ impl UserProvider for MockUserProvider {
         }
     }
 
-    async fn authorize(
-        &self,
-        catalog: &str,
-        schema: &str,
-        user_info: &UserInfo,
-    ) -> servers::auth::Result<()> {
+    async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfoRef) -> Result<()> {
         if catalog == self.catalog && schema == self.schema && user_info.username() == self.username
         {
             Ok(())
@@ -114,6 +105,8 @@ impl UserProvider for MockUserProvider {
 
 #[tokio::test]
 async fn test_auth_by_plain_text() {
+    use crate::error;
+
     let user_provider = MockUserProvider::default();
     assert_eq!("mock_user_provider", user_provider.name());
 
@@ -123,9 +116,9 @@ async fn test_auth_by_plain_text() {
             Identity::UserId("greptime", None),
             Password::PlainText("greptime".to_string().into()),
         )
-        .await;
-    assert!(auth_result.is_ok());
-    assert_eq!("greptime", auth_result.unwrap().username());
+        .await
+        .unwrap();
+    assert_eq!("greptime", auth_result.username());
 
     // auth failed, unsupported password type
     let auth_result = user_provider
@@ -137,7 +130,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UnsupportedPasswordType { .. }
+        error::Error::UnsupportedPasswordType { .. }
     ));
 
     // auth failed, err: user not exist.
@@ -150,7 +143,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UserNotFound { .. }
+        error::Error::UserNotFound { .. }
     ));
 
     // auth failed, err: wrong password
@@ -163,7 +156,7 @@ async fn test_auth_by_plain_text() {
     assert!(auth_result.is_err());
     assert!(matches!(
         auth_result.err().unwrap(),
-        servers::auth::Error::UserPasswordMismatch { .. }
+        error::Error::UserPasswordMismatch { .. }
     ))
 }
 
@@ -176,8 +169,8 @@ async fn test_schema_validate() {
         username: "test_user",
     });
 
-    let right_user = UserInfo::new("test_user");
-    let wrong_user = UserInfo::default();
+    let right_user = DefaultUserInfo::with_name("test_user");
+    let wrong_user = DefaultUserInfo::with_name("greptime");
 
     // check catalog
     let re = validator
@@ -193,6 +186,8 @@ async fn test_schema_validate() {
     let re = validator.authorize("greptime", "public", &wrong_user).await;
     assert!(re.is_err());
     // check ok
-    let re = validator.authorize("greptime", "public", &right_user).await;
-    assert!(re.is_ok());
+    validator
+        .authorize("greptime", "public", &right_user)
+        .await
+        .unwrap();
 }

src/auth/src/user_info.rs

@@ -0,0 +1,47 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use crate::UserInfoRef;
+
+pub trait UserInfo: Debug + Sync + Send {
+    fn as_any(&self) -> &dyn Any;
+    fn username(&self) -> &str;
+}
+
+#[derive(Debug)]
+pub(crate) struct DefaultUserInfo {
+    username: String,
+}
+
+impl DefaultUserInfo {
+    pub(crate) fn with_name(username: impl Into<String>) -> UserInfoRef {
+        Arc::new(Self {
+            username: username.into(),
+        })
+    }
+}
+
+impl UserInfo for DefaultUserInfo {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn username(&self) -> &str {
+        self.username.as_str()
+    }
+}

src/auth/src/user_provider.rs

@@ -0,0 +1,46 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub(crate) mod static_user_provider;
+
+use crate::common::{Identity, Password};
+use crate::error::Result;
+use crate::UserInfoRef;
+
+#[async_trait::async_trait]
+pub trait UserProvider: Send + Sync {
+    fn name(&self) -> &str;
+
+    /// [`authenticate`] checks whether a user is valid and allowed to access the database.
+    async fn authenticate(&self, id: Identity<'_>, password: Password<'_>) -> Result<UserInfoRef>;
+
+    /// [`authorize`] checks whether a connection request
+    /// from a certain user to a certain catalog/schema is legal.
+    /// This method should be called after [`authenticate`].
+    async fn authorize(&self, catalog: &str, schema: &str, user_info: &UserInfoRef) -> Result<()>;
+
+    /// [`auth`] is a combination of [`authenticate`] and [`authorize`].
+    /// In most cases it's preferred for both convenience and performance.
+    async fn auth(
+        &self,
+        id: Identity<'_>,
+        password: Password<'_>,
+        catalog: &str,
+        schema: &str,
+    ) -> Result<UserInfoRef> {
+        let user_info = self.authenticate(id, password).await?;
+        self.authorize(catalog, schema, &user_info).await?;
+        Ok(user_info)
+    }
+}

src/auth/src/user_provider/static_user_provider.rs

@@ -19,20 +19,17 @@ use std::io::BufRead;
 use std::path::Path;
 
 use async_trait::async_trait;
-use digest;
-use digest::Digest;
 use secrecy::ExposeSecret;
-use session::context::UserInfo;
-use sha1::Sha1;
 use snafu::{ensure, OptionExt, ResultExt};
 
-use crate::auth::{
-    Error, HashedPassword, Identity, IllegalParamSnafu, InvalidConfigSnafu, IoSnafu, Password,
-    Result, Salt, UnsupportedPasswordTypeSnafu, UserNotFoundSnafu, UserPasswordMismatchSnafu,
-    UserProvider,
+use crate::error::{
+    Error, IllegalParamSnafu, InvalidConfigSnafu, IoSnafu, Result, UnsupportedPasswordTypeSnafu,
+    UserNotFoundSnafu, UserPasswordMismatchSnafu,
 };
+use crate::user_info::DefaultUserInfo;
+use crate::{auth_mysql, Identity, Password, UserInfoRef, UserProvider};
 
-pub const STATIC_USER_PROVIDER: &str = "static_user_provider";
+pub(crate) const STATIC_USER_PROVIDER: &str = "static_user_provider";
 
 impl TryFrom<&str> for StaticUserProvider {
     type Error = Error;
@@ -91,7 +88,7 @@ impl TryFrom<&str> for StaticUserProvider {
     }
 }
 
-pub struct StaticUserProvider {
+pub(crate) struct StaticUserProvider {
     users: HashMap<String, Vec<u8>>,
 }
 
@@ -105,7 +102,7 @@ impl UserProvider for StaticUserProvider {
         &self,
         input_id: Identity<'_>,
         input_pwd: Password<'_>,
-    ) -> Result<UserInfo> {
+    ) -> Result<UserInfoRef> {
         match input_id {
             Identity::UserId(username, _) => {
                 ensure!(
@@ -127,7 +124,7 @@ impl UserProvider for StaticUserProvider {
                             }
                         );
                         return if save_pwd == pwd.expose_secret().as_bytes() {
-                            Ok(UserInfo::new(username))
+                            Ok(DefaultUserInfo::with_name(username))
                         } else {
                             UserPasswordMismatchSnafu {
                                 username: username.to_string(),
@@ -136,14 +133,8 @@ impl UserProvider for StaticUserProvider {
                         };
                     }
                     Password::MysqlNativePassword(auth_data, salt) => {
-                        ensure!(
-                            auth_data.len() == 20,
-                            IllegalParamSnafu {
-                                msg: "Illegal MySQL native password format, length != 20"
-                            }
-                        );
                         auth_mysql(auth_data, salt, username, save_pwd)
-                            .map(|_| UserInfo::new(username))
+                            .map(|_| DefaultUserInfo::with_name(username))
                     }
                     Password::PgMD5(_, _) => UnsupportedPasswordTypeSnafu {
                         password_type: "pg_md5",
@@ -154,88 +145,28 @@ impl UserProvider for StaticUserProvider {
         }
     }
 
-    async fn authorize(&self, _catalog: &str, _schema: &str, _user_info: &UserInfo) -> Result<()> {
+    async fn authorize(
+        &self,
+        _catalog: &str,
+        _schema: &str,
+        _user_info: &UserInfoRef,
+    ) -> Result<()> {
         // default allow all
         Ok(())
     }
 }
 
-pub fn auth_mysql(
-    auth_data: HashedPassword,
-    salt: Salt,
-    username: &str,
-    save_pwd: &[u8],
-) -> Result<()> {
-    // ref: https://github.com/mysql/mysql-server/blob/a246bad76b9271cb4333634e954040a970222e0a/sql/auth/password.cc#L62
-    let hash_stage_2 = double_sha1(save_pwd);
-    let tmp = sha1_two(salt, &hash_stage_2);
-    // xor auth_data and tmp
-    let mut xor_result = [0u8; 20];
-    for i in 0..20 {
-        xor_result[i] = auth_data[i] ^ tmp[i];
-    }
-    let candidate_stage_2 = sha1_one(&xor_result);
-    if candidate_stage_2 == hash_stage_2 {
-        Ok(())
-    } else {
-        UserPasswordMismatchSnafu {
-            username: username.to_string(),
-        }
-        .fail()
-    }
-}
-
-fn sha1_two(input_1: &[u8], input_2: &[u8]) -> Vec<u8> {
-    let mut hasher = Sha1::new();
-    hasher.update(input_1);
-    hasher.update(input_2);
-    hasher.finalize().to_vec()
-}
-
-fn sha1_one(data: &[u8]) -> Vec<u8> {
-    let mut hasher = Sha1::new();
-    hasher.update(data);
-    hasher.finalize().to_vec()
-}
-
-fn double_sha1(data: &[u8]) -> Vec<u8> {
-    sha1_one(&sha1_one(data))
-}
-
 #[cfg(test)]
 pub mod test {
     use std::fs::File;
     use std::io::{LineWriter, Write};
 
     use common_test_util::temp_dir::create_temp_dir;
-    use session::context::UserInfo;
 
-    use crate::auth::user_provider::{double_sha1, sha1_one, sha1_two, StaticUserProvider};
-    use crate::auth::{Identity, Password, UserProvider};
-
-    #[test]
-    fn test_sha() {
-        let sha_1_answer: Vec<u8> = vec![
-            124, 74, 141, 9, 202, 55, 98, 175, 97, 229, 149, 32, 148, 61, 194, 100, 148, 248, 148,
-            27,
-        ];
-        let sha_1 = sha1_one("123456".as_bytes());
-        assert_eq!(sha_1, sha_1_answer);
-
-        let double_sha1_answer: Vec<u8> = vec![
-            107, 180, 131, 126, 183, 67, 41, 16, 94, 228, 86, 141, 218, 125, 198, 126, 210, 202,
-            42, 217,
-        ];
-        let double_sha1 = double_sha1("123456".as_bytes());
-        assert_eq!(double_sha1, double_sha1_answer);
-
-        let sha1_2_answer: Vec<u8> = vec![
-            132, 115, 215, 211, 99, 186, 164, 206, 168, 152, 217, 192, 117, 47, 240, 252, 142, 244,
-            37, 204,
-        ];
-        let sha1_2 = sha1_two("123456".as_bytes(), "654321".as_bytes());
-        assert_eq!(sha1_2, sha1_2_answer);
-    }
+    use crate::user_info::DefaultUserInfo;
+    use crate::user_provider::static_user_provider::StaticUserProvider;
+    use crate::user_provider::{Identity, Password};
+    use crate::UserProvider;
 
     async fn test_authenticate(provider: &dyn UserProvider, username: &str, password: &str) {
         let re = provider
@@ -244,16 +175,17 @@ pub mod test {
                 Password::PlainText(password.to_string().into()),
             )
             .await;
-        assert!(re.is_ok());
+        let _ = re.unwrap();
     }
 
     #[tokio::test]
     async fn test_authorize() {
+        let user_info = DefaultUserInfo::with_name("root");
         let provider = StaticUserProvider::try_from("cmd:root=123456,admin=654321").unwrap();
-        let re = provider
-            .authorize("catalog", "schema", &UserInfo::new("root"))
-            .await;
-        assert!(re.is_ok());
+        provider
+            .authorize("catalog", "schema", &user_info)
+            .await
+            .unwrap();
     }
 
     #[tokio::test]
@@ -270,7 +202,6 @@ pub mod test {
         {
             // write a tmp file
             let file = File::create(&file_path);
-            assert!(file.is_ok());
             let file = file.unwrap();
             let mut lw = LineWriter::new(file);
             assert!(lw
@@ -279,7 +210,7 @@ pub mod test {
 admin=654321",
                 )
                 .is_ok());
-            assert!(lw.flush().is_ok());
+            lw.flush().unwrap();
         }
 
         let param = format!("file:{file_path}");

src/auth/tests/mod.rs

@@ -0,0 +1,61 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(assert_matches)]
+use std::assert_matches::assert_matches;
+use std::sync::Arc;
+
+use api::v1::greptime_request::Request;
+use auth::error::Error::InternalState;
+use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq, PermissionResp, UserInfoRef};
+use sql::statements::show::{ShowDatabases, ShowKind};
+use sql::statements::statement::Statement;
+
+struct DummyPermissionChecker;
+
+impl PermissionChecker for DummyPermissionChecker {
+    fn check_permission(
+        &self,
+        _user_info: Option<UserInfoRef>,
+        req: PermissionReq,
+    ) -> auth::error::Result<PermissionResp> {
+        match req {
+            PermissionReq::GrpcRequest(_) => Ok(PermissionResp::Allow),
+            PermissionReq::SqlStatement(_) => Ok(PermissionResp::Reject),
+            _ => Err(InternalState {
+                msg: "testing".to_string(),
+            }),
+        }
+    }
+}
+
+#[test]
+fn test_permission_checker() {
+    let checker: PermissionCheckerRef = Arc::new(DummyPermissionChecker);
+
+    let grpc_result = checker.check_permission(
+        None,
+        PermissionReq::GrpcRequest(&Request::Query(Default::default())),
+    );
+    assert_matches!(grpc_result, Ok(PermissionResp::Allow));
+
+    let sql_result = checker.check_permission(
+        None,
+        PermissionReq::SqlStatement(&Statement::ShowDatabases(ShowDatabases::new(ShowKind::All))),
+    );
+    assert_matches!(sql_result, Ok(PermissionResp::Reject));
+
+    let err_result = checker.check_permission(None, PermissionReq::Opentsdb);
+    assert_matches!(err_result, Err(InternalState { msg }) if msg == "testing");
+}

src/catalog/Cargo.toml

@@ -8,48 +8,45 @@ license.workspace = true
 testing = []
 
 [dependencies]
-api = { path = "../api" }
+api = { workspace = true }
 arc-swap = "1.0"
 arrow-schema.workspace = true
 async-stream.workspace = true
 async-trait = "0.1"
-backoff = { version = "0.4", features = ["tokio"] }
-common-catalog = { path = "../common/catalog" }
-common-error = { path = "../common/error" }
-common-grpc = { path = "../common/grpc" }
-common-meta = { path = "../common/meta" }
-common-query = { path = "../common/query" }
-common-recordbatch = { path = "../common/recordbatch" }
-common-runtime = { path = "../common/runtime" }
-common-telemetry = { path = "../common/telemetry" }
-common-time = { path = "../common/time" }
+common-catalog = { workspace = true }
+common-error = { workspace = true }
+common-grpc = { workspace = true }
+common-meta = { workspace = true }
+common-query = { workspace = true }
+common-recordbatch = { workspace = true }
+common-runtime = { workspace = true }
+common-telemetry = { workspace = true }
+common-time = { workspace = true }
 dashmap = "5.4"
 datafusion.workspace = true
-datatypes = { path = "../datatypes" }
+datatypes = { workspace = true }
 futures = "0.3"
 futures-util.workspace = true
-key-lock = "0.1"
-lazy_static = "1.4"
-meta-client = { path = "../meta-client" }
+lazy_static.workspace = true
+meta-client = { workspace = true }
 metrics.workspace = true
 moka = { version = "0.11", features = ["future"] }
 parking_lot = "0.12"
-regex = "1.6"
-serde = "1.0"
+regex.workspace = true
+serde.workspace = true
 serde_json = "1.0"
-session = { path = "../session" }
+session = { workspace = true }
 snafu = { version = "0.7", features = ["backtraces"] }
-storage = { path = "../storage" }
-store-api = { path = "../store-api" }
-table = { path = "../table" }
+store-api = { workspace = true }
+table = { workspace = true }
 tokio.workspace = true
 
 [dev-dependencies]
-catalog = { path = ".", features = ["testing"] }
-common-test-util = { path = "../common/test-util" }
+catalog = { workspace = true, features = ["testing"] }
 chrono.workspace = true
-log-store = { path = "../log-store" }
-mito = { path = "../mito", features = ["test"] }
-object-store = { path = "../object-store" }
-storage = { path = "../storage" }
+common-test-util = { workspace = true }
+log-store = { workspace = true }
+mito = { workspace = true, features = ["test"] }
+object-store = { workspace = true }
+storage = { workspace = true }
 tokio.workspace = true

src/catalog/src/error.rs

@@ -16,10 +16,10 @@ use std::any::Any;
 use std::fmt::Debug;
 
 use common_error::ext::{BoxedError, ErrorExt};
-use common_error::prelude::{Snafu, StatusCode};
+use common_error::status_code::StatusCode;
 use datafusion::error::DataFusionError;
 use datatypes::prelude::ConcreteDataType;
-use snafu::Location;
+use snafu::{Location, Snafu};
 use tokio::task::JoinError;
 
 use crate::DeregisterTableRequest;
@@ -27,23 +27,36 @@ use crate::DeregisterTableRequest;
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
+    #[snafu(display("Failed to list catalogs, source: {}", source))]
+    ListCatalogs {
+        location: Location,
+        source: BoxedError,
+    },
+
+    #[snafu(display("Failed to list {}'s schemas, source: {}", catalog, source))]
+    ListSchemas {
+        location: Location,
+        catalog: String,
+        source: BoxedError,
+    },
+
     #[snafu(display(
         "Failed to re-compile script due to internal error, source: {}",
         source
     ))]
     CompileScriptInternal {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
     #[snafu(display("Failed to open system catalog table, source: {}", source))]
     OpenSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("Failed to create system catalog table, source: {}", source))]
     CreateSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -54,7 +67,7 @@ pub enum Error {
     ))]
     CreateTable {
         table_info: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -94,7 +107,7 @@ pub enum Error {
     #[snafu(display("Table engine not found: {}, source: {}", engine_name, source))]
     TableEngineNotFound {
         engine_name: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -132,7 +145,7 @@ pub enum Error {
     #[snafu(display("Failed to open table, table info: {}, source: {}", table_info, source))]
     OpenTable {
         table_info: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -147,13 +160,13 @@ pub enum Error {
 
     #[snafu(display("Failed to read system catalog table records"))]
     ReadSystemCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
     #[snafu(display("Failed to create recordbatch, source: {}", source))]
     CreateRecordBatch {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_recordbatch::error::Error,
     },
 
@@ -162,7 +175,7 @@ pub enum Error {
         source
     ))]
     InsertCatalogRecord {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -173,7 +186,7 @@ pub enum Error {
     ))]
     DeregisterTable {
         request: DeregisterTableRequest,
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
@@ -182,36 +195,43 @@ pub enum Error {
 
     #[snafu(display("Failed to scan system catalog table, source: {}", source))]
     SystemCatalogTableScan {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("{source}"))]
     Internal {
-        #[snafu(backtrace)]
+        location: Location,
         source: BoxedError,
     },
 
+    #[snafu(display(
+        "Failed to upgrade weak catalog manager reference. location: {}",
+        location
+    ))]
+    UpgradeWeakCatalogManagerRef { location: Location },
+
     #[snafu(display("Failed to execute system catalog table scan, source: {}", source))]
     SystemCatalogTableScanExec {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_query::error::Error,
     },
+
     #[snafu(display("Cannot parse catalog value, source: {}", source))]
     InvalidCatalogValue {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_catalog::error::Error,
     },
 
     #[snafu(display("Failed to perform metasrv operation, source: {}", source))]
     MetaSrv {
-        #[snafu(backtrace)]
+        location: Location,
         source: meta_client::error::Error,
     },
 
     #[snafu(display("Invalid table info in catalog, source: {}", source))]
     InvalidTableInfoInCatalog {
-        #[snafu(backtrace)]
+        location: Location,
         source: datatypes::error::Error,
     },
 
@@ -230,12 +250,18 @@ pub enum Error {
 
     #[snafu(display("Table schema mismatch, source: {}", source))]
     TableSchemaMismatch {
-        #[snafu(backtrace)]
+        location: Location,
         source: table::error::Error,
     },
 
     #[snafu(display("A generic error has occurred, msg: {}", msg))]
     Generic { msg: String, location: Location },
+
+    #[snafu(display("Table metadata manager error: {}", source))]
+    TableMetadataManager {
+        source: common_meta::error::Error,
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -256,9 +282,11 @@ impl ErrorExt for Error {
             | Error::EmptyValue { .. }
             | Error::ValueDeserialize { .. } => StatusCode::StorageUnavailable,
 
-            Error::Generic { .. } | Error::SystemCatalogTypeMismatch { .. } => StatusCode::Internal,
+            Error::Generic { .. }
+            | Error::SystemCatalogTypeMismatch { .. }
+            | Error::UpgradeWeakCatalogManagerRef { .. } => StatusCode::Internal,
 
-            Error::ReadSystemCatalog { source, .. } | Error::CreateRecordBatch { source } => {
+            Error::ReadSystemCatalog { source, .. } | Error::CreateRecordBatch { source, .. } => {
                 source.status_code()
             }
             Error::InvalidCatalogValue { source, .. } => source.status_code(),
@@ -269,26 +297,31 @@ impl ErrorExt for Error {
                 StatusCode::InvalidArguments
             }
 
+            Error::ListCatalogs { source, .. } | Error::ListSchemas { source, .. } => {
+                source.status_code()
+            }
+
             Error::OpenSystemCatalog { source, .. }
             | Error::CreateSystemCatalog { source, .. }
             | Error::InsertCatalogRecord { source, .. }
             | Error::OpenTable { source, .. }
             | Error::CreateTable { source, .. }
             | Error::DeregisterTable { source, .. }
-            | Error::TableSchemaMismatch { source } => source.status_code(),
+            | Error::TableSchemaMismatch { source, .. } => source.status_code(),
 
             Error::MetaSrv { source, .. } => source.status_code(),
-            Error::SystemCatalogTableScan { source } => source.status_code(),
-            Error::SystemCatalogTableScanExec { source } => source.status_code(),
-            Error::InvalidTableInfoInCatalog { source } => source.status_code(),
+            Error::SystemCatalogTableScan { source, .. } => source.status_code(),
+            Error::SystemCatalogTableScanExec { source, .. } => source.status_code(),
+            Error::InvalidTableInfoInCatalog { source, .. } => source.status_code(),
 
-            Error::CompileScriptInternal { source } | Error::Internal { source } => {
+            Error::CompileScriptInternal { source, .. } | Error::Internal { source, .. } => {
                 source.status_code()
             }
 
             Error::Unimplemented { .. } | Error::NotSupported { .. } => StatusCode::Unsupported,
             Error::QueryAccessDenied { .. } => StatusCode::AccessDenied,
             Error::Datafusion { .. } => StatusCode::EngineExecuteQuery,
+            Error::TableMetadataManager { source, .. } => source.status_code(),
         }
     }
 

src/catalog/src/helper.rs

@@ -1,380 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::fmt::{Display, Formatter};
-
-use common_catalog::error::{
-    DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu, SerializeCatalogEntryValueSnafu,
-};
-use lazy_static::lazy_static;
-use regex::Regex;
-use serde::{Deserialize, Serialize, Serializer};
-use snafu::{ensure, OptionExt, ResultExt};
-use table::metadata::{RawTableInfo, TableId, TableVersion};
-
-pub const CATALOG_KEY_PREFIX: &str = "__c";
-pub const SCHEMA_KEY_PREFIX: &str = "__s";
-pub const TABLE_GLOBAL_KEY_PREFIX: &str = "__tg";
-pub const TABLE_REGIONAL_KEY_PREFIX: &str = "__tr";
-
-const ALPHANUMERICS_NAME_PATTERN: &str = "[a-zA-Z_][a-zA-Z0-9_]*";
-
-lazy_static! {
-    static ref CATALOG_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{CATALOG_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})$"
-    ))
-    .unwrap();
-}
-
-lazy_static! {
-    static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{SCHEMA_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})$"
-    ))
-    .unwrap();
-}
-
-lazy_static! {
-    static ref TABLE_GLOBAL_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{TABLE_GLOBAL_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})$"
-    ))
-    .unwrap();
-}
-
-lazy_static! {
-    static ref TABLE_REGIONAL_KEY_PATTERN: Regex = Regex::new(&format!(
-        "^{TABLE_REGIONAL_KEY_PREFIX}-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})-({ALPHANUMERICS_NAME_PATTERN})-([0-9]+)$"
-    ))
-    .unwrap();
-}
-
-pub fn build_catalog_prefix() -> String {
-    format!("{CATALOG_KEY_PREFIX}-")
-}
-
-pub fn build_schema_prefix(catalog_name: impl AsRef<str>) -> String {
-    format!("{SCHEMA_KEY_PREFIX}-{}-", catalog_name.as_ref())
-}
-
-pub fn build_table_global_prefix(
-    catalog_name: impl AsRef<str>,
-    schema_name: impl AsRef<str>,
-) -> String {
-    format!(
-        "{TABLE_GLOBAL_KEY_PREFIX}-{}-{}-",
-        catalog_name.as_ref(),
-        schema_name.as_ref()
-    )
-}
-
-pub fn build_table_regional_prefix(
-    catalog_name: impl AsRef<str>,
-    schema_name: impl AsRef<str>,
-) -> String {
-    format!(
-        "{}-{}-{}-",
-        TABLE_REGIONAL_KEY_PREFIX,
-        catalog_name.as_ref(),
-        schema_name.as_ref()
-    )
-}
-
-/// Table global info has only one key across all datanodes so it does not have `node_id` field.
-#[derive(Clone)]
-pub struct TableGlobalKey {
-    pub catalog_name: String,
-    pub schema_name: String,
-    pub table_name: String,
-}
-
-impl Display for TableGlobalKey {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(TABLE_GLOBAL_KEY_PREFIX)?;
-        f.write_str("-")?;
-        f.write_str(&self.catalog_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.schema_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.table_name)
-    }
-}
-
-impl TableGlobalKey {
-    pub fn parse<S: AsRef<str>>(s: S) -> Result<Self, Error> {
-        let key = s.as_ref();
-        let captures = TABLE_GLOBAL_KEY_PATTERN
-            .captures(key)
-            .context(InvalidCatalogSnafu { key })?;
-        ensure!(captures.len() == 4, InvalidCatalogSnafu { key });
-
-        Ok(Self {
-            catalog_name: captures[1].to_string(),
-            schema_name: captures[2].to_string(),
-            table_name: captures[3].to_string(),
-        })
-    }
-}
-
-/// Table global info contains necessary info for a datanode to create table regions, including
-/// table id, table meta(schema...), region id allocation across datanodes.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct TableGlobalValue {
-    /// Id of datanode that created the global table info kv. only for debugging.
-    pub node_id: u64,
-    /// Allocation of region ids across all datanodes.
-    pub regions_id_map: HashMap<u64, Vec<u32>>,
-    pub table_info: RawTableInfo,
-}
-
-impl TableGlobalValue {
-    pub fn table_id(&self) -> TableId {
-        self.table_info.ident.table_id
-    }
-}
-
-/// Table regional info that varies between datanode, so it contains a `node_id` field.
-pub struct TableRegionalKey {
-    pub catalog_name: String,
-    pub schema_name: String,
-    pub table_name: String,
-    pub node_id: u64,
-}
-
-impl Display for TableRegionalKey {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(TABLE_REGIONAL_KEY_PREFIX)?;
-        f.write_str("-")?;
-        f.write_str(&self.catalog_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.schema_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.table_name)?;
-        f.write_str("-")?;
-        f.serialize_u64(self.node_id)
-    }
-}
-
-impl TableRegionalKey {
-    pub fn parse<S: AsRef<str>>(s: S) -> Result<Self, Error> {
-        let key = s.as_ref();
-        let captures = TABLE_REGIONAL_KEY_PATTERN
-            .captures(key)
-            .context(InvalidCatalogSnafu { key })?;
-        ensure!(captures.len() == 5, InvalidCatalogSnafu { key });
-        let node_id = captures[4]
-            .to_string()
-            .parse()
-            .map_err(|_| InvalidCatalogSnafu { key }.build())?;
-        Ok(Self {
-            catalog_name: captures[1].to_string(),
-            schema_name: captures[2].to_string(),
-            table_name: captures[3].to_string(),
-            node_id,
-        })
-    }
-}
-
-/// Regional table info of specific datanode, including table version on that datanode and
-/// region ids allocated by metasrv.
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct TableRegionalValue {
-    pub version: TableVersion,
-    pub regions_ids: Vec<u32>,
-    pub engine_name: Option<String>,
-}
-
-pub struct CatalogKey {
-    pub catalog_name: String,
-}
-
-impl Display for CatalogKey {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(CATALOG_KEY_PREFIX)?;
-        f.write_str("-")?;
-        f.write_str(&self.catalog_name)
-    }
-}
-
-impl CatalogKey {
-    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-        let key = s.as_ref();
-        let captures = CATALOG_KEY_PATTERN
-            .captures(key)
-            .context(InvalidCatalogSnafu { key })?;
-        ensure!(captures.len() == 2, InvalidCatalogSnafu { key });
-        Ok(Self {
-            catalog_name: captures[1].to_string(),
-        })
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct CatalogValue;
-
-pub struct SchemaKey {
-    pub catalog_name: String,
-    pub schema_name: String,
-}
-
-impl Display for SchemaKey {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(SCHEMA_KEY_PREFIX)?;
-        f.write_str("-")?;
-        f.write_str(&self.catalog_name)?;
-        f.write_str("-")?;
-        f.write_str(&self.schema_name)
-    }
-}
-
-impl SchemaKey {
-    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-        let key = s.as_ref();
-        let captures = SCHEMA_KEY_PATTERN
-            .captures(key)
-            .context(InvalidCatalogSnafu { key })?;
-        ensure!(captures.len() == 3, InvalidCatalogSnafu { key });
-        Ok(Self {
-            catalog_name: captures[1].to_string(),
-            schema_name: captures[2].to_string(),
-        })
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct SchemaValue;
-
-macro_rules! define_catalog_value {
-    ( $($val_ty: ty), *) => {
-            $(
-                impl $val_ty {
-                    pub fn parse(s: impl AsRef<str>) -> Result<Self, Error> {
-                        serde_json::from_str(s.as_ref())
-                            .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() })
-                    }
-
-                    pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self, Error> {
-                         Self::parse(&String::from_utf8_lossy(bytes.as_ref()))
-                    }
-
-                    pub fn as_bytes(&self) -> Result<Vec<u8>, Error> {
-                        Ok(serde_json::to_string(self)
-                            .context(SerializeCatalogEntryValueSnafu)?
-                            .into_bytes())
-                    }
-                }
-            )*
-        }
-}
-
-define_catalog_value!(
-    TableRegionalValue,
-    TableGlobalValue,
-    CatalogValue,
-    SchemaValue
-);
-
-#[cfg(test)]
-mod tests {
-    use datatypes::prelude::ConcreteDataType;
-    use datatypes::schema::{ColumnSchema, RawSchema, Schema};
-    use table::metadata::{RawTableMeta, TableIdent, TableType};
-
-    use super::*;
-
-    #[test]
-    fn test_parse_catalog_key() {
-        let key = "__c-C";
-        let catalog_key = CatalogKey::parse(key).unwrap();
-        assert_eq!("C", catalog_key.catalog_name);
-        assert_eq!(key, catalog_key.to_string());
-    }
-
-    #[test]
-    fn test_parse_schema_key() {
-        let key = "__s-C-S";
-        let schema_key = SchemaKey::parse(key).unwrap();
-        assert_eq!("C", schema_key.catalog_name);
-        assert_eq!("S", schema_key.schema_name);
-        assert_eq!(key, schema_key.to_string());
-    }
-
-    #[test]
-    fn test_parse_table_key() {
-        let key = "__tg-C-S-T";
-        let entry = TableGlobalKey::parse(key).unwrap();
-        assert_eq!("C", entry.catalog_name);
-        assert_eq!("S", entry.schema_name);
-        assert_eq!("T", entry.table_name);
-        assert_eq!(key, &entry.to_string());
-    }
-
-    #[test]
-    fn test_build_prefix() {
-        assert_eq!("__c-", build_catalog_prefix());
-        assert_eq!("__s-CATALOG-", build_schema_prefix("CATALOG"));
-        assert_eq!(
-            "__tg-CATALOG-SCHEMA-",
-            build_table_global_prefix("CATALOG", "SCHEMA")
-        );
-    }
-
-    #[test]
-    fn test_serialize_schema() {
-        let schema = Schema::new(vec![ColumnSchema::new(
-            "name",
-            ConcreteDataType::string_datatype(),
-            true,
-        )]);
-
-        let meta = RawTableMeta {
-            schema: RawSchema::from(&schema),
-            engine: "mito".to_string(),
-            created_on: chrono::DateTime::default(),
-            primary_key_indices: vec![0, 1],
-            next_column_id: 3,
-            engine_options: Default::default(),
-            value_indices: vec![2, 3],
-            options: Default::default(),
-            region_numbers: vec![1],
-        };
-
-        let table_info = RawTableInfo {
-            ident: TableIdent {
-                table_id: 42,
-                version: 1,
-            },
-            name: "table_1".to_string(),
-            desc: Some("blah".to_string()),
-            catalog_name: "catalog_1".to_string(),
-            schema_name: "schema_1".to_string(),
-            meta,
-            table_type: TableType::Base,
-        };
-
-        let value = TableGlobalValue {
-            node_id: 0,
-            regions_id_map: HashMap::from([(0, vec![1, 2, 3])]),
-            table_info,
-        };
-        let serialized = serde_json::to_string(&value).unwrap();
-        let deserialized = TableGlobalValue::parse(serialized).unwrap();
-        assert_eq!(value, deserialized);
-    }
-
-    #[test]
-    fn test_table_global_value_compatibility() {
-        let s = r#"{"node_id":1,"regions_id_map":{"1":[0]},"table_info":{"ident":{"table_id":1098,"version":1},"name":"container_cpu_limit","desc":"Created on insertion","catalog_name":"greptime","schema_name":"dd","meta":{"schema":{"column_schemas":[{"name":"container_id","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"container_name","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"docker_image","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"host","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"image_name","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"image_tag","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"interval","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"runtime","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"short_image","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"type","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"dd_value","data_type":{"Float64":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":true,"default_constraint":null,"metadata":{"greptime:time_index":"true"}},{"name":"git.repository_url","data_type":{"String":null},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}}],"timestamp_index":11,"version":1},"primary_key_indices":[0,1,2,3,4,5,6,7,8,9,12],"value_indices":[10,11],"engine":"mito","next_column_id":12,"region_numbers":[],"engine_options":{},"options":{},"created_on":"1970-01-01T00:00:00Z"},"table_type":"Base"}}"#;
-        TableGlobalValue::parse(s).unwrap();
-    }
-}

src/catalog/src/information_schema.rs

@@ -15,150 +15,162 @@
 mod columns;
 mod tables;
 
-use std::any::Any;
-use std::sync::Arc;
+use std::collections::HashMap;
+use std::sync::{Arc, Weak};
 
-use async_trait::async_trait;
-use common_error::prelude::BoxedError;
-use common_query::physical_plan::PhysicalPlanRef;
-use common_query::prelude::Expr;
+use common_catalog::consts::INFORMATION_SCHEMA_NAME;
+use common_error::ext::BoxedError;
 use common_recordbatch::{RecordBatchStreamAdaptor, SendableRecordBatchStream};
 use datatypes::schema::SchemaRef;
 use futures_util::StreamExt;
 use snafu::ResultExt;
-use store_api::storage::ScanRequest;
+use store_api::data_source::DataSource;
+use store_api::storage::{ScanRequest, TableId};
 use table::error::{SchemaConversionSnafu, TablesRecordBatchSnafu};
-use table::{Result as TableResult, Table, TableRef};
+use table::metadata::{
+    FilterPushDownType, TableInfoBuilder, TableInfoRef, TableMetaBuilder, TableType,
+};
+use table::thin_table::{ThinTable, ThinTableAdapter};
+use table::TableRef;
 
 use self::columns::InformationSchemaColumns;
 use crate::error::Result;
 use crate::information_schema::tables::InformationSchemaTables;
-use crate::{CatalogProviderRef, SchemaProvider};
+use crate::CatalogManager;
 
-const TABLES: &str = "tables";
-const COLUMNS: &str = "columns";
+pub const TABLES: &str = "tables";
+pub const COLUMNS: &str = "columns";
 
-pub(crate) struct InformationSchemaProvider {
+pub struct InformationSchemaProvider {
     catalog_name: String,
-    catalog_provider: CatalogProviderRef,
-    tables: Vec<String>,
+    catalog_manager: Weak<dyn CatalogManager>,
 }
 
 impl InformationSchemaProvider {
-    pub(crate) fn new(catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
+    pub fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
         Self {
             catalog_name,
-            catalog_provider,
-            tables: vec![TABLES.to_string(), COLUMNS.to_string()],
+            catalog_manager,
         }
     }
-}
 
-#[async_trait]
-impl SchemaProvider for InformationSchemaProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
+    /// Build a map of [TableRef] in information schema.
+    /// Including `tables` and `columns`.
+    pub fn build(
+        catalog_name: String,
+        catalog_manager: Weak<dyn CatalogManager>,
+    ) -> HashMap<String, TableRef> {
+        let provider = Self::new(catalog_name, catalog_manager);
+
+        let mut schema = HashMap::new();
+        schema.insert(TABLES.to_owned(), provider.table(TABLES).unwrap());
+        schema.insert(COLUMNS.to_owned(), provider.table(COLUMNS).unwrap());
+        schema
     }
 
-    async fn table_names(&self) -> Result<Vec<String>> {
-        Ok(self.tables.clone())
+    pub fn table(&self, name: &str) -> Option<TableRef> {
+        self.information_table(name).map(|table| {
+            let table_info = Self::table_info(self.catalog_name.clone(), &table);
+            let filter_pushdown = FilterPushDownType::Unsupported;
+            let thin_table = ThinTable::new(table_info, filter_pushdown);
+
+            let data_source = Arc::new(InformationTableDataSource::new(table));
+            Arc::new(ThinTableAdapter::new(thin_table, data_source)) as _
+        })
     }
 
-    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
-        let stream_builder = match name.to_ascii_lowercase().as_ref() {
-            TABLES => Arc::new(InformationSchemaTables::new(
+    fn information_table(&self, name: &str) -> Option<InformationTableRef> {
+        match name.to_ascii_lowercase().as_str() {
+            TABLES => Some(Arc::new(InformationSchemaTables::new(
                 self.catalog_name.clone(),
-                self.catalog_provider.clone(),
-            )) as _,
-            COLUMNS => Arc::new(InformationSchemaColumns::new(
+                self.catalog_manager.clone(),
+            )) as _),
+            COLUMNS => Some(Arc::new(InformationSchemaColumns::new(
                 self.catalog_name.clone(),
-                self.catalog_provider.clone(),
-            )) as _,
-            _ => {
-                return Ok(None);
-            }
-        };
-
-        Ok(Some(Arc::new(InformationTable::new(stream_builder))))
+                self.catalog_manager.clone(),
+            )) as _),
+            _ => None,
+        }
     }
 
-    async fn table_exist(&self, name: &str) -> Result<bool> {
-        let normalized_name = name.to_ascii_lowercase();
-        Ok(self.tables.contains(&normalized_name))
+    fn table_info(catalog_name: String, table: &InformationTableRef) -> TableInfoRef {
+        let table_meta = TableMetaBuilder::default()
+            .schema(table.schema())
+            .primary_key_indices(vec![])
+            .next_column_id(0)
+            .build()
+            .unwrap();
+        let table_info = TableInfoBuilder::default()
+            .table_id(table.table_id())
+            .name(table.table_name().to_owned())
+            .catalog_name(catalog_name)
+            .schema_name(INFORMATION_SCHEMA_NAME.to_owned())
+            .meta(table_meta)
+            .table_type(table.table_type())
+            .build()
+            .unwrap();
+        Arc::new(table_info)
     }
 }
 
-// TODO(ruihang): make it a more generic trait:
-// https://github.com/GreptimeTeam/greptimedb/pull/1639#discussion_r1205001903
-pub trait InformationStreamBuilder: Send + Sync {
-    fn to_stream(&self) -> Result<SendableRecordBatchStream>;
+trait InformationTable {
+    fn table_id(&self) -> TableId;
+
+    fn table_name(&self) -> &'static str;
 
     fn schema(&self) -> SchemaRef;
-}
 
-pub struct InformationTable {
-    stream_builder: Arc<dyn InformationStreamBuilder>,
-}
+    fn to_stream(&self) -> Result<SendableRecordBatchStream>;
 
-impl InformationTable {
-    pub fn new(stream_builder: Arc<dyn InformationStreamBuilder>) -> Self {
-        Self { stream_builder }
+    fn table_type(&self) -> TableType {
+        TableType::Temporary
     }
 }
 
-#[async_trait]
-impl Table for InformationTable {
-    fn as_any(&self) -> &dyn Any {
-        self
+type InformationTableRef = Arc<dyn InformationTable + Send + Sync>;
+
+struct InformationTableDataSource {
+    table: InformationTableRef,
+}
+
+impl InformationTableDataSource {
+    fn new(table: InformationTableRef) -> Self {
+        Self { table }
     }
 
-    fn schema(&self) -> SchemaRef {
-        self.stream_builder.schema()
+    fn try_project(&self, projection: &[usize]) -> std::result::Result<SchemaRef, BoxedError> {
+        let schema = self
+            .table
+            .schema()
+            .try_project(projection)
+            .context(SchemaConversionSnafu)
+            .map_err(BoxedError::new)?;
+        Ok(Arc::new(schema))
     }
+}
 
-    fn table_info(&self) -> table::metadata::TableInfoRef {
-        unreachable!("Should not call table_info() of InformationTable directly")
-    }
-
-    /// Scan the table and returns a SendableRecordBatchStream.
-    async fn scan(
+impl DataSource for InformationTableDataSource {
+    fn get_stream(
         &self,
-        _projection: Option<&Vec<usize>>,
-        _filters: &[Expr],
-        // limit can be used to reduce the amount scanned
-        // from the datasource as a performance optimization.
-        // If set, it contains the amount of rows needed by the `LogicalPlan`,
-        // The datasource should return *at least* this number of rows if available.
-        _limit: Option<usize>,
-    ) -> TableResult<PhysicalPlanRef> {
-        unimplemented!()
-    }
-
-    async fn scan_to_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
+        request: ScanRequest,
+    ) -> std::result::Result<SendableRecordBatchStream, BoxedError> {
         let projection = request.projection;
-        let projected_schema = if let Some(projection) = &projection {
-            Arc::new(
-                self.schema()
-                    .try_project(projection)
-                    .context(SchemaConversionSnafu)?,
-            )
-        } else {
-            self.schema()
+        let projected_schema = match &projection {
+            Some(projection) => self.try_project(projection)?,
+            None => self.table.schema(),
         };
+
         let stream = self
-            .stream_builder
+            .table
             .to_stream()
             .map_err(BoxedError::new)
-            .context(TablesRecordBatchSnafu)?
-            .map(move |batch| {
-                batch.and_then(|batch| {
-                    if let Some(projection) = &projection {
-                        batch.try_project(projection)
-                    } else {
-                        Ok(batch)
-                    }
-                })
+            .context(TablesRecordBatchSnafu)
+            .map_err(BoxedError::new)?
+            .map(move |batch| match &projection {
+                Some(p) => batch.and_then(|b| b.try_project(p)),
+                None => batch,
             });
+
         let stream = RecordBatchStreamAdaptor {
             schema: projected_schema,
             stream: Box::pin(stream),

src/catalog/src/information_schema/columns.rs

@@ -12,33 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::{
-    SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
+    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME, SEMANTIC_TYPE_FIELD,
+    SEMANTIC_TYPE_PRIMARY_KEY, SEMANTIC_TYPE_TIME_INDEX,
 };
-use common_error::prelude::BoxedError;
+use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use datafusion::datasource::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
 use datatypes::prelude::{ConcreteDataType, DataType};
 use datatypes::scalars::ScalarVectorBuilder;
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::vectors::{StringVectorBuilder, VectorRef};
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::TableId;
 
-use super::InformationStreamBuilder;
-use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
-use crate::CatalogProviderRef;
+use super::tables::InformationSchemaTables;
+use super::{InformationTable, COLUMNS, TABLES};
+use crate::error::{
+    CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
+};
+use crate::CatalogManager;
 
 pub(super) struct InformationSchemaColumns {
     schema: SchemaRef,
     catalog_name: String,
-    catalog_provider: CatalogProviderRef,
+    catalog_manager: Weak<dyn CatalogManager>,
 }
 
 const TABLE_CATALOG: &str = "table_catalog";
@@ -49,32 +54,43 @@ const DATA_TYPE: &str = "data_type";
 const SEMANTIC_TYPE: &str = "semantic_type";
 
 impl InformationSchemaColumns {
-    pub(super) fn new(catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
-        let schema = Arc::new(Schema::new(vec![
+    pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
+        Self {
+            schema: Self::schema(),
+            catalog_name,
+            catalog_manager,
+        }
+    }
+
+    fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
             ColumnSchema::new(TABLE_CATALOG, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(TABLE_SCHEMA, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(TABLE_NAME, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(COLUMN_NAME, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(DATA_TYPE, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(SEMANTIC_TYPE, ConcreteDataType::string_datatype(), false),
-        ]));
-        Self {
-            schema,
-            catalog_name,
-            catalog_provider,
-        }
+        ]))
     }
 
     fn builder(&self) -> InformationSchemaColumnsBuilder {
         InformationSchemaColumnsBuilder::new(
             self.schema.clone(),
             self.catalog_name.clone(),
-            self.catalog_provider.clone(),
+            self.catalog_manager.clone(),
         )
     }
 }
 
-impl InformationStreamBuilder for InformationSchemaColumns {
+impl InformationTable for InformationSchemaColumns {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_COLUMNS_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        COLUMNS
+    }
+
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -103,7 +119,7 @@ impl InformationStreamBuilder for InformationSchemaColumns {
 struct InformationSchemaColumnsBuilder {
     schema: SchemaRef,
     catalog_name: String,
-    catalog_provider: CatalogProviderRef,
+    catalog_manager: Weak<dyn CatalogManager>,
 
     catalog_names: StringVectorBuilder,
     schema_names: StringVectorBuilder,
@@ -114,11 +130,15 @@ struct InformationSchemaColumnsBuilder {
 }
 
 impl InformationSchemaColumnsBuilder {
-    fn new(schema: SchemaRef, catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
+    fn new(
+        schema: SchemaRef,
+        catalog_name: String,
+        catalog_manager: Weak<dyn CatalogManager>,
+    ) -> Self {
         Self {
             schema,
             catalog_name,
-            catalog_provider,
+            catalog_manager,
             catalog_names: StringVectorBuilder::with_capacity(42),
             schema_names: StringVectorBuilder::with_capacity(42),
             table_names: StringVectorBuilder::with_capacity(42),
@@ -131,13 +151,44 @@ impl InformationSchemaColumnsBuilder {
     /// Construct the `information_schema.tables` virtual table
     async fn make_tables(&mut self) -> Result<RecordBatch> {
         let catalog_name = self.catalog_name.clone();
+        let catalog_manager = self
+            .catalog_manager
+            .upgrade()
+            .context(UpgradeWeakCatalogManagerRefSnafu)?;
+
+        for schema_name in catalog_manager.schema_names(&catalog_name).await? {
+            if !catalog_manager
+                .schema_exist(&catalog_name, &schema_name)
+                .await?
+            {
+                continue;
+            }
+            for table_name in catalog_manager
+                .table_names(&catalog_name, &schema_name)
+                .await?
+            {
+                let (keys, schema) = if let Some(table) = catalog_manager
+                    .table(&catalog_name, &schema_name, &table_name)
+                    .await?
+                {
+                    let keys = &table.table_info().meta.primary_key_indices;
+                    let schema = table.schema();
+                    (keys.clone(), schema)
+                } else {
+                    // TODO: this specific branch is only a workaround for FrontendCatalogManager.
+                    if schema_name == INFORMATION_SCHEMA_NAME {
+                        if table_name == COLUMNS {
+                            (vec![], InformationSchemaColumns::schema())
+                        } else if table_name == TABLES {
+                            (vec![], InformationSchemaTables::schema())
+                        } else {
+                            continue;
+                        }
+                    } else {
+                        continue;
+                    }
+                };
 
-        for schema_name in self.catalog_provider.schema_names().await? {
-            let Some(schema) = self.catalog_provider.schema(&schema_name).await? else { continue };
-            for table_name in schema.table_names().await? {
-                let Some(table) = schema.table(&table_name).await? else { continue };
-                let keys = &table.table_info().meta.primary_key_indices;
-                let schema = table.schema();
                 for (idx, column) in schema.column_schemas().iter().enumerate() {
                     let semantic_type = if column.is_time_index() {
                         SEMANTIC_TYPE_TIME_INDEX

src/catalog/src/information_schema/tables.rs

@@ -12,60 +12,78 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
-use common_catalog::consts::INFORMATION_SCHEMA_NAME;
-use common_error::prelude::BoxedError;
+use common_catalog::consts::{
+    INFORMATION_SCHEMA_COLUMNS_TABLE_ID, INFORMATION_SCHEMA_NAME,
+    INFORMATION_SCHEMA_TABLES_TABLE_ID,
+};
+use common_error::ext::BoxedError;
 use common_query::physical_plan::TaskContext;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use datafusion::datasource::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
 use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
 use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder};
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::TableId;
 use table::metadata::TableType;
 
-use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
-use crate::information_schema::InformationStreamBuilder;
-use crate::CatalogProviderRef;
+use super::{COLUMNS, TABLES};
+use crate::error::{
+    CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu,
+};
+use crate::information_schema::InformationTable;
+use crate::CatalogManager;
 
 pub(super) struct InformationSchemaTables {
     schema: SchemaRef,
     catalog_name: String,
-    catalog_provider: CatalogProviderRef,
+    catalog_manager: Weak<dyn CatalogManager>,
 }
 
 impl InformationSchemaTables {
-    pub(super) fn new(catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
-        let schema = Arc::new(Schema::new(vec![
+    pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
+        Self {
+            schema: Self::schema(),
+            catalog_name,
+            catalog_manager,
+        }
+    }
+
+    pub(crate) fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
             ColumnSchema::new("table_catalog", ConcreteDataType::string_datatype(), false),
             ColumnSchema::new("table_schema", ConcreteDataType::string_datatype(), false),
             ColumnSchema::new("table_name", ConcreteDataType::string_datatype(), false),
             ColumnSchema::new("table_type", ConcreteDataType::string_datatype(), false),
             ColumnSchema::new("table_id", ConcreteDataType::uint32_datatype(), true),
             ColumnSchema::new("engine", ConcreteDataType::string_datatype(), true),
-        ]));
-        Self {
-            schema,
-            catalog_name,
-            catalog_provider,
-        }
+        ]))
     }
 
     fn builder(&self) -> InformationSchemaTablesBuilder {
         InformationSchemaTablesBuilder::new(
             self.schema.clone(),
             self.catalog_name.clone(),
-            self.catalog_provider.clone(),
+            self.catalog_manager.clone(),
         )
     }
 }
 
-impl InformationStreamBuilder for InformationSchemaTables {
+impl InformationTable for InformationSchemaTables {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_TABLES_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        TABLES
+    }
+
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -97,7 +115,7 @@ impl InformationStreamBuilder for InformationSchemaTables {
 struct InformationSchemaTablesBuilder {
     schema: SchemaRef,
     catalog_name: String,
-    catalog_provider: CatalogProviderRef,
+    catalog_manager: Weak<dyn CatalogManager>,
 
     catalog_names: StringVectorBuilder,
     schema_names: StringVectorBuilder,
@@ -108,11 +126,15 @@ struct InformationSchemaTablesBuilder {
 }
 
 impl InformationSchemaTablesBuilder {
-    fn new(schema: SchemaRef, catalog_name: String, catalog_provider: CatalogProviderRef) -> Self {
+    fn new(
+        schema: SchemaRef,
+        catalog_name: String,
+        catalog_manager: Weak<dyn CatalogManager>,
+    ) -> Self {
         Self {
             schema,
             catalog_name,
-            catalog_provider,
+            catalog_manager,
             catalog_names: StringVectorBuilder::with_capacity(42),
             schema_names: StringVectorBuilder::with_capacity(42),
             table_names: StringVectorBuilder::with_capacity(42),
@@ -125,24 +147,60 @@ impl InformationSchemaTablesBuilder {
     /// Construct the `information_schema.tables` virtual table
     async fn make_tables(&mut self) -> Result<RecordBatch> {
         let catalog_name = self.catalog_name.clone();
+        let catalog_manager = self
+            .catalog_manager
+            .upgrade()
+            .context(UpgradeWeakCatalogManagerRefSnafu)?;
 
-        for schema_name in self.catalog_provider.schema_names().await? {
-            if schema_name == INFORMATION_SCHEMA_NAME {
+        for schema_name in catalog_manager.schema_names(&catalog_name).await? {
+            if !catalog_manager
+                .schema_exist(&catalog_name, &schema_name)
+                .await?
+            {
                 continue;
             }
 
-            let Some(schema) = self.catalog_provider.schema(&schema_name).await? else { continue };
-            for table_name in schema.table_names().await? {
-                let Some(table) = schema.table(&table_name).await? else { continue };
-                let table_info = table.table_info();
-                self.add_table(
-                    &catalog_name,
-                    &schema_name,
-                    &table_name,
-                    table.table_type(),
-                    Some(table_info.ident.table_id),
-                    Some(&table_info.meta.engine),
-                );
+            for table_name in catalog_manager
+                .table_names(&catalog_name, &schema_name)
+                .await?
+            {
+                if let Some(table) = catalog_manager
+                    .table(&catalog_name, &schema_name, &table_name)
+                    .await?
+                {
+                    let table_info = table.table_info();
+                    self.add_table(
+                        &catalog_name,
+                        &schema_name,
+                        &table_name,
+                        table.table_type(),
+                        Some(table_info.ident.table_id),
+                        Some(&table_info.meta.engine),
+                    );
+                } else {
+                    // TODO: this specific branch is only a workaround for FrontendCatalogManager.
+                    if schema_name == INFORMATION_SCHEMA_NAME {
+                        if table_name == COLUMNS {
+                            self.add_table(
+                                &catalog_name,
+                                &schema_name,
+                                &table_name,
+                                TableType::Temporary,
+                                Some(INFORMATION_SCHEMA_COLUMNS_TABLE_ID),
+                                None,
+                            );
+                        } else if table_name == TABLES {
+                            self.add_table(
+                                &catalog_name,
+                                &schema_name,
+                                &table_name,
+                                TableType::Temporary,
+                                Some(INFORMATION_SCHEMA_TABLES_TABLE_ID),
+                                None,
+                            );
+                        }
+                    }
+                };
             }
         }
 

src/catalog/src/lib.rs

@@ -14,80 +14,64 @@
 
 #![feature(trait_upcasting)]
 #![feature(assert_matches)]
+#![feature(try_blocks)]
 
 use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use api::v1::meta::{RegionStat, TableName};
+use api::v1::meta::{RegionStat, TableIdent, TableName};
 use common_telemetry::{info, warn};
 use snafu::ResultExt;
 use table::engine::{EngineContext, TableEngineRef};
-use table::metadata::TableId;
+use table::metadata::{TableId, TableType};
 use table::requests::CreateTableRequest;
 use table::TableRef;
 
 use crate::error::{CreateTableSnafu, Result};
-pub use crate::schema::{SchemaProvider, SchemaProviderRef};
 
 pub mod error;
-pub mod helper;
-pub(crate) mod information_schema;
+pub mod information_schema;
 pub mod local;
 mod metrics;
 pub mod remote;
-pub mod schema;
 pub mod system;
 pub mod table_source;
 pub mod tables;
 
-/// Represents a catalog, comprising a number of named schemas.
-#[async_trait::async_trait]
-pub trait CatalogProvider: Sync + Send {
-    /// Returns the catalog provider as [`Any`](std::any::Any)
-    /// so that it can be downcast to a specific implementation.
-    fn as_any(&self) -> &dyn Any;
-
-    /// Retrieves the list of available schema names in this catalog.
-    async fn schema_names(&self) -> Result<Vec<String>>;
-
-    /// Registers schema to this catalog.
-    async fn register_schema(
-        &self,
-        name: String,
-        schema: SchemaProviderRef,
-    ) -> Result<Option<SchemaProviderRef>>;
-
-    /// Retrieves a specific schema from the catalog by name, provided it exists.
-    async fn schema(&self, name: &str) -> Result<Option<SchemaProviderRef>>;
-}
-
-pub type CatalogProviderRef = Arc<dyn CatalogProvider>;
-
 #[async_trait::async_trait]
 pub trait CatalogManager: Send + Sync {
+    fn as_any(&self) -> &dyn Any;
+
     /// Starts a catalog manager.
     async fn start(&self) -> Result<()>;
 
-    async fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Result<Option<CatalogProviderRef>>;
-
-    /// Registers a table within given catalog/schema to catalog manager,
-    /// returns whether the table registered.
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
-
-    /// Deregisters a table within given catalog/schema to catalog manager,
-    /// returns whether the table deregistered.
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool>;
+    /// Registers a catalog to catalog manager, returns whether the catalog exist before.
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool>;
 
     /// Register a schema with catalog name and schema name. Retuens whether the
     /// schema registered.
+    ///
+    /// # Errors
+    ///
+    /// This method will/should fail if catalog not exist
     async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool>;
 
+    /// Deregisters a database within given catalog/schema to catalog manager
+    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool>;
+
+    /// Registers a table within given catalog/schema to catalog manager,
+    /// returns whether the table registered.
+    ///
+    /// # Errors
+    ///
+    /// This method will/should fail if catalog or schema not exist
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<bool>;
+
+    /// Deregisters a table within given catalog/schema to catalog manager
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()>;
+
     /// Rename a table to [RenameTableRequest::new_table_name], returns whether the table is renamed.
     async fn rename_table(&self, request: RenameTableRequest) -> Result<bool>;
 
@@ -97,9 +81,15 @@ pub trait CatalogManager: Send + Sync {
 
     async fn catalog_names(&self) -> Result<Vec<String>>;
 
-    async fn catalog(&self, catalog: &str) -> Result<Option<CatalogProviderRef>>;
+    async fn schema_names(&self, catalog: &str) -> Result<Vec<String>>;
 
-    async fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>>;
+    async fn table_names(&self, catalog: &str, schema: &str) -> Result<Vec<String>>;
+
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool>;
+
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool>;
+
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool>;
 
     /// Returns the table by catalog, schema and table name.
     async fn table(
@@ -108,8 +98,6 @@ pub trait CatalogManager: Send + Sync {
         schema: &str,
         table_name: &str,
     ) -> Result<Option<TableRef>>;
-
-    fn as_any(&self) -> &dyn Any;
 }
 
 pub type CatalogManagerRef = Arc<dyn CatalogManager>;
@@ -164,17 +152,15 @@ pub struct DeregisterTableRequest {
 }
 
 #[derive(Debug, Clone)]
-pub struct RegisterSchemaRequest {
+pub struct DeregisterSchemaRequest {
     pub catalog: String,
     pub schema: String,
 }
 
-pub trait CatalogProviderFactory {
-    fn create(&self, catalog_name: String) -> CatalogProviderRef;
-}
-
-pub trait SchemaProviderFactory {
-    fn create(&self, catalog_name: String, schema_name: String) -> SchemaProviderRef;
+#[derive(Debug, Clone)]
+pub struct RegisterSchemaRequest {
+    pub catalog: String,
+    pub schema: String,
 }
 
 pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
@@ -202,7 +188,7 @@ pub(crate) async fn handle_system_table_request<'a, M: CatalogManager>(
                         table_name,
                     ),
                 })?;
-            manager
+            let _ = manager
                 .register_table(RegisterTableRequest {
                     catalog: catalog_name.clone(),
                     schema: schema_name.clone(),
@@ -231,31 +217,51 @@ pub async fn datanode_stat(catalog_manager: &CatalogManagerRef) -> (u64, Vec<Reg
     let mut region_number: u64 = 0;
     let mut region_stats = Vec::new();
 
-    let Ok(catalog_names) = catalog_manager.catalog_names().await else { return (region_number, region_stats) };
+    let Ok(catalog_names) = catalog_manager.catalog_names().await else {
+        return (region_number, region_stats);
+    };
     for catalog_name in catalog_names {
-        let Ok(Some(catalog)) = catalog_manager.catalog(&catalog_name).await else { continue };
-
-        let Ok(schema_names) = catalog.schema_names().await else { continue };
+        let Ok(schema_names) = catalog_manager.schema_names(&catalog_name).await else {
+            continue;
+        };
         for schema_name in schema_names {
-            let Ok(Some(schema)) = catalog.schema(&schema_name).await else { continue };
-
-            let Ok(table_names) = schema.table_names().await else { continue };
+            let Ok(table_names) = catalog_manager
+                .table_names(&catalog_name, &schema_name)
+                .await
+            else {
+                continue;
+            };
             for table_name in table_names {
-                let Ok(Some(table)) = schema.table(&table_name).await else { continue };
+                let Ok(Some(table)) = catalog_manager
+                    .table(&catalog_name, &schema_name, &table_name)
+                    .await
+                else {
+                    continue;
+                };
 
-                let region_numbers = &table.table_info().meta.region_numbers;
+                if table.table_type() != TableType::Base {
+                    continue;
+                }
+
+                let table_info = table.table_info();
+                let region_numbers = &table_info.meta.region_numbers;
                 region_number += region_numbers.len() as u64;
 
-                let engine = &table.table_info().meta.engine;
+                let engine = &table_info.meta.engine;
+                let table_id = table_info.ident.table_id;
 
                 match table.region_stats() {
                     Ok(stats) => {
                         let stats = stats.into_iter().map(|stat| RegionStat {
                             region_id: stat.region_id,
-                            table_name: Some(TableName {
-                                catalog_name: catalog_name.clone(),
-                                schema_name: schema_name.clone(),
-                                table_name: table_name.clone(),
+                            table_ident: Some(TableIdent {
+                                table_id,
+                                table_name: Some(TableName {
+                                    catalog_name: catalog_name.clone(),
+                                    schema_name: schema_name.clone(),
+                                    table_name: table_name.clone(),
+                                }),
+                                engine: engine.clone(),
                             }),
                             approximate_bytes: stat.disk_usage_bytes as i64,
                             attrs: HashMap::from([("engine_name".to_owned(), engine.clone())]),

src/catalog/src/local.rs

@@ -16,6 +16,4 @@ pub mod manager;
 pub mod memory;
 
 pub use manager::LocalCatalogManager;
-pub use memory::{
-    new_memory_catalog_list, MemoryCatalogManager, MemoryCatalogProvider, MemorySchemaProvider,
-};
+pub use memory::{new_memory_catalog_manager, MemoryCatalogManager};

src/catalog/src/local/manager.rs

@@ -18,7 +18,8 @@ use std::sync::Arc;
 
 use common_catalog::consts::{
     DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
-    MITO_ENGINE, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_NAME,
+    MITO_ENGINE, NUMBERS_TABLE_ID, SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID,
+    SYSTEM_CATALOG_TABLE_NAME,
 };
 use common_catalog::format_full_table_name;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
@@ -32,7 +33,7 @@ use table::engine::manager::TableEngineManagerRef;
 use table::engine::EngineContext;
 use table::metadata::TableId;
 use table::requests::OpenTableRequest;
-use table::table::numbers::NumbersTable;
+use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
 use table::table::TableIdProvider;
 use table::TableRef;
 
@@ -40,18 +41,17 @@ use crate::error::{
     self, CatalogNotFoundSnafu, IllegalManagerStateSnafu, OpenTableSnafu, ReadSystemCatalogSnafu,
     Result, SchemaExistsSnafu, SchemaNotFoundSnafu, SystemCatalogSnafu,
     SystemCatalogTypeMismatchSnafu, TableEngineNotFoundSnafu, TableExistsSnafu, TableNotExistSnafu,
-    TableNotFoundSnafu,
+    TableNotFoundSnafu, UnimplementedSnafu,
 };
-use crate::local::memory::{MemoryCatalogManager, MemoryCatalogProvider, MemorySchemaProvider};
+use crate::local::memory::MemoryCatalogManager;
 use crate::system::{
     decode_system_catalog, Entry, SystemCatalogTable, TableEntry, ENTRY_TYPE_INDEX, KEY_INDEX,
     VALUE_INDEX,
 };
 use crate::tables::SystemCatalog;
 use crate::{
-    handle_system_table_request, CatalogManager, CatalogProviderRef, DeregisterTableRequest,
+    handle_system_table_request, CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest,
     RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
-    SchemaProviderRef,
 };
 
 /// A `CatalogManager` consists of a system catalog and a bunch of user catalogs.
@@ -74,11 +74,11 @@ impl LocalCatalogManager {
                 engine_name: MITO_ENGINE,
             })?;
         let table = SystemCatalogTable::new(engine.clone()).await?;
-        let memory_catalog_list = crate::local::memory::new_memory_catalog_list()?;
+        let memory_catalog_manager = crate::local::memory::new_memory_catalog_manager()?;
         let system_catalog = Arc::new(SystemCatalog::new(table));
         Ok(Self {
             system: system_catalog,
-            catalogs: memory_catalog_list,
+            catalogs: memory_catalog_manager,
             engine_manager,
             next_table_id: AtomicU32::new(MIN_USER_TABLE_ID),
             init_lock: Mutex::new(false),
@@ -116,26 +116,43 @@ impl LocalCatalogManager {
     }
 
     async fn init_system_catalog(&self) -> Result<()> {
-        let system_schema = Arc::new(MemorySchemaProvider::new());
-        system_schema.register_table_sync(
-            SYSTEM_CATALOG_TABLE_NAME.to_string(),
-            self.system.information_schema.system.clone(),
-        )?;
-        let system_catalog = Arc::new(MemoryCatalogProvider::new());
-        system_catalog.register_schema_sync(INFORMATION_SCHEMA_NAME.to_string(), system_schema)?;
+        // register default catalog and default schema
         self.catalogs
-            .register_catalog_sync(SYSTEM_CATALOG_NAME.to_string(), system_catalog)?;
+            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())?;
+        self.catalogs.register_schema_sync(RegisterSchemaRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+        })?;
 
-        let default_catalog = Arc::new(MemoryCatalogProvider::new());
-        let default_schema = Arc::new(MemorySchemaProvider::new());
+        // register SystemCatalogTable
+        self.catalogs
+            .register_catalog_sync(SYSTEM_CATALOG_NAME.to_string())?;
+        self.catalogs.register_schema_sync(RegisterSchemaRequest {
+            catalog: SYSTEM_CATALOG_NAME.to_string(),
+            schema: INFORMATION_SCHEMA_NAME.to_string(),
+        })?;
+        let register_table_req = RegisterTableRequest {
+            catalog: SYSTEM_CATALOG_NAME.to_string(),
+            schema: INFORMATION_SCHEMA_NAME.to_string(),
+            table_name: SYSTEM_CATALOG_TABLE_NAME.to_string(),
+            table_id: SYSTEM_CATALOG_TABLE_ID,
+            table: self.system.information_schema.system.as_table_ref(),
+        };
+        self.catalogs.register_table(register_table_req).await?;
 
         // Add numbers table for test
-        let table = Arc::new(NumbersTable::default());
-        default_schema.register_table_sync("numbers".to_string(), table)?;
+        let register_number_table_req = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: NUMBERS_TABLE_NAME.to_string(),
+            table_id: NUMBERS_TABLE_ID,
+            table: NumbersTable::table(NUMBERS_TABLE_ID),
+        };
 
-        default_catalog.register_schema_sync(DEFAULT_SCHEMA_NAME.to_string(), default_schema)?;
         self.catalogs
-            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string(), default_catalog)?;
+            .register_table(register_number_table_req)
+            .await?;
+
         Ok(())
     }
 
@@ -207,30 +224,25 @@ impl LocalCatalogManager {
         for entry in entries {
             match entry {
                 Entry::Catalog(c) => {
-                    self.catalogs.register_catalog_if_absent(
-                        c.catalog_name.clone(),
-                        Arc::new(MemoryCatalogProvider::new()),
-                    );
+                    self.catalogs
+                        .register_catalog_sync(c.catalog_name.clone())?;
                     info!("Register catalog: {}", c.catalog_name);
                 }
                 Entry::Schema(s) => {
-                    self.catalogs
-                        .catalog(&s.catalog_name)
-                        .await?
-                        .context(CatalogNotFoundSnafu {
-                            catalog_name: &s.catalog_name,
-                        })?
-                        .register_schema(
-                            s.schema_name.clone(),
-                            Arc::new(MemorySchemaProvider::new()),
-                        )
-                        .await?;
+                    let req = RegisterSchemaRequest {
+                        catalog: s.catalog_name.clone(),
+                        schema: s.schema_name.clone(),
+                    };
+                    let _ = self.catalogs.register_schema_sync(req)?;
                     info!("Registered schema: {:?}", s);
                 }
                 Entry::Table(t) => {
+                    max_table_id = max_table_id.max(t.table_id);
+                    if t.is_deleted {
+                        continue;
+                    }
                     self.open_and_register_table(&t).await?;
                     info!("Registered table: {:?}", t);
-                    max_table_id = max_table_id.max(t.table_id);
                 }
             }
         }
@@ -245,23 +257,11 @@ impl LocalCatalogManager {
     }
 
     async fn open_and_register_table(&self, t: &TableEntry) -> Result<()> {
-        let catalog =
-            self.catalogs
-                .catalog(&t.catalog_name)
-                .await?
-                .context(CatalogNotFoundSnafu {
-                    catalog_name: &t.catalog_name,
-                })?;
-        let schema = catalog
-            .schema(&t.schema_name)
-            .await?
-            .context(SchemaNotFoundSnafu {
-                catalog: &t.catalog_name,
-                schema: &t.schema_name,
-            })?;
+        self.check_catalog_schema_exist(&t.catalog_name, &t.schema_name)
+            .await?;
 
         let context = EngineContext {};
-        let request = OpenTableRequest {
+        let open_request = OpenTableRequest {
             catalog_name: t.catalog_name.clone(),
             schema_name: t.schema_name.clone(),
             table_name: t.table_name.clone(),
@@ -275,8 +275,8 @@ impl LocalCatalogManager {
                 engine_name: &t.engine,
             })?;
 
-        let option = engine
-            .open_table(&context, request)
+        let table_ref = engine
+            .open_table(&context, open_request)
             .await
             .with_context(|_| OpenTableSnafu {
                 table_info: format!(
@@ -291,7 +291,48 @@ impl LocalCatalogManager {
                 ),
             })?;
 
-        schema.register_table(t.table_name.clone(), option).await?;
+        let register_request = RegisterTableRequest {
+            catalog: t.catalog_name.clone(),
+            schema: t.schema_name.clone(),
+            table_name: t.table_name.clone(),
+            table_id: t.table_id,
+            table: table_ref,
+        };
+        let _ = self.catalogs.register_table(register_request).await?;
+
+        Ok(())
+    }
+
+    async fn check_state(&self) -> Result<()> {
+        let started = self.init_lock.lock().await;
+        ensure!(
+            *started,
+            IllegalManagerStateSnafu {
+                msg: "Catalog manager not started",
+            }
+        );
+        Ok(())
+    }
+
+    async fn check_catalog_schema_exist(
+        &self,
+        catalog_name: &str,
+        schema_name: &str,
+    ) -> Result<()> {
+        if !self.catalogs.catalog_exist(catalog_name).await? {
+            return CatalogNotFoundSnafu { catalog_name }.fail()?;
+        }
+        if !self
+            .catalogs
+            .schema_exist(catalog_name, schema_name)
+            .await?
+        {
+            return SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            }
+            .fail()?;
+        }
         Ok(())
     }
 }
@@ -312,34 +353,21 @@ impl CatalogManager for LocalCatalogManager {
     }
 
     async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        let started = self.init_lock.lock().await;
+        self.check_state().await?;
 
-        ensure!(
-            *started,
-            IllegalManagerStateSnafu {
-                msg: "Catalog manager not started",
-            }
-        );
+        let catalog_name = request.catalog.clone();
+        let schema_name = request.schema.clone();
 
-        let catalog_name = &request.catalog;
-        let schema_name = &request.schema;
-
-        let catalog = self
-            .catalogs
-            .catalog(catalog_name)
-            .await?
-            .context(CatalogNotFoundSnafu { catalog_name })?;
-        let schema = catalog
-            .schema(schema_name)
-            .await?
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: catalog_name,
-                schema: schema_name,
-            })?;
+        self.check_catalog_schema_exist(&catalog_name, &schema_name)
+            .await?;
 
         {
             let _lock = self.register_lock.lock().await;
-            if let Some(existing) = schema.table(&request.table_name).await? {
+            if let Some(existing) = self
+                .catalogs
+                .table(&request.catalog, &request.schema, &request.table_name)
+                .await?
+            {
                 if existing.table_info().ident.table_id != request.table_id {
                     error!(
                         "Unexpected table register request: {:?}, existing: {:?}",
@@ -348,8 +376,8 @@ impl CatalogManager for LocalCatalogManager {
                     );
                     return TableExistsSnafu {
                         table: format_full_table_name(
-                            catalog_name,
-                            schema_name,
+                            &catalog_name,
+                            &schema_name,
                             &request.table_name,
                         ),
                     }
@@ -358,24 +386,25 @@ impl CatalogManager for LocalCatalogManager {
                 // Try to register table with same table id, just ignore.
                 Ok(false)
             } else {
-                let engine = request.table.table_info().meta.engine.to_string();
                 // table does not exist
-                self.system
+                let engine = request.table.table_info().meta.engine.to_string();
+                let table_name = request.table_name.clone();
+                let table_id = request.table_id;
+                let _ = self.catalogs.register_table(request).await?;
+                let _ = self
+                    .system
                     .register_table(
                         catalog_name.clone(),
                         schema_name.clone(),
-                        request.table_name.clone(),
-                        request.table_id,
+                        table_name,
+                        table_id,
                         engine,
                     )
                     .await?;
-                schema
-                    .register_table(request.table_name, request.table)
-                    .await?;
                 increment_gauge!(
                     crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
                     1.0,
-                    &[crate::metrics::db_label(catalog_name, schema_name)],
+                    &[crate::metrics::db_label(&catalog_name, &schema_name)],
                 );
                 Ok(true)
             }
@@ -383,41 +412,27 @@ impl CatalogManager for LocalCatalogManager {
     }
 
     async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        let started = self.init_lock.lock().await;
-
-        ensure!(
-            *started,
-            IllegalManagerStateSnafu {
-                msg: "Catalog manager not started",
-            }
-        );
+        self.check_state().await?;
 
         let catalog_name = &request.catalog;
         let schema_name = &request.schema;
 
-        let catalog = self
-            .catalogs
-            .catalog(catalog_name)
-            .await?
-            .context(CatalogNotFoundSnafu { catalog_name })?;
-
-        let schema = catalog
-            .schema(schema_name)
-            .await?
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: catalog_name,
-                schema: schema_name,
-            })?;
-
-        let _lock = self.register_lock.lock().await;
+        self.check_catalog_schema_exist(catalog_name, schema_name)
+            .await?;
         ensure!(
-            !schema.table_exist(&request.new_table_name).await?,
+            self.catalogs
+                .table(catalog_name, schema_name, &request.new_table_name)
+                .await?
+                .is_none(),
             TableExistsSnafu {
                 table: &request.new_table_name
             }
         );
-        let old_table = schema
-            .table(&request.table_name)
+
+        let _lock = self.register_lock.lock().await;
+        let old_table = self
+            .catalogs
+            .table(catalog_name, schema_name, &request.table_name)
             .await?
             .context(TableNotExistSnafu {
                 table: &request.table_name,
@@ -425,7 +440,8 @@ impl CatalogManager for LocalCatalogManager {
 
         let engine = old_table.table_info().meta.engine.to_string();
         // rename table in system catalog
-        self.system
+        let _ = self
+            .system
             .register_table(
                 catalog_name.clone(),
                 schema_name.clone(),
@@ -435,18 +451,11 @@ impl CatalogManager for LocalCatalogManager {
             )
             .await?;
 
-        let renamed = schema
-            .rename_table(&request.table_name, request.new_table_name.clone())
-            .await
-            .is_ok();
-        Ok(renamed)
+        self.catalogs.rename_table(request).await
     }
 
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool> {
-        {
-            let started = *self.init_lock.lock().await;
-            ensure!(started, IllegalManagerStateSnafu { msg: "not started" });
-        }
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
+        self.check_state().await?;
 
         {
             let _ = self.register_lock.lock().await;
@@ -467,61 +476,51 @@ impl CatalogManager for LocalCatalogManager {
                 .ident
                 .table_id;
 
-            if !self.system.deregister_table(&request, table_id).await? {
-                return Ok(false);
-            }
-
+            self.system.deregister_table(&request, table_id).await?;
             self.catalogs.deregister_table(request).await
         }
     }
 
     async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        let started = self.init_lock.lock().await;
-        ensure!(
-            *started,
-            IllegalManagerStateSnafu {
-                msg: "Catalog manager not started",
-            }
-        );
+        self.check_state().await?;
+
         let catalog_name = &request.catalog;
         let schema_name = &request.schema;
 
-        let catalog = self
-            .catalogs
-            .catalog(catalog_name)
-            .await?
-            .context(CatalogNotFoundSnafu { catalog_name })?;
+        if !self.catalogs.catalog_exist(catalog_name).await? {
+            return CatalogNotFoundSnafu { catalog_name }.fail()?;
+        }
 
         {
             let _lock = self.register_lock.lock().await;
             ensure!(
-                catalog.schema(schema_name).await?.is_none(),
+                !self
+                    .catalogs
+                    .schema_exist(catalog_name, schema_name)
+                    .await?,
                 SchemaExistsSnafu {
                     schema: schema_name,
                 }
             );
-            self.system
-                .register_schema(request.catalog, schema_name.clone())
+            let _ = self
+                .system
+                .register_schema(request.catalog.clone(), schema_name.clone())
                 .await?;
-            catalog
-                .register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))
-                .await?;
-
-            Ok(true)
+            self.catalogs.register_schema_sync(request)
         }
     }
 
+    async fn deregister_schema(&self, _request: DeregisterSchemaRequest) -> Result<bool> {
+        UnimplementedSnafu {
+            operation: "deregister schema",
+        }
+        .fail()
+    }
+
     async fn register_system_table(&self, request: RegisterSystemTableRequest) -> Result<()> {
         let catalog_name = request.create_table_request.catalog_name.clone();
         let schema_name = request.create_table_request.schema_name.clone();
 
-        ensure!(
-            !*self.init_lock.lock().await,
-            IllegalManagerStateSnafu {
-                msg: "Catalog manager already started",
-            }
-        );
-
         let mut sys_table_requests = self.system_table_requests.lock().await;
         sys_table_requests.push(request);
         increment_gauge!(
@@ -532,15 +531,8 @@ impl CatalogManager for LocalCatalogManager {
         Ok(())
     }
 
-    async fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
-        self.catalogs
-            .catalog(catalog)
-            .await?
-            .context(CatalogNotFoundSnafu {
-                catalog_name: catalog,
-            })?
-            .schema(schema)
-            .await
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
+        self.catalogs.schema_exist(catalog, schema).await
     }
 
     async fn table(
@@ -549,39 +541,37 @@ impl CatalogManager for LocalCatalogManager {
         schema_name: &str,
         table_name: &str,
     ) -> Result<Option<TableRef>> {
-        let catalog = self
-            .catalogs
-            .catalog(catalog_name)
-            .await?
-            .context(CatalogNotFoundSnafu { catalog_name })?;
-        let schema = catalog
-            .schema(schema_name)
-            .await?
-            .with_context(|| SchemaNotFoundSnafu {
-                catalog: catalog_name,
-                schema: schema_name,
-            })?;
-        schema.table(table_name).await
+        self.catalogs
+            .table(catalog_name, schema_name, table_name)
+            .await
     }
 
-    async fn catalog(&self, catalog: &str) -> Result<Option<CatalogProviderRef>> {
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
         if catalog.eq_ignore_ascii_case(SYSTEM_CATALOG_NAME) {
-            Ok(Some(self.system.clone()))
+            Ok(true)
         } else {
-            self.catalogs.catalog(catalog).await
+            self.catalogs.catalog_exist(catalog).await
         }
     }
 
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+        self.catalogs.table_exist(catalog, schema, table).await
+    }
+
     async fn catalog_names(&self) -> Result<Vec<String>> {
         self.catalogs.catalog_names().await
     }
 
-    async fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Result<Option<CatalogProviderRef>> {
-        self.catalogs.register_catalog(name, catalog).await
+    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
+        self.catalogs.schema_names(catalog_name).await
+    }
+
+    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
+        self.catalogs.table_names(catalog_name, schema_name).await
+    }
+
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
+        self.catalogs.clone().register_catalog(name).await
     }
 
     fn as_any(&self) -> &dyn Any {
@@ -607,6 +597,7 @@ mod tests {
                 table_name: "T1".to_string(),
                 table_id: 1,
                 engine: MITO_ENGINE.to_string(),
+                is_deleted: false,
             }),
             Entry::Catalog(CatalogEntry {
                 catalog_name: "C2".to_string(),
@@ -628,6 +619,7 @@ mod tests {
                 table_name: "T2".to_string(),
                 table_id: 2,
                 engine: MITO_ENGINE.to_string(),
+                is_deleted: false,
             }),
         ];
         let res = LocalCatalogManager::sort_entries(vec);

src/catalog/src/local/memory.rs

@@ -16,51 +16,35 @@ use std::any::Any;
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::{Arc, RwLock};
+use std::sync::{Arc, RwLock, Weak};
 
-use async_trait::async_trait;
-use common_catalog::consts::MIN_USER_TABLE_ID;
-use common_telemetry::error;
+use common_catalog::consts::{
+    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MIN_USER_TABLE_ID,
+};
 use metrics::{decrement_gauge, increment_gauge};
-use snafu::{ensure, OptionExt};
+use snafu::OptionExt;
 use table::metadata::TableId;
 use table::table::TableIdProvider;
 use table::TableRef;
 
 use crate::error::{
-    self, CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, TableNotFoundSnafu,
+    CatalogNotFoundSnafu, Result, SchemaNotFoundSnafu, TableExistsSnafu, TableNotFoundSnafu,
 };
-use crate::schema::SchemaProvider;
+use crate::information_schema::InformationSchemaProvider;
 use crate::{
-    CatalogManager, CatalogProvider, CatalogProviderRef, DeregisterTableRequest,
-    RegisterSchemaRequest, RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
-    SchemaProviderRef,
+    CatalogManager, DeregisterSchemaRequest, DeregisterTableRequest, RegisterSchemaRequest,
+    RegisterSystemTableRequest, RegisterTableRequest, RenameTableRequest,
 };
 
+type SchemaEntries = HashMap<String, HashMap<String, TableRef>>;
+
 /// Simple in-memory list of catalogs
 pub struct MemoryCatalogManager {
     /// Collection of catalogs containing schemas and ultimately Tables
-    pub catalogs: RwLock<HashMap<String, CatalogProviderRef>>,
+    pub catalogs: RwLock<HashMap<String, SchemaEntries>>,
     pub table_id: AtomicU32,
 }
 
-impl Default for MemoryCatalogManager {
-    fn default() -> Self {
-        let manager = Self {
-            table_id: AtomicU32::new(MIN_USER_TABLE_ID),
-            catalogs: Default::default(),
-        };
-        let default_catalog = Arc::new(MemoryCatalogProvider::new());
-        manager
-            .register_catalog_sync("greptime".to_string(), default_catalog.clone())
-            .unwrap();
-        default_catalog
-            .register_schema_sync("public".to_string(), Arc::new(MemorySchemaProvider::new()))
-            .unwrap();
-        manager
-    }
-}
-
 #[async_trait::async_trait]
 impl TableIdProvider for MemoryCatalogManager {
     async fn next_table_id(&self) -> table::error::Result<TableId> {
@@ -76,81 +60,75 @@ impl CatalogManager for MemoryCatalogManager {
     }
 
     async fn register_table(&self, request: RegisterTableRequest) -> Result<bool> {
-        let schema = self
-            .catalog(&request.catalog)
-            .context(CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?
-            .schema(&request.schema)
-            .await?
-            .context(SchemaNotFoundSnafu {
-                catalog: &request.catalog,
-                schema: &request.schema,
-            })?;
-        increment_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
-        schema
-            .register_table(request.table_name, request.table)
-            .await
-            .map(|v| v.is_none())
+        self.register_table_sync(request)
     }
 
     async fn rename_table(&self, request: RenameTableRequest) -> Result<bool> {
-        let catalog = self
-            .catalog(&request.catalog)
-            .context(CatalogNotFoundSnafu {
-                catalog_name: &request.catalog,
-            })?;
-        let schema =
-            catalog
-                .schema(&request.schema)
-                .await?
-                .with_context(|| SchemaNotFoundSnafu {
-                    catalog: &request.catalog,
-                    schema: &request.schema,
-                })?;
-        Ok(schema
-            .rename_table(&request.table_name, request.new_table_name)
-            .await
-            .is_ok())
-    }
-
-    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<bool> {
-        let schema = self
-            .catalog(&request.catalog)
-            .context(CatalogNotFoundSnafu {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
                 catalog_name: &request.catalog,
             })?
-            .schema(&request.schema)
-            .await?
+            .get_mut(&request.schema)
             .with_context(|| SchemaNotFoundSnafu {
                 catalog: &request.catalog,
                 schema: &request.schema,
             })?;
-        decrement_gauge!(
-            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
-            1.0,
-            &[crate::metrics::db_label(&request.catalog, &request.schema)],
-        );
-        schema
-            .deregister_table(&request.table_name)
-            .await
-            .map(|v| v.is_some())
+
+        // check old and new table names
+        if !schema.contains_key(&request.table_name) {
+            return TableNotFoundSnafu {
+                table_info: request.table_name.to_string(),
+            }
+            .fail()?;
+        }
+        if schema.contains_key(&request.new_table_name) {
+            return TableExistsSnafu {
+                table: &request.new_table_name,
+            }
+            .fail();
+        }
+
+        let table = schema.remove(&request.table_name).unwrap();
+        let _ = schema.insert(request.new_table_name, table);
+
+        Ok(true)
+    }
+
+    async fn deregister_table(&self, request: DeregisterTableRequest) -> Result<()> {
+        self.deregister_table_sync(request)
     }
 
     async fn register_schema(&self, request: RegisterSchemaRequest) -> Result<bool> {
-        let catalog = self
-            .catalog(&request.catalog)
-            .context(CatalogNotFoundSnafu {
+        self.register_schema_sync(request)
+    }
+
+    async fn deregister_schema(&self, request: DeregisterSchemaRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schemas = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
                 catalog_name: &request.catalog,
             })?;
-        catalog
-            .register_schema(request.schema, Arc::new(MemorySchemaProvider::new()))
-            .await?;
-        increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
+        let table_count = schemas
+            .remove(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?
+            .len();
+        decrement_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            table_count as f64,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
+
+        decrement_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
+        );
         Ok(true)
     }
 
@@ -159,12 +137,8 @@ impl CatalogManager for MemoryCatalogManager {
         Ok(())
     }
 
-    async fn schema(&self, catalog: &str, schema: &str) -> Result<Option<SchemaProviderRef>> {
-        if let Some(c) = self.catalog(catalog) {
-            c.schema(schema).await
-        } else {
-            Ok(None)
-        }
+    async fn schema_exist(&self, catalog: &str, schema: &str) -> Result<bool> {
+        self.schema_exist_sync(catalog, schema)
     }
 
     async fn table(
@@ -173,27 +147,69 @@ impl CatalogManager for MemoryCatalogManager {
         schema: &str,
         table_name: &str,
     ) -> Result<Option<TableRef>> {
-        let Some(catalog) = self
-            .catalog(catalog) else { return Ok(None)};
-        let Some(s) = catalog.schema(schema).await? else { return Ok(None) };
-        s.table(table_name).await
+        let result = try {
+            self.catalogs
+                .read()
+                .unwrap()
+                .get(catalog)?
+                .get(schema)?
+                .get(table_name)
+                .cloned()?
+        };
+        Ok(result)
     }
 
-    async fn catalog(&self, catalog: &str) -> Result<Option<CatalogProviderRef>> {
-        Ok(self.catalogs.read().unwrap().get(catalog).cloned())
+    async fn catalog_exist(&self, catalog: &str) -> Result<bool> {
+        self.catalog_exist_sync(catalog)
+    }
+
+    async fn table_exist(&self, catalog: &str, schema: &str, table: &str) -> Result<bool> {
+        let catalogs = self.catalogs.read().unwrap();
+        Ok(catalogs
+            .get(catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .get(schema)
+            .with_context(|| SchemaNotFoundSnafu { catalog, schema })?
+            .contains_key(table))
     }
 
     async fn catalog_names(&self) -> Result<Vec<String>> {
         Ok(self.catalogs.read().unwrap().keys().cloned().collect())
     }
 
-    async fn register_catalog(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Result<Option<CatalogProviderRef>> {
-        increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
-        self.register_catalog_sync(name, catalog)
+    async fn schema_names(&self, catalog_name: &str) -> Result<Vec<String>> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog_name)
+            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
+            .keys()
+            .cloned()
+            .collect())
+    }
+
+    async fn table_names(&self, catalog_name: &str, schema_name: &str) -> Result<Vec<String>> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog_name)
+            .with_context(|| CatalogNotFoundSnafu { catalog_name })?
+            .get(schema_name)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: catalog_name,
+                schema: schema_name,
+            })?
+            .keys()
+            .cloned()
+            .collect())
+    }
+
+    async fn register_catalog(self: Arc<Self>, name: String) -> Result<bool> {
+        self.register_catalog_sync(name)
     }
 
     fn as_any(&self) -> &dyn Any {
@@ -202,313 +218,288 @@ impl CatalogManager for MemoryCatalogManager {
 }
 
 impl MemoryCatalogManager {
-    /// Registers a catalog and return `None` if no catalog with the same name was already
-    /// registered, or `Some` with the previously registered catalog.
-    pub fn register_catalog_if_absent(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Option<CatalogProviderRef> {
+    /// Creates a manager with some default setups
+    /// (e.g. default catalog/schema and information schema)
+    pub fn with_default_setup() -> Arc<Self> {
+        let manager = Arc::new(Self {
+            table_id: AtomicU32::new(MIN_USER_TABLE_ID),
+            catalogs: Default::default(),
+        });
+
+        // Safety: default catalog/schema is registered in order so no CatalogNotFound error will occur
+        manager
+            .register_catalog_sync(DEFAULT_CATALOG_NAME.to_string())
+            .unwrap();
+        manager
+            .register_schema_sync(RegisterSchemaRequest {
+                catalog: DEFAULT_CATALOG_NAME.to_string(),
+                schema: DEFAULT_SCHEMA_NAME.to_string(),
+            })
+            .unwrap();
+
+        manager
+    }
+
+    fn schema_exist_sync(&self, catalog: &str, schema: &str) -> Result<bool> {
+        Ok(self
+            .catalogs
+            .read()
+            .unwrap()
+            .get(catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: catalog,
+            })?
+            .contains_key(schema))
+    }
+
+    fn catalog_exist_sync(&self, catalog: &str) -> Result<bool> {
+        Ok(self.catalogs.read().unwrap().get(catalog).is_some())
+    }
+
+    /// Registers a catalog if it does not exist and returns false if the schema exists.
+    pub fn register_catalog_sync(self: &Arc<Self>, name: String) -> Result<bool> {
         let mut catalogs = self.catalogs.write().unwrap();
-        let entry = catalogs.entry(name);
-        match entry {
-            Entry::Occupied(v) => Some(v.get().clone()),
-            Entry::Vacant(v) => {
-                v.insert(catalog);
-                None
+
+        match catalogs.entry(name.clone()) {
+            Entry::Vacant(e) => {
+                let catalog = self.create_catalog_entry(name);
+                e.insert(catalog);
+                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_CATALOG_COUNT, 1.0);
+                Ok(true)
             }
+            Entry::Occupied(_) => Ok(false),
         }
     }
 
-    pub fn register_catalog_sync(
-        &self,
-        name: String,
-        catalog: CatalogProviderRef,
-    ) -> Result<Option<CatalogProviderRef>> {
+    pub fn deregister_table_sync(&self, request: DeregisterTableRequest) -> Result<()> {
         let mut catalogs = self.catalogs.write().unwrap();
-        Ok(catalogs.insert(name, catalog))
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .get_mut(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
+        let result = schema.remove(&request.table_name);
+        if result.is_some() {
+            decrement_gauge!(
+                crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+                1.0,
+                &[crate::metrics::db_label(&request.catalog, &request.schema)],
+            );
+        }
+        Ok(())
     }
 
-    fn catalog(&self, catalog_name: &str) -> Option<CatalogProviderRef> {
-        self.catalogs.read().unwrap().get(catalog_name).cloned()
-    }
-}
+    /// Registers a schema if it does not exist.
+    /// It returns an error if the catalog does not exist,
+    /// and returns false if the schema exists.
+    pub fn register_schema_sync(&self, request: RegisterSchemaRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let catalog = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?;
 
-impl Default for MemoryCatalogProvider {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-/// Simple in-memory implementation of a catalog.
-pub struct MemoryCatalogProvider {
-    schemas: RwLock<HashMap<String, Arc<dyn SchemaProvider>>>,
-}
-
-impl MemoryCatalogProvider {
-    /// Instantiates a new MemoryCatalogProvider with an empty collection of schemas.
-    pub fn new() -> Self {
-        Self {
-            schemas: RwLock::new(HashMap::new()),
+        match catalog.entry(request.schema) {
+            Entry::Vacant(e) => {
+                e.insert(HashMap::new());
+                increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
+                Ok(true)
+            }
+            Entry::Occupied(_) => Ok(false),
         }
     }
 
-    pub fn schema_names_sync(&self) -> Result<Vec<String>> {
-        let schemas = self.schemas.read().unwrap();
-        Ok(schemas.keys().cloned().collect())
-    }
+    /// Registers a schema and returns an error if the catalog or schema does not exist.
+    pub fn register_table_sync(&self, request: RegisterTableRequest) -> Result<bool> {
+        let mut catalogs = self.catalogs.write().unwrap();
+        let schema = catalogs
+            .get_mut(&request.catalog)
+            .with_context(|| CatalogNotFoundSnafu {
+                catalog_name: &request.catalog,
+            })?
+            .get_mut(&request.schema)
+            .with_context(|| SchemaNotFoundSnafu {
+                catalog: &request.catalog,
+                schema: &request.schema,
+            })?;
 
-    pub fn register_schema_sync(
-        &self,
-        name: String,
-        schema: SchemaProviderRef,
-    ) -> Result<Option<SchemaProviderRef>> {
-        let mut schemas = self.schemas.write().unwrap();
-        ensure!(
-            !schemas.contains_key(&name),
-            error::SchemaExistsSnafu { schema: &name }
+        if schema.contains_key(&request.table_name) {
+            return TableExistsSnafu {
+                table: &request.table_name,
+            }
+            .fail();
+        }
+        schema.insert(request.table_name, request.table);
+        increment_gauge!(
+            crate::metrics::METRIC_CATALOG_MANAGER_TABLE_COUNT,
+            1.0,
+            &[crate::metrics::db_label(&request.catalog, &request.schema)],
         );
-        increment_gauge!(crate::metrics::METRIC_CATALOG_MANAGER_SCHEMA_COUNT, 1.0);
-        Ok(schemas.insert(name, schema))
+        Ok(true)
     }
 
-    pub fn schema_sync(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
-        let schemas = self.schemas.read().unwrap();
-        Ok(schemas.get(name).cloned())
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogProvider for MemoryCatalogProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
+    fn create_catalog_entry(self: &Arc<Self>, catalog: String) -> SchemaEntries {
+        let information_schema = InformationSchemaProvider::build(
+            catalog,
+            Arc::downgrade(self) as Weak<dyn CatalogManager>,
+        );
+        let mut catalog = HashMap::new();
+        catalog.insert(INFORMATION_SCHEMA_NAME.to_string(), information_schema);
+        catalog
     }
 
-    async fn schema_names(&self) -> Result<Vec<String>> {
-        self.schema_names_sync()
-    }
+    #[cfg(any(test, feature = "testing"))]
+    pub fn new_with_table(table: TableRef) -> Arc<Self> {
+        let manager = Self::with_default_setup();
+        let catalog = &table.table_info().catalog_name;
+        let schema = &table.table_info().schema_name;
 
-    async fn register_schema(
-        &self,
-        name: String,
-        schema: SchemaProviderRef,
-    ) -> Result<Option<SchemaProviderRef>> {
-        self.register_schema_sync(name, schema)
-    }
-
-    async fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>> {
-        self.schema_sync(name)
-    }
-}
-
-/// Simple in-memory implementation of a schema.
-pub struct MemorySchemaProvider {
-    tables: RwLock<HashMap<String, TableRef>>,
-}
-
-impl MemorySchemaProvider {
-    /// Instantiates a new MemorySchemaProvider with an empty collection of tables.
-    pub fn new() -> Self {
-        Self {
-            tables: RwLock::new(HashMap::new()),
+        if !manager.catalog_exist_sync(catalog).unwrap() {
+            manager.register_catalog_sync(catalog.to_string()).unwrap();
         }
-    }
 
-    pub fn register_table_sync(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
-        let mut tables = self.tables.write().unwrap();
-        if let Some(existing) = tables.get(name.as_str()) {
-            // if table with the same name but different table id exists, then it's a fatal bug
-            if existing.table_info().ident.table_id != table.table_info().ident.table_id {
-                error!(
-                    "Unexpected table register: {:?}, existing: {:?}",
-                    table.table_info(),
-                    existing.table_info()
-                );
-                return TableExistsSnafu { table: name }.fail()?;
-            }
-            Ok(Some(existing.clone()))
-        } else {
-            Ok(tables.insert(name, table))
+        if !manager.schema_exist_sync(catalog, schema).unwrap() {
+            manager
+                .register_schema_sync(RegisterSchemaRequest {
+                    catalog: catalog.to_string(),
+                    schema: schema.to_string(),
+                })
+                .unwrap();
         }
-    }
 
-    pub fn rename_table_sync(&self, name: &str, new_name: String) -> Result<TableRef> {
-        let mut tables = self.tables.write().unwrap();
-        let Some(table) = tables.remove(name) else {
-            return TableNotFoundSnafu {
-                table_info: name.to_string(),
-            }
-                .fail()?;
+        let request = RegisterTableRequest {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table_name: table.table_info().name.clone(),
+            table_id: table.table_info().ident.table_id,
+            table,
         };
-        let e = match tables.entry(new_name) {
-            Entry::Vacant(e) => e,
-            Entry::Occupied(e) => {
-                return TableExistsSnafu { table: e.key() }.fail();
-            }
-        };
-        e.insert(table.clone());
-        Ok(table)
-    }
-
-    pub fn table_exist_sync(&self, name: &str) -> Result<bool> {
-        let tables = self.tables.read().unwrap();
-        Ok(tables.contains_key(name))
-    }
-
-    pub fn deregister_table_sync(&self, name: &str) -> Result<Option<TableRef>> {
-        let mut tables = self.tables.write().unwrap();
-        Ok(tables.remove(name))
-    }
-}
-
-impl Default for MemorySchemaProvider {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-#[async_trait]
-impl SchemaProvider for MemorySchemaProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    async fn table_names(&self) -> Result<Vec<String>> {
-        let tables = self.tables.read().unwrap();
-        Ok(tables.keys().cloned().collect())
-    }
-
-    async fn table(&self, name: &str) -> Result<Option<TableRef>> {
-        let tables = self.tables.read().unwrap();
-        Ok(tables.get(name).cloned())
-    }
-
-    async fn register_table(&self, name: String, table: TableRef) -> Result<Option<TableRef>> {
-        self.register_table_sync(name, table)
-    }
-
-    async fn rename_table(&self, name: &str, new_name: String) -> Result<TableRef> {
-        self.rename_table_sync(name, new_name)
-    }
-
-    async fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
-        self.deregister_table_sync(name)
-    }
-
-    async fn table_exist(&self, name: &str) -> Result<bool> {
-        self.table_exist_sync(name)
+        let _ = manager.register_table_sync(request).unwrap();
+        manager
     }
 }
 
 /// Create a memory catalog list contains a numbers table for test
-pub fn new_memory_catalog_list() -> Result<Arc<MemoryCatalogManager>> {
-    Ok(Arc::new(MemoryCatalogManager::default()))
+pub fn new_memory_catalog_manager() -> Result<Arc<MemoryCatalogManager>> {
+    Ok(MemoryCatalogManager::with_default_setup())
 }
 
 #[cfg(test)]
 mod tests {
     use common_catalog::consts::*;
     use common_error::ext::ErrorExt;
-    use common_error::prelude::StatusCode;
-    use table::table::numbers::NumbersTable;
+    use common_error::status_code::StatusCode;
+    use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
 
     use super::*;
 
     #[tokio::test]
     async fn test_new_memory_catalog_list() {
-        let catalog_list = new_memory_catalog_list().unwrap();
-        let default_catalog = CatalogManager::catalog(&*catalog_list, DEFAULT_CATALOG_NAME)
-            .await
-            .unwrap()
-            .unwrap();
+        let catalog_list = new_memory_catalog_manager().unwrap();
 
-        let default_schema = default_catalog
-            .schema(DEFAULT_SCHEMA_NAME)
-            .await
-            .unwrap()
-            .unwrap();
+        let register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: NUMBERS_TABLE_NAME.to_string(),
+            table_id: NUMBERS_TABLE_ID,
+            table: NumbersTable::table(NUMBERS_TABLE_ID),
+        };
 
-        default_schema
-            .register_table("numbers".to_string(), Arc::new(NumbersTable::default()))
+        let _ = catalog_list.register_table(register_request).await.unwrap();
+        let table = catalog_list
+            .table(
+                DEFAULT_CATALOG_NAME,
+                DEFAULT_SCHEMA_NAME,
+                NUMBERS_TABLE_NAME,
+            )
             .await
             .unwrap();
-
-        let table = default_schema.table("numbers").await.unwrap();
-        assert!(table.is_some());
-        assert!(default_schema.table("not_exists").await.unwrap().is_none());
-    }
-
-    #[tokio::test]
-    async fn test_mem_provider() {
-        let provider = MemorySchemaProvider::new();
-        let table_name = "numbers";
-        assert!(!provider.table_exist_sync(table_name).unwrap());
-        provider.deregister_table_sync(table_name).unwrap();
-        let test_table = NumbersTable::default();
-        // register table successfully
-        assert!(provider
-            .register_table_sync(table_name.to_string(), Arc::new(test_table))
+        let _ = table.unwrap();
+        assert!(catalog_list
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "not_exists")
+            .await
             .unwrap()
             .is_none());
-        assert!(provider.table_exist_sync(table_name).unwrap());
-        let other_table = NumbersTable::new(12);
-        let result = provider.register_table_sync(table_name.to_string(), Arc::new(other_table));
-        let err = result.err().unwrap();
-        assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
     }
 
     #[tokio::test]
-    async fn test_mem_provider_rename_table() {
-        let provider = MemorySchemaProvider::new();
-        let table_name = "num";
-        assert!(!provider.table_exist_sync(table_name).unwrap());
-        let test_table: TableRef = Arc::new(NumbersTable::default());
+    async fn test_mem_manager_rename_table() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+        let table_name = "test_table";
+        assert!(!catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
         // register test table
-        assert!(provider
-            .register_table_sync(table_name.to_string(), test_table.clone())
-            .unwrap()
-            .is_none());
-        assert!(provider.table_exist_sync(table_name).unwrap());
+        let table_id = 2333;
+        let register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            table_id,
+            table: NumbersTable::table(table_id),
+        };
+        assert!(catalog.register_table(register_request).await.unwrap());
+        assert!(catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
 
         // rename test table
-        let new_table_name = "numbers";
-        provider
-            .rename_table_sync(table_name, new_table_name.to_string())
-            .unwrap();
+        let new_table_name = "test_table_renamed";
+        let rename_request = RenameTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            new_table_name: new_table_name.to_string(),
+            table_id,
+        };
+        let _ = catalog.rename_table(rename_request).await.unwrap();
 
         // test old table name not exist
-        assert!(!provider.table_exist_sync(table_name).unwrap());
-        provider.deregister_table_sync(table_name).unwrap();
+        assert!(!catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap());
 
         // test new table name exists
-        assert!(provider.table_exist_sync(new_table_name).unwrap());
-        let registered_table = provider.table(new_table_name).await.unwrap().unwrap();
-        assert_eq!(
-            registered_table.table_info().ident.table_id,
-            test_table.table_info().ident.table_id
-        );
+        assert!(catalog
+            .table_exist(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap());
+        let registered_table = catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(registered_table.table_info().ident.table_id, table_id);
 
-        let other_table = Arc::new(NumbersTable::new(2));
-        let result = provider
-            .register_table(new_table_name.to_string(), other_table)
-            .await;
+        let dup_register_request = RegisterTableRequest {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: new_table_name.to_string(),
+            table_id: table_id + 1,
+            table: NumbersTable::table(table_id + 1),
+        };
+        let result = catalog.register_table(dup_register_request).await;
         let err = result.err().unwrap();
         assert_eq!(StatusCode::TableAlreadyExists, err.status_code());
     }
 
     #[tokio::test]
     async fn test_catalog_rename_table() {
-        let catalog = MemoryCatalogManager::default();
-        let schema = catalog
-            .schema(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
-            .await
-            .unwrap()
-            .unwrap();
-
-        // register table
+        let catalog = MemoryCatalogManager::with_default_setup();
         let table_name = "num";
         let table_id = 2333;
-        let table: TableRef = Arc::new(NumbersTable::new(table_id));
+        let table = NumbersTable::table(table_id);
+
+        // register table
         let register_table_req = RegisterTableRequest {
             catalog: DEFAULT_CATALOG_NAME.to_string(),
             schema: DEFAULT_SCHEMA_NAME.to_string(),
@@ -517,7 +508,11 @@ mod tests {
             table,
         };
         assert!(catalog.register_table(register_table_req).await.unwrap());
-        assert!(schema.table_exist(table_name).await.unwrap());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_some());
 
         // rename table
         let new_table_name = "numbers_new";
@@ -529,8 +524,16 @@ mod tests {
             table_id,
         };
         assert!(catalog.rename_table(rename_table_req).await.unwrap());
-        assert!(!schema.table_exist(table_name).await.unwrap());
-        assert!(schema.table_exist(new_table_name).await.unwrap());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_none());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
+            .await
+            .unwrap()
+            .is_some());
 
         let registered_table = catalog
             .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, new_table_name)
@@ -541,52 +544,87 @@ mod tests {
     }
 
     #[test]
-    pub fn test_register_if_absent() {
-        let list = MemoryCatalogManager::default();
+    pub fn test_register_catalog_sync() {
+        let list = MemoryCatalogManager::with_default_setup();
         assert!(list
-            .register_catalog_if_absent(
-                "test_catalog".to_string(),
-                Arc::new(MemoryCatalogProvider::new())
-            )
-            .is_none());
-        list.register_catalog_if_absent(
-            "test_catalog".to_string(),
-            Arc::new(MemoryCatalogProvider::new()),
-        )
-        .unwrap();
-        list.as_any()
-            .downcast_ref::<MemoryCatalogManager>()
-            .unwrap();
+            .register_catalog_sync("test_catalog".to_string())
+            .unwrap());
+        assert!(!list
+            .register_catalog_sync("test_catalog".to_string())
+            .unwrap());
     }
 
     #[tokio::test]
     pub async fn test_catalog_deregister_table() {
-        let catalog = MemoryCatalogManager::default();
-        let schema = catalog
-            .schema(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
-            .await
-            .unwrap()
-            .unwrap();
+        let catalog = MemoryCatalogManager::with_default_setup();
+        let table_name = "foo_table";
 
         let register_table_req = RegisterTableRequest {
             catalog: DEFAULT_CATALOG_NAME.to_string(),
             schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: "numbers".to_string(),
+            table_name: table_name.to_string(),
             table_id: 2333,
-            table: Arc::new(NumbersTable::default()),
+            table: NumbersTable::table(2333),
         };
-        catalog.register_table(register_table_req).await.unwrap();
-        assert!(schema.table_exist("numbers").await.unwrap());
+        let _ = catalog.register_table(register_table_req).await.unwrap();
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_some());
 
         let deregister_table_req = DeregisterTableRequest {
             catalog: DEFAULT_CATALOG_NAME.to_string(),
             schema: DEFAULT_SCHEMA_NAME.to_string(),
-            table_name: "numbers".to_string(),
+            table_name: table_name.to_string(),
         };
         catalog
             .deregister_table(deregister_table_req)
             .await
             .unwrap();
-        assert!(!schema.table_exist("numbers").await.unwrap());
+        assert!(catalog
+            .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, table_name)
+            .await
+            .unwrap()
+            .is_none());
+    }
+
+    #[tokio::test]
+    async fn test_catalog_deregister_schema() {
+        let catalog = MemoryCatalogManager::with_default_setup();
+
+        // Registers a catalog, a schema, and a table.
+        let catalog_name = "foo_catalog".to_string();
+        let schema_name = "foo_schema".to_string();
+        let table_name = "foo_table".to_string();
+        let schema = RegisterSchemaRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+        };
+        let table = RegisterTableRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+            table_name,
+            table_id: 0,
+            table: NumbersTable::table(0),
+        };
+        catalog
+            .clone()
+            .register_catalog(catalog_name.clone())
+            .await
+            .unwrap();
+        catalog.register_schema(schema).await.unwrap();
+        catalog.register_table(table).await.unwrap();
+
+        let request = DeregisterSchemaRequest {
+            catalog: catalog_name.clone(),
+            schema: schema_name.clone(),
+        };
+
+        assert!(catalog.deregister_schema(request).await.unwrap());
+        assert!(!catalog
+            .schema_exist(&catalog_name, &schema_name)
+            .await
+            .unwrap());
     }
 }

src/catalog/src/remote.rs

@@ -12,76 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::any::Any;
-use std::fmt::Debug;
-use std::pin::Pin;
 use std::sync::Arc;
 
-pub use client::CachedMetaKvBackend;
-use futures::Stream;
-use futures_util::StreamExt;
-pub use manager::{RemoteCatalogManager, RemoteCatalogProvider, RemoteSchemaProvider};
-
-use crate::error::Error;
+pub use client::{CachedMetaKvBackend, MetaKvBackend};
+pub use manager::RemoteCatalogManager;
 
 mod client;
 mod manager;
 
 #[cfg(feature = "testing")]
 pub mod mock;
-
-#[derive(Debug, Clone)]
-pub struct Kv(pub Vec<u8>, pub Vec<u8>);
-
-pub type ValueIter<'a, E> = Pin<Box<dyn Stream<Item = Result<Kv, E>> + Send + 'a>>;
-
-#[async_trait::async_trait]
-pub trait KvBackend: Send + Sync {
-    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
-    where
-        'a: 'b;
-
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error>;
-
-    /// Compare and set value of key. `expect` is the expected value, if backend's current value associated
-    /// with key is the same as `expect`, the value will be updated to `val`.
-    ///
-    /// - If the compare-and-set operation successfully updated value, this method will return an `Ok(Ok())`
-    /// - If associated value is not the same as `expect`, no value will be updated and an `Ok(Err(Vec<u8>))`
-    /// will be returned, the `Err(Vec<u8>)` indicates the current associated value of key.
-    /// - If any error happens during operation, an `Err(Error)` will be returned.
-    async fn compare_and_set(
-        &self,
-        key: &[u8],
-        expect: &[u8],
-        val: &[u8],
-    ) -> Result<Result<(), Option<Vec<u8>>>, Error>;
-
-    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error>;
-
-    async fn delete(&self, key: &[u8]) -> Result<(), Error> {
-        self.delete_range(key, &[]).await
-    }
-
-    /// Default get is implemented based on `range` method.
-    async fn get(&self, key: &[u8]) -> Result<Option<Kv>, Error> {
-        let mut iter = self.range(key);
-        while let Some(r) = iter.next().await {
-            let kv = r?;
-            if kv.0 == key {
-                return Ok(Some(kv));
-            }
-        }
-        return Ok(None);
-    }
-
-    /// MoveValue atomically renames the key to the given updated key.
-    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<(), Error>;
-
-    fn as_any(&self) -> &dyn Any;
-}
-
-pub type KvBackendRef = Arc<dyn KvBackend>;
+pub mod region_alive_keeper;
 
 #[async_trait::async_trait]
 pub trait KvCacheInvalidator: Send + Sync {
@@ -89,71 +30,3 @@ pub trait KvCacheInvalidator: Send + Sync {
 }
 
 pub type KvCacheInvalidatorRef = Arc<dyn KvCacheInvalidator>;
-
-#[cfg(test)]
-mod tests {
-    use async_stream::stream;
-
-    use super::*;
-
-    struct MockKvBackend {}
-
-    #[async_trait::async_trait]
-    impl KvBackend for MockKvBackend {
-        fn range<'a, 'b>(&'a self, _key: &[u8]) -> ValueIter<'b, Error>
-        where
-            'a: 'b,
-        {
-            Box::pin(stream!({
-                for i in 0..3 {
-                    yield Ok(Kv(
-                        i.to_string().as_bytes().to_vec(),
-                        i.to_string().as_bytes().to_vec(),
-                    ))
-                }
-            }))
-        }
-
-        async fn set(&self, _key: &[u8], _val: &[u8]) -> Result<(), Error> {
-            unimplemented!()
-        }
-
-        async fn compare_and_set(
-            &self,
-            _key: &[u8],
-            _expect: &[u8],
-            _val: &[u8],
-        ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
-            unimplemented!()
-        }
-
-        async fn delete_range(&self, _key: &[u8], _end: &[u8]) -> Result<(), Error> {
-            unimplemented!()
-        }
-
-        async fn move_value(&self, _from_key: &[u8], _to_key: &[u8]) -> Result<(), Error> {
-            unimplemented!()
-        }
-
-        fn as_any(&self) -> &dyn Any {
-            self
-        }
-    }
-
-    #[tokio::test]
-    async fn test_get() {
-        let backend = MockKvBackend {};
-
-        let result = backend.get(0.to_string().as_bytes()).await;
-        assert_eq!(0.to_string().as_bytes(), result.unwrap().unwrap().0);
-
-        let result = backend.get(1.to_string().as_bytes()).await;
-        assert_eq!(1.to_string().as_bytes(), result.unwrap().unwrap().0);
-
-        let result = backend.get(2.to_string().as_bytes()).await;
-        assert_eq!(2.to_string().as_bytes(), result.unwrap().unwrap().0);
-
-        let result = backend.get(3.to_string().as_bytes()).await;
-        assert!(result.unwrap().is_none());
-    }
-}

src/catalog/src/remote/client.rs

@@ -17,54 +17,59 @@ use std::fmt::Debug;
 use std::sync::Arc;
 use std::time::Duration;
 
-use async_stream::stream;
+use common_error::ext::BoxedError;
+use common_meta::error::Error::{CacheNotGet, GetKvCache};
+use common_meta::error::{CacheNotGetSnafu, Error, MetaSrvSnafu, Result};
+use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
 use common_meta::rpc::store::{
-    CompareAndPutRequest, DeleteRangeRequest, MoveValueRequest, PutRequest, RangeRequest,
+    BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
+    BatchPutResponse, CompareAndPutRequest, CompareAndPutResponse, DeleteRangeRequest,
+    DeleteRangeResponse, MoveValueRequest, MoveValueResponse, PutRequest, PutResponse,
+    RangeRequest, RangeResponse,
 };
-use common_telemetry::{info, timer};
+use common_meta::rpc::KeyValue;
+use common_telemetry::timer;
 use meta_client::client::MetaClient;
 use moka::future::{Cache, CacheBuilder};
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 
 use super::KvCacheInvalidator;
-use crate::error::{Error, GenericSnafu, MetaSrvSnafu, Result};
 use crate::metrics::{METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET};
-use crate::remote::{Kv, KvBackend, KvBackendRef, ValueIter};
 
 const CACHE_MAX_CAPACITY: u64 = 10000;
 const CACHE_TTL_SECOND: u64 = 10 * 60;
 const CACHE_TTI_SECOND: u64 = 5 * 60;
 
-pub type CacheBackendRef = Arc<Cache<Vec<u8>, Option<Kv>>>;
+pub type CacheBackendRef = Arc<Cache<Vec<u8>, KeyValue>>;
+
 pub struct CachedMetaKvBackend {
     kv_backend: KvBackendRef,
     cache: CacheBackendRef,
+    name: String,
+}
+
+impl TxnService for CachedMetaKvBackend {
+    type Error = Error;
 }
 
 #[async_trait::async_trait]
 impl KvBackend for CachedMetaKvBackend {
-    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
-    where
-        'a: 'b,
-    {
-        self.kv_backend.range(key)
+    fn name(&self) -> &str {
+        &self.name
     }
 
-    async fn get(&self, key: &[u8]) -> Result<Option<Kv>> {
-        let _timer = timer!(METRIC_CATALOG_KV_GET);
-
-        let init = async {
-            let _timer = timer!(METRIC_CATALOG_KV_REMOTE_GET);
-
-            self.kv_backend.get(key).await
-        };
-
-        let schema_provider = self.cache.try_get_with_by_ref(key, init).await;
-        schema_provider.map_err(|e| GenericSnafu { msg: e.to_string() }.build())
+    fn as_any(&self) -> &dyn Any {
+        self
     }
 
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<()> {
-        let ret = self.kv_backend.set(key, val).await;
+    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
+        self.kv_backend.range(req).await
+    }
+
+    async fn put(&self, req: PutRequest) -> Result<PutResponse> {
+        let key = &req.key.clone();
+
+        let ret = self.kv_backend.put(req).await;
 
         if ret.is_ok() {
             self.invalidate_key(key).await;
@@ -73,8 +78,32 @@ impl KvBackend for CachedMetaKvBackend {
         ret
     }
 
-    async fn delete(&self, key: &[u8]) -> Result<()> {
-        let ret = self.kv_backend.delete_range(key, &[]).await;
+    async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse> {
+        let keys = req
+            .kvs
+            .iter()
+            .map(|kv| kv.key().to_vec())
+            .collect::<Vec<_>>();
+
+        let resp = self.kv_backend.batch_put(req).await;
+
+        if resp.is_ok() {
+            for key in keys {
+                self.invalidate_key(&key).await;
+            }
+        }
+
+        resp
+    }
+
+    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
+        self.kv_backend.batch_get(req).await
+    }
+
+    async fn compare_and_put(&self, req: CompareAndPutRequest) -> Result<CompareAndPutResponse> {
+        let key = &req.key.clone();
+
+        let ret = self.kv_backend.compare_and_put(req).await;
 
         if ret.is_ok() {
             self.invalidate_key(key).await;
@@ -83,28 +112,51 @@ impl KvBackend for CachedMetaKvBackend {
         ret
     }
 
-    async fn delete_range(&self, _key: &[u8], _end: &[u8]) -> Result<()> {
-        // TODO(fys): implement it
-        unimplemented!()
-    }
+    async fn delete_range(&self, mut req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
+        let prev_kv = req.prev_kv;
 
-    async fn compare_and_set(
-        &self,
-        key: &[u8],
-        expect: &[u8],
-        val: &[u8],
-    ) -> Result<std::result::Result<(), Option<Vec<u8>>>> {
-        let ret = self.kv_backend.compare_and_set(key, expect, val).await;
+        req.prev_kv = true;
+        let resp = self.kv_backend.delete_range(req).await;
+        match resp {
+            Ok(mut resp) => {
+                for prev_kv in resp.prev_kvs.iter() {
+                    self.invalidate_key(prev_kv.key()).await;
+                }
 
-        if ret.is_ok() {
-            self.invalidate_key(key).await;
+                if !prev_kv {
+                    resp.prev_kvs = vec![];
+                }
+                Ok(resp)
+            }
+            Err(e) => Err(e),
         }
-
-        ret
     }
 
-    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<()> {
-        let ret = self.kv_backend.move_value(from_key, to_key).await;
+    async fn batch_delete(&self, mut req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
+        let prev_kv = req.prev_kv;
+
+        req.prev_kv = true;
+        let resp = self.kv_backend.batch_delete(req).await;
+        match resp {
+            Ok(mut resp) => {
+                for prev_kv in resp.prev_kvs.iter() {
+                    self.invalidate_key(prev_kv.key()).await;
+                }
+
+                if !prev_kv {
+                    resp.prev_kvs = vec![];
+                }
+                Ok(resp)
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
+        let from_key = &req.from_key.clone();
+        let to_key = &req.to_key.clone();
+
+        let ret = self.kv_backend.move_value(req).await;
 
         if ret.is_ok() {
             self.invalidate_key(from_key).await;
@@ -114,8 +166,31 @@ impl KvBackend for CachedMetaKvBackend {
         ret
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
+    async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
+        let _timer = timer!(METRIC_CATALOG_KV_GET);
+
+        let init = async {
+            let _timer = timer!(METRIC_CATALOG_KV_REMOTE_GET);
+            self.kv_backend.get(key).await.map(|val| {
+                val.with_context(|| CacheNotGetSnafu {
+                    key: String::from_utf8_lossy(key),
+                })
+            })?
+        };
+
+        // currently moka doesn't have `optionally_try_get_with_by_ref`
+        // TODO(fys): change to moka method when available
+        // https://github.com/moka-rs/moka/issues/254
+        match self.cache.try_get_with_by_ref(key, init).await {
+            Ok(val) => Ok(Some(val)),
+            Err(e) => match e.as_ref() {
+                CacheNotGet { .. } => Ok(None),
+                _ => Err(e),
+            },
+        }
+        .map_err(|e| GetKvCache {
+            err_msg: e.to_string(),
+        })
     }
 }
 
@@ -128,15 +203,8 @@ impl KvCacheInvalidator for CachedMetaKvBackend {
 
 impl CachedMetaKvBackend {
     pub fn new(client: Arc<MetaClient>) -> Self {
-        let cache = Arc::new(
-            CacheBuilder::new(CACHE_MAX_CAPACITY)
-                .time_to_live(Duration::from_secs(CACHE_TTL_SECOND))
-                .time_to_idle(Duration::from_secs(CACHE_TTI_SECOND))
-                .build(),
-        );
         let kv_backend = Arc::new(MetaKvBackend { client });
-
-        Self { kv_backend, cache }
+        Self::wrap(kv_backend)
     }
 
     pub fn wrap(kv_backend: KvBackendRef) -> Self {
@@ -147,7 +215,12 @@ impl CachedMetaKvBackend {
                 .build(),
         );
 
-        Self { kv_backend, cache }
+        let name = format!("CachedKvBackend({})", kv_backend.name());
+        Self {
+            kv_backend,
+            cache,
+            name,
+        }
     }
 
     pub fn cache(&self) -> &CacheBackendRef {
@@ -160,88 +233,97 @@ pub struct MetaKvBackend {
     pub client: Arc<MetaClient>,
 }
 
+impl TxnService for MetaKvBackend {
+    type Error = Error;
+}
+
 /// Implement `KvBackend` trait for `MetaKvBackend` instead of opendal's `Accessor` since
 /// `MetaClient`'s range method can return both keys and values, which can reduce IO overhead
 /// comparing to `Accessor`'s list and get method.
 #[async_trait::async_trait]
 impl KvBackend for MetaKvBackend {
-    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
-    where
-        'a: 'b,
-    {
-        let key = key.to_vec();
-        Box::pin(stream!({
-            let mut resp = self
-                .client
-                .range(RangeRequest::new().with_prefix(key))
-                .await
-                .context(MetaSrvSnafu)?;
-            let kvs = resp.take_kvs();
-            for mut kv in kvs.into_iter() {
-                yield Ok(Kv(kv.take_key(), kv.take_value()))
-            }
-        }))
+    fn name(&self) -> &str {
+        "MetaKvBackend"
     }
 
-    async fn get(&self, key: &[u8]) -> Result<Option<Kv>> {
+    async fn range(&self, req: RangeRequest) -> Result<RangeResponse> {
+        self.client
+            .range(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
+    }
+
+    async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
         let mut response = self
             .client
             .range(RangeRequest::new().with_key(key))
             .await
+            .map_err(BoxedError::new)
             .context(MetaSrvSnafu)?;
-        Ok(response
-            .take_kvs()
-            .get_mut(0)
-            .map(|kv| Kv(kv.take_key(), kv.take_value())))
+        Ok(response.take_kvs().get_mut(0).map(|kv| KeyValue {
+            key: kv.take_key(),
+            value: kv.take_value(),
+        }))
     }
 
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<()> {
-        let req = PutRequest::new()
-            .with_key(key.to_vec())
-            .with_value(val.to_vec());
-        let _ = self.client.put(req).await.context(MetaSrvSnafu)?;
-        Ok(())
+    async fn batch_put(&self, req: BatchPutRequest) -> Result<BatchPutResponse> {
+        self.client
+            .batch_put(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
     }
 
-    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<()> {
-        let req = DeleteRangeRequest::new().with_range(key.to_vec(), end.to_vec());
-        let resp = self.client.delete_range(req).await.context(MetaSrvSnafu)?;
-        info!(
-            "Delete range, key: {}, end: {}, deleted: {}",
-            String::from_utf8_lossy(key),
-            String::from_utf8_lossy(end),
-            resp.deleted()
-        );
-
-        Ok(())
+    async fn put(&self, req: PutRequest) -> Result<PutResponse> {
+        self.client
+            .put(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
     }
 
-    async fn compare_and_set(
+    async fn delete_range(&self, req: DeleteRangeRequest) -> Result<DeleteRangeResponse> {
+        self.client
+            .delete_range(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
+    }
+
+    async fn batch_delete(&self, req: BatchDeleteRequest) -> Result<BatchDeleteResponse> {
+        self.client
+            .batch_delete(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
+    }
+
+    async fn batch_get(&self, req: BatchGetRequest) -> Result<BatchGetResponse> {
+        self.client
+            .batch_get(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
+    }
+
+    async fn compare_and_put(
         &self,
-        key: &[u8],
-        expect: &[u8],
-        val: &[u8],
-    ) -> Result<std::result::Result<(), Option<Vec<u8>>>> {
-        let request = CompareAndPutRequest::new()
-            .with_key(key.to_vec())
-            .with_expect(expect.to_vec())
-            .with_value(val.to_vec());
-        let mut response = self
-            .client
+        request: CompareAndPutRequest,
+    ) -> Result<CompareAndPutResponse> {
+        self.client
             .compare_and_put(request)
             .await
-            .context(MetaSrvSnafu)?;
-        if response.is_success() {
-            Ok(Ok(()))
-        } else {
-            Ok(Err(response.take_prev_kv().map(|v| v.value().to_vec())))
-        }
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
     }
 
-    async fn move_value(&self, from_key: &[u8], to_key: &[u8]) -> Result<()> {
-        let req = MoveValueRequest::new(from_key, to_key);
-        self.client.move_value(req).await.context(MetaSrvSnafu)?;
-        Ok(())
+    async fn move_value(&self, req: MoveValueRequest) -> Result<MoveValueResponse> {
+        self.client
+            .move_value(req)
+            .await
+            .map_err(BoxedError::new)
+            .context(MetaSrvSnafu)
     }
 
     fn as_any(&self) -> &dyn Any {

src/catalog/src/remote/mock.rs

@@ -12,160 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::any::Any;
-use std::collections::btree_map::Entry;
-use std::collections::{BTreeMap, HashMap};
-use std::fmt::{Display, Formatter};
-use std::str::FromStr;
-use std::sync::Arc;
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock as StdRwLock};
 
-use async_stream::stream;
-use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_recordbatch::RecordBatch;
-use common_telemetry::logging::info;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, Schema};
 use datatypes::vectors::StringVector;
-use serde::Serializer;
-use table::engine::{EngineContext, TableEngine, TableReference};
+use table::engine::{CloseTableResult, EngineContext, TableEngine};
 use table::metadata::TableId;
-use table::requests::{AlterTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest};
+use table::requests::{
+    AlterTableRequest, CloseTableRequest, CreateTableRequest, DropTableRequest, OpenTableRequest,
+    TruncateTableRequest,
+};
 use table::test_util::MemTable;
 use table::TableRef;
-use tokio::sync::RwLock;
-
-use crate::error::Error;
-use crate::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
-use crate::remote::{Kv, KvBackend, ValueIter};
-
-pub struct MockKvBackend {
-    map: RwLock<BTreeMap<Vec<u8>, Vec<u8>>>,
-}
-
-impl Default for MockKvBackend {
-    fn default() -> Self {
-        let mut map = BTreeMap::default();
-        let catalog_value = CatalogValue {}.as_bytes().unwrap();
-        let schema_value = SchemaValue {}.as_bytes().unwrap();
-
-        let default_catalog_key = CatalogKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-        }
-        .to_string();
-
-        let default_schema_key = SchemaKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
-        }
-        .to_string();
-
-        // create default catalog and schema
-        map.insert(default_catalog_key.into(), catalog_value);
-        map.insert(default_schema_key.into(), schema_value);
-
-        let map = RwLock::new(map);
-        Self { map }
-    }
-}
-
-impl Display for MockKvBackend {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        futures::executor::block_on(async {
-            let map = self.map.read().await;
-            for (k, v) in map.iter() {
-                f.serialize_str(&String::from_utf8_lossy(k))?;
-                f.serialize_str(" -> ")?;
-                f.serialize_str(&String::from_utf8_lossy(v))?;
-                f.serialize_str("\n")?;
-            }
-            Ok(())
-        })
-    }
-}
-
-#[async_trait::async_trait]
-impl KvBackend for MockKvBackend {
-    fn range<'a, 'b>(&'a self, key: &[u8]) -> ValueIter<'b, Error>
-    where
-        'a: 'b,
-    {
-        let prefix = key.to_vec();
-        let prefix_string = String::from_utf8_lossy(&prefix).to_string();
-        Box::pin(stream!({
-            let maps = self.map.read().await.clone();
-            for (k, v) in maps.range(prefix.clone()..) {
-                let key_string = String::from_utf8_lossy(k).to_string();
-                let matches = key_string.starts_with(&prefix_string);
-                if matches {
-                    yield Ok(Kv(k.clone(), v.clone()))
-                } else {
-                    info!("Stream finished");
-                    return;
-                }
-            }
-        }))
-    }
-
-    async fn set(&self, key: &[u8], val: &[u8]) -> Result<(), Error> {
-        let mut map = self.map.write().await;
-        map.insert(key.to_vec(), val.to_vec());
-        Ok(())
-    }
-
-    async fn compare_and_set(
-        &self,
-        key: &[u8],
-        expect: &[u8],
-        val: &[u8],
-    ) -> Result<Result<(), Option<Vec<u8>>>, Error> {
-        let mut map = self.map.write().await;
-        let existing = map.entry(key.to_vec());
-        match existing {
-            Entry::Vacant(e) => {
-                if expect.is_empty() {
-                    e.insert(val.to_vec());
-                    Ok(Ok(()))
-                } else {
-                    Ok(Err(None))
-                }
-            }
-            Entry::Occupied(mut existing) => {
-                if existing.get() == expect {
-                    existing.insert(val.to_vec());
-                    Ok(Ok(()))
-                } else {
-                    Ok(Err(Some(existing.get().clone())))
-                }
-            }
-        }
-    }
-
-    async fn delete_range(&self, key: &[u8], end: &[u8]) -> Result<(), Error> {
-        let mut map = self.map.write().await;
-        if end.is_empty() {
-            let _ = map.remove(key);
-        } else {
-            let start = key.to_vec();
-            let end = end.to_vec();
-            let range = start..end;
-
-            map.retain(|k, _| !range.contains(k));
-        }
-        Ok(())
-    }
-
-    async fn move_value(&self, _from_key: &[u8], _to_key: &[u8]) -> Result<(), Error> {
-        unimplemented!()
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
 
 #[derive(Default)]
 pub struct MockTableEngine {
-    tables: RwLock<HashMap<String, TableRef>>,
+    tables: StdRwLock<HashMap<TableId, TableRef>>,
 }
 
 #[async_trait::async_trait]
@@ -180,19 +45,8 @@ impl TableEngine for MockTableEngine {
         _ctx: &EngineContext,
         request: CreateTableRequest,
     ) -> table::Result<TableRef> {
-        let table_name = request.table_name.clone();
-        let catalog_name = request.catalog_name.clone();
-        let schema_name = request.schema_name.clone();
+        let table_id = request.id;
 
-        let default_table_id = "0".to_owned();
-        let table_id = TableId::from_str(
-            request
-                .table_options
-                .extra_options
-                .get("table_id")
-                .unwrap_or(&default_table_id),
-        )
-        .unwrap();
         let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
             "name",
             ConcreteDataType::string_datatype(),
@@ -202,16 +56,16 @@ impl TableEngine for MockTableEngine {
         let data = vec![Arc::new(StringVector::from(vec!["a", "b", "c"])) as _];
         let record_batch = RecordBatch::new(schema, data).unwrap();
         let table: TableRef = Arc::new(MemTable::new_with_catalog(
-            &table_name,
+            &request.table_name,
             record_batch,
             table_id,
-            catalog_name,
-            schema_name,
+            request.catalog_name,
+            request.schema_name,
             vec![0],
         )) as Arc<_>;
 
-        let mut tables = self.tables.write().await;
-        tables.insert(table_name, table.clone() as TableRef);
+        let mut tables = self.tables.write().unwrap();
+        let _ = tables.insert(table_id, table.clone() as TableRef);
         Ok(table)
     }
 
@@ -220,7 +74,7 @@ impl TableEngine for MockTableEngine {
         _ctx: &EngineContext,
         request: OpenTableRequest,
     ) -> table::Result<Option<TableRef>> {
-        Ok(self.tables.read().await.get(&request.table_name).cloned())
+        Ok(self.tables.read().unwrap().get(&request.table_id).cloned())
     }
 
     async fn alter_table(
@@ -234,25 +88,13 @@ impl TableEngine for MockTableEngine {
     fn get_table(
         &self,
         _ctx: &EngineContext,
-        table_ref: &TableReference,
+        table_id: TableId,
     ) -> table::Result<Option<TableRef>> {
-        futures::executor::block_on(async {
-            Ok(self
-                .tables
-                .read()
-                .await
-                .get(&table_ref.to_string())
-                .cloned())
-        })
+        Ok(self.tables.read().unwrap().get(&table_id).cloned())
     }
 
-    fn table_exists(&self, _ctx: &EngineContext, table_ref: &TableReference) -> bool {
-        futures::executor::block_on(async {
-            self.tables
-                .read()
-                .await
-                .contains_key(&table_ref.to_string())
-        })
+    fn table_exists(&self, _ctx: &EngineContext, table_id: TableId) -> bool {
+        self.tables.read().unwrap().contains_key(&table_id)
     }
 
     async fn drop_table(
@@ -263,7 +105,24 @@ impl TableEngine for MockTableEngine {
         unimplemented!()
     }
 
+    async fn close_table(
+        &self,
+        _ctx: &EngineContext,
+        request: CloseTableRequest,
+    ) -> table::Result<CloseTableResult> {
+        let _ = self.tables.write().unwrap().remove(&request.table_id);
+        Ok(CloseTableResult::Released(vec![]))
+    }
+
     async fn close(&self) -> table::Result<()> {
         Ok(())
     }
+
+    async fn truncate_table(
+        &self,
+        _ctx: &EngineContext,
+        _request: TruncateTableRequest,
+    ) -> table::Result<bool> {
+        Ok(true)
+    }
 }

src/catalog/src/remote/region_alive_keeper.rs

@@ -0,0 +1,867 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_meta::error::InvalidProtoMsgSnafu;
+use common_meta::heartbeat::handler::{
+    HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
+};
+use common_meta::ident::TableIdent;
+use common_meta::RegionIdent;
+use common_telemetry::{debug, error, info, warn};
+use snafu::{OptionExt, ResultExt};
+use store_api::storage::RegionNumber;
+use table::engine::manager::TableEngineManagerRef;
+use table::engine::{CloseTableResult, EngineContext, TableEngineRef};
+use table::metadata::TableId;
+use table::requests::CloseTableRequest;
+use table::TableRef;
+use tokio::sync::{mpsc, oneshot, Mutex};
+use tokio::task::JoinHandle;
+use tokio::time::{Duration, Instant};
+
+use crate::error::{Result, TableEngineNotFoundSnafu};
+use crate::local::MemoryCatalogManager;
+use crate::DeregisterTableRequest;
+
+/// [RegionAliveKeepers] manages all [RegionAliveKeeper] in a scope of tables.
+pub struct RegionAliveKeepers {
+    table_engine_manager: TableEngineManagerRef,
+    keepers: Arc<Mutex<HashMap<TableId, Arc<RegionAliveKeeper>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+
+    /// The epoch when [RegionAliveKeepers] is created. It's used to get a monotonically non-decreasing
+    /// elapsed time when submitting heartbeats to Metasrv (because [Instant] is monotonically
+    /// non-decreasing). The heartbeat request will carry the duration since this epoch, and the
+    /// duration acts like an "invariant point" for region's keep alive lease.
+    epoch: Instant,
+}
+
+impl RegionAliveKeepers {
+    pub fn new(
+        table_engine_manager: TableEngineManagerRef,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            table_engine_manager,
+            keepers: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+            epoch: Instant::now(),
+        }
+    }
+
+    pub async fn find_keeper(&self, table_id: TableId) -> Option<Arc<RegionAliveKeeper>> {
+        self.keepers.lock().await.get(&table_id).cloned()
+    }
+
+    pub async fn register_table(
+        &self,
+        table_ident: TableIdent,
+        table: TableRef,
+        catalog_manager: Arc<MemoryCatalogManager>,
+    ) -> Result<()> {
+        let table_id = table_ident.table_id;
+        let keeper = self.find_keeper(table_id).await;
+        if keeper.is_some() {
+            return Ok(());
+        }
+
+        let table_engine = self
+            .table_engine_manager
+            .engine(&table_ident.engine)
+            .context(TableEngineNotFoundSnafu {
+                engine_name: &table_ident.engine,
+            })?;
+
+        let keeper = Arc::new(RegionAliveKeeper::new(
+            table_engine,
+            catalog_manager,
+            table_ident.clone(),
+            self.heartbeat_interval_millis,
+        ));
+        for r in table.table_info().meta.region_numbers.iter() {
+            keeper.register_region(*r).await;
+        }
+
+        let mut keepers = self.keepers.lock().await;
+        let _ = keepers.insert(table_id, keeper.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            keeper.start().await;
+
+            info!("RegionAliveKeeper for table {table_ident} is started!");
+        } else {
+            info!("RegionAliveKeeper for table {table_ident} is registered but not started yet!");
+        }
+        Ok(())
+    }
+
+    pub async fn deregister_table(
+        &self,
+        table_ident: &TableIdent,
+    ) -> Option<Arc<RegionAliveKeeper>> {
+        let table_id = table_ident.table_id;
+        self.keepers.lock().await.remove(&table_id).map(|x| {
+            info!("Deregister RegionAliveKeeper for table {table_ident}");
+            x
+        })
+    }
+
+    pub async fn register_region(&self, region_ident: &RegionIdent) {
+        let table_id = region_ident.table_ident.table_id;
+        let Some(keeper) = self.find_keeper(table_id).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        keeper.register_region(region_ident.region_number).await
+    }
+
+    pub async fn deregister_region(&self, region_ident: &RegionIdent) {
+        let table_id = region_ident.table_ident.table_id;
+        let Some(keeper) = self.find_keeper(table_id).await else {
+            // Alive keeper could be affected by lagging msg, just warn and ignore.
+            warn!("Alive keeper for region {region_ident} is not found!");
+            return;
+        };
+        let _ = keeper.deregister_region(region_ident.region_number).await;
+    }
+
+    pub async fn start(&self) {
+        let keepers = self.keepers.lock().await;
+        for keeper in keepers.values() {
+            keeper.start().await;
+        }
+        self.started.store(true, Ordering::Relaxed);
+
+        info!(
+            "RegionAliveKeepers for tables {:?} are started!",
+            keepers.keys().map(|x| x.to_string()).collect::<Vec<_>>(),
+        );
+    }
+
+    pub fn epoch(&self) -> Instant {
+        self.epoch
+    }
+}
+
+#[async_trait]
+impl HeartbeatResponseHandler for RegionAliveKeepers {
+    fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
+        !ctx.response.region_leases.is_empty()
+    }
+
+    async fn handle(
+        &self,
+        ctx: &mut HeartbeatResponseHandlerContext,
+    ) -> common_meta::error::Result<HandleControl> {
+        let leases = ctx.response.region_leases.drain(..).collect::<Vec<_>>();
+        for lease in leases {
+            let table_ident: TableIdent = match lease
+                .table_ident
+                .context(InvalidProtoMsgSnafu {
+                    err_msg: "'table_ident' is missing in RegionLease",
+                })
+                .and_then(|x| x.try_into())
+            {
+                Ok(x) => x,
+                Err(e) => {
+                    error!(e; "");
+                    continue;
+                }
+            };
+
+            let table_id = table_ident.table_id;
+            let Some(keeper) = self.keepers.lock().await.get(&table_id).cloned() else {
+                // Alive keeper could be affected by lagging msg, just warn and ignore.
+                warn!("Alive keeper for table {table_ident} is not found!");
+                continue;
+            };
+
+            let start_instant = self.epoch + Duration::from_millis(lease.duration_since_epoch);
+            let deadline = start_instant + Duration::from_secs(lease.lease_seconds);
+            keeper.keep_lived(lease.regions, deadline).await;
+        }
+        Ok(HandleControl::Continue)
+    }
+}
+
+/// [RegionAliveKeeper] starts a countdown for each region in a table. When deadline is reached,
+/// the region will be closed.
+/// The deadline is controlled by Metasrv. It works like "lease" for regions: a Datanode submits its
+/// opened regions to Metasrv, in heartbeats. If Metasrv decides some region could be resided in this
+/// Datanode, it will "extend" the region's "lease", with a deadline for [RegionAliveKeeper] to
+/// countdown.
+pub struct RegionAliveKeeper {
+    catalog_manager: Arc<MemoryCatalogManager>,
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    countdown_task_handles: Arc<Mutex<HashMap<RegionNumber, Arc<CountdownTaskHandle>>>>,
+    heartbeat_interval_millis: u64,
+    started: AtomicBool,
+}
+
+impl RegionAliveKeeper {
+    fn new(
+        table_engine: TableEngineRef,
+        catalog_manager: Arc<MemoryCatalogManager>,
+        table_ident: TableIdent,
+        heartbeat_interval_millis: u64,
+    ) -> Self {
+        Self {
+            catalog_manager,
+            table_engine,
+            table_ident,
+            countdown_task_handles: Arc::new(Mutex::new(HashMap::new())),
+            heartbeat_interval_millis,
+            started: AtomicBool::new(false),
+        }
+    }
+
+    async fn find_handle(&self, region: &RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .get(region)
+            .cloned()
+    }
+
+    async fn register_region(&self, region: RegionNumber) {
+        if self.find_handle(&region).await.is_some() {
+            return;
+        }
+
+        let countdown_task_handles = Arc::downgrade(&self.countdown_task_handles);
+        let on_task_finished = async move {
+            if let Some(x) = countdown_task_handles.upgrade() {
+                let _ = x.lock().await.remove(&region);
+            } // Else the countdown task handles map could be dropped because the keeper is dropped.
+        };
+        let catalog_manager = self.catalog_manager.clone();
+        let ident = self.table_ident.clone();
+        let handle = Arc::new(CountdownTaskHandle::new(
+            self.table_engine.clone(),
+            self.table_ident.clone(),
+            region,
+            move |result: Option<CloseTableResult>| {
+                if matches!(result, Some(CloseTableResult::Released(_))) {
+                    let result = catalog_manager.deregister_table_sync(DeregisterTableRequest {
+                        catalog: ident.catalog.to_string(),
+                        schema: ident.schema.to_string(),
+                        table_name: ident.table.to_string(),
+                    });
+
+                    info!(
+                        "Deregister table: {} after countdown task finished, result: {result:?}",
+                        ident.table_id
+                    );
+                } else {
+                    debug!("Countdown task returns: {result:?}");
+                }
+                on_task_finished
+            },
+        ));
+
+        let mut handles = self.countdown_task_handles.lock().await;
+        let _ = handles.insert(region, handle.clone());
+
+        if self.started.load(Ordering::Relaxed) {
+            handle.start(self.heartbeat_interval_millis).await;
+
+            info!(
+                "Region alive countdown for region {region} in table {} is started!",
+                self.table_ident
+            );
+        } else {
+            info!(
+                "Region alive countdown for region {region} in table {} is registered but not started yet!",
+                self.table_ident
+            );
+        }
+    }
+
+    async fn deregister_region(&self, region: RegionNumber) -> Option<Arc<CountdownTaskHandle>> {
+        self.countdown_task_handles
+            .lock()
+            .await
+            .remove(&region)
+            .map(|x| {
+                info!(
+                    "Deregister alive countdown for region {region} in table {}",
+                    self.table_ident
+                );
+                x
+            })
+    }
+
+    async fn start(&self) {
+        let handles = self.countdown_task_handles.lock().await;
+        for handle in handles.values() {
+            handle.start(self.heartbeat_interval_millis).await;
+        }
+
+        self.started.store(true, Ordering::Relaxed);
+        info!(
+            "Region alive countdowns for regions {:?} in table {} are started!",
+            handles.keys().copied().collect::<Vec<_>>(),
+            self.table_ident
+        );
+    }
+
+    async fn keep_lived(&self, designated_regions: Vec<RegionNumber>, deadline: Instant) {
+        for region in designated_regions {
+            if let Some(handle) = self.find_handle(&region).await {
+                handle.reset_deadline(deadline).await;
+            }
+            // Else the region alive keeper might be triggered by lagging messages, we can safely ignore it.
+        }
+    }
+
+    pub async fn deadline(&self, region: RegionNumber) -> Option<Instant> {
+        let mut deadline = None;
+        if let Some(handle) = self.find_handle(&region).await {
+            let (s, r) = oneshot::channel();
+            if handle.tx.send(CountdownCommand::Deadline(s)).await.is_ok() {
+                deadline = r.await.ok()
+            }
+        }
+        deadline
+    }
+
+    pub fn table_ident(&self) -> &TableIdent {
+        &self.table_ident
+    }
+}
+
+#[derive(Debug)]
+enum CountdownCommand {
+    Start(u64),
+    Reset(Instant),
+    Deadline(oneshot::Sender<Instant>),
+}
+
+struct CountdownTaskHandle {
+    tx: mpsc::Sender<CountdownCommand>,
+    handler: JoinHandle<()>,
+    table_ident: TableIdent,
+    region: RegionNumber,
+}
+
+impl CountdownTaskHandle {
+    /// Creates a new [CountdownTaskHandle] and starts the countdown task.
+    /// # Params
+    /// - `on_task_finished`: a callback to be invoked when the task is finished. Note that it will not
+    ///   be invoked if the task is cancelled (by dropping the handle). This is because we want something
+    ///   meaningful to be done when the task is finished, e.g. deregister the handle from the map.
+    ///   While dropping the handle does not necessarily mean the task is finished.
+    fn new<Fut>(
+        table_engine: TableEngineRef,
+        table_ident: TableIdent,
+        region: RegionNumber,
+        on_task_finished: impl FnOnce(Option<CloseTableResult>) -> Fut + Send + 'static,
+    ) -> Self
+    where
+        Fut: Future<Output = ()> + Send,
+    {
+        let (tx, rx) = mpsc::channel(1024);
+
+        let mut countdown_task = CountdownTask {
+            table_engine,
+            table_ident: table_ident.clone(),
+            region,
+            rx,
+        };
+        let handler = common_runtime::spawn_bg(async move {
+            let result = countdown_task.run().await;
+            on_task_finished(result).await;
+        });
+
+        Self {
+            tx,
+            handler,
+            table_ident,
+            region,
+        }
+    }
+
+    async fn start(&self, heartbeat_interval_millis: u64) {
+        if let Err(e) = self
+            .tx
+            .send(CountdownCommand::Start(heartbeat_interval_millis))
+            .await
+        {
+            warn!(
+                "Failed to start region alive keeper countdown: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+
+    async fn reset_deadline(&self, deadline: Instant) {
+        if let Err(e) = self.tx.send(CountdownCommand::Reset(deadline)).await {
+            warn!(
+                "Failed to reset region alive keeper deadline: {e}. \
+                Maybe the task is stopped due to region been closed."
+            );
+        }
+    }
+}
+
+impl Drop for CountdownTaskHandle {
+    fn drop(&mut self) {
+        debug!(
+            "Aborting region alive countdown task for region {} in table {}",
+            self.region, self.table_ident,
+        );
+        self.handler.abort();
+    }
+}
+
+struct CountdownTask {
+    table_engine: TableEngineRef,
+    table_ident: TableIdent,
+    region: RegionNumber,
+    rx: mpsc::Receiver<CountdownCommand>,
+}
+
+impl CountdownTask {
+    // returns true if
+    async fn run(&mut self) -> Option<CloseTableResult> {
+        // 30 years. See `Instant::far_future`.
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
+
+        // Make sure the alive countdown is not gonna happen before heartbeat task is started (the
+        // "start countdown" command will be sent from heartbeat task).
+        let countdown = tokio::time::sleep_until(far_future);
+        tokio::pin!(countdown);
+
+        let region = &self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            tokio::select! {
+                command = self.rx.recv() => {
+                    match command {
+                        Some(CountdownCommand::Start(heartbeat_interval_millis)) => {
+                            // Set first deadline in 4 heartbeats (roughly after 20 seconds from now if heartbeat
+                            // interval is set to default 5 seconds), to make Datanode and Metasrv more tolerable to
+                            // network or other jitters during startup.
+                            let first_deadline = Instant::now() + Duration::from_millis(heartbeat_interval_millis) * 4;
+                            countdown.set(tokio::time::sleep_until(first_deadline));
+                        },
+                        Some(CountdownCommand::Reset(deadline)) => {
+                            if countdown.deadline() < deadline {
+                                debug!(
+                                    "Reset deadline of region {region} of table {table_ident} to approximately {} seconds later",
+                                    (deadline - Instant::now()).as_secs_f32(),
+                                );
+                                countdown.set(tokio::time::sleep_until(deadline));
+                            }
+                            // Else the countdown could be either:
+                            // - not started yet;
+                            // - during startup protection;
+                            // - received a lagging heartbeat message.
+                            // All can be safely ignored.
+                        },
+                        None => {
+                            info!(
+                                "The handle of countdown task for region {region} of table {table_ident} \
+                                is dropped, RegionAliveKeeper out."
+                            );
+                            break;
+                        },
+                        Some(CountdownCommand::Deadline(tx)) => {
+                            let _ = tx.send(countdown.deadline());
+                        }
+                    }
+                }
+                () = &mut countdown => {
+                    let result = self.close_region().await;
+                    warn!(
+                        "Region {region} of table {table_ident} is closed, result: {result:?}. \
+                        RegionAliveKeeper out.",
+                    );
+                    return Some(result);
+                }
+            }
+        }
+        None
+    }
+
+    async fn close_region(&self) -> CloseTableResult {
+        let ctx = EngineContext::default();
+        let region = self.region;
+        let table_ident = &self.table_ident;
+        loop {
+            let request = CloseTableRequest {
+                catalog_name: table_ident.catalog.clone(),
+                schema_name: table_ident.schema.clone(),
+                table_name: table_ident.table.clone(),
+                table_id: table_ident.table_id,
+                region_numbers: vec![region],
+                flush: true,
+            };
+            match self.table_engine.close_table(&ctx, request).await {
+                Ok(result) => return result,
+                // If region is failed to close, immediately retry. Maybe we should panic instead?
+                Err(e) => error!(e;
+                    "Failed to close region {region} of table {table_ident}. \
+                    For the integrity of data, retry closing and retry without wait.",
+                ),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+
+    use api::v1::meta::{HeartbeatResponse, RegionLease};
+    use common_meta::heartbeat::mailbox::HeartbeatMailbox;
+    use datatypes::schema::RawSchema;
+    use table::engine::manager::MemoryTableEngineManager;
+    use table::engine::TableEngine;
+    use table::requests::{CreateTableRequest, TableOptions};
+    use table::test_util::EmptyTable;
+
+    use super::*;
+    use crate::remote::mock::MockTableEngine;
+
+    async fn prepare_keepers() -> (TableIdent, RegionAliveKeepers) {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_engine_manager = Arc::new(MemoryTableEngineManager::new(table_engine));
+        let keepers = RegionAliveKeepers::new(table_engine_manager, 5000);
+
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id: 1,
+            engine: "MockTableEngine".to_string(),
+        };
+        let table = Arc::new(EmptyTable::new(CreateTableRequest {
+            id: 1,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![1, 2, 3],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "MockTableEngine".to_string(),
+        }));
+        let catalog_manager = MemoryCatalogManager::new_with_table(table.clone());
+        keepers
+            .register_table(table_ident.clone(), table, catalog_manager)
+            .await
+            .unwrap();
+        assert!(keepers
+            .keepers
+            .lock()
+            .await
+            .contains_key(&table_ident.table_id));
+
+        (table_ident, keepers)
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_handle_heartbeat_response() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers.start().await;
+        let startup_protection_until = Instant::now() + Duration::from_secs(21);
+
+        let duration_since_epoch = (Instant::now() - keepers.epoch).as_millis() as _;
+        let lease_seconds = 100;
+        let response = HeartbeatResponse {
+            region_leases: vec![RegionLease {
+                table_ident: Some(table_ident.clone().into()),
+                regions: vec![1, 3], // Not extending region 2's lease time.
+                duration_since_epoch,
+                lease_seconds,
+            }],
+            ..Default::default()
+        };
+        let keep_alive_until = keepers.epoch
+            + Duration::from_millis(duration_since_epoch)
+            + Duration::from_secs(lease_seconds);
+
+        let (tx, _) = mpsc::channel(8);
+        let mailbox = Arc::new(HeartbeatMailbox::new(tx));
+        let mut ctx = HeartbeatResponseHandlerContext::new(mailbox, response);
+
+        assert!(keepers.handle(&mut ctx).await.unwrap() == HandleControl::Continue);
+
+        // sleep to wait for background task spawned in `handle`
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        async fn test(
+            keeper: &Arc<RegionAliveKeeper>,
+            region_number: RegionNumber,
+            startup_protection_until: Instant,
+            keep_alive_until: Instant,
+            is_kept_live: bool,
+        ) {
+            let deadline = keeper.deadline(region_number).await.unwrap();
+            if is_kept_live {
+                assert!(deadline > startup_protection_until && deadline == keep_alive_until);
+            } else {
+                assert!(deadline <= startup_protection_until);
+            }
+        }
+
+        let keeper = &keepers
+            .keepers
+            .lock()
+            .await
+            .get(&table_ident.table_id)
+            .cloned()
+            .unwrap();
+
+        // Test region 1 and 3 is kept lived. Their deadlines are updated to desired instant.
+        test(keeper, 1, startup_protection_until, keep_alive_until, true).await;
+        test(keeper, 3, startup_protection_until, keep_alive_until, true).await;
+
+        // Test region 2 is not kept lived. It's deadline is not updated: still during startup protection period.
+        test(keeper, 2, startup_protection_until, keep_alive_until, false).await;
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keepers() {
+        let (table_ident, keepers) = prepare_keepers().await;
+
+        keepers
+            .register_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 4,
+            })
+            .await;
+
+        keepers.start().await;
+        for keeper in keepers.keepers.lock().await.values() {
+            let regions = {
+                let handles = keeper.countdown_task_handles.lock().await;
+                handles.keys().copied().collect::<Vec<_>>()
+            };
+            for region in regions {
+                // assert countdown tasks are started
+                let deadline = keeper.deadline(region).await.unwrap();
+                assert!(deadline <= Instant::now() + Duration::from_secs(20));
+            }
+        }
+
+        keepers
+            .deregister_region(&RegionIdent {
+                cluster_id: 1,
+                datanode_id: 1,
+                table_ident: table_ident.clone(),
+                region_number: 1,
+            })
+            .await;
+        let mut regions = keepers
+            .find_keeper(table_ident.table_id)
+            .await
+            .unwrap()
+            .countdown_task_handles
+            .lock()
+            .await
+            .keys()
+            .copied()
+            .collect::<Vec<_>>();
+        regions.sort();
+        assert_eq!(regions, vec![2, 3, 4]);
+
+        let keeper = keepers.deregister_table(&table_ident).await.unwrap();
+        assert!(Arc::try_unwrap(keeper).is_ok(), "keeper is not dropped");
+        assert!(keepers.keepers.lock().await.is_empty());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_region_alive_keeper() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let catalog_manager = MemoryCatalogManager::with_default_setup();
+        let keeper = RegionAliveKeeper::new(table_engine, catalog_manager, table_ident, 1000);
+
+        let region = 1;
+        assert!(keeper.find_handle(&region).await.is_none());
+        keeper.register_region(region).await;
+        let _ = keeper.find_handle(&region).await.unwrap();
+
+        let ten_seconds_later = || Instant::now() + Duration::from_secs(10);
+
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        assert!(keeper.find_handle(&2).await.is_none());
+        assert!(keeper.find_handle(&3).await.is_none());
+
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert if keeper is not started, keep_lived is of no use
+        assert!(keeper.deadline(region).await.unwrap() > far_future);
+
+        keeper.start().await;
+        keeper.keep_lived(vec![1, 2, 3], ten_seconds_later()).await;
+        // assert keep_lived works if keeper is started
+        assert!(keeper.deadline(region).await.unwrap() <= ten_seconds_later());
+
+        let handle = keeper.deregister_region(region).await.unwrap();
+        assert!(Arc::try_unwrap(handle).is_ok(), "handle is not dropped");
+        assert!(keeper.find_handle(&region).await.is_none());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_handle() {
+        let table_engine = Arc::new(MockTableEngine::default());
+        let table_ident = TableIdent {
+            catalog: "my_catalog".to_string(),
+            schema: "my_schema".to_string(),
+            table: "my_table".to_string(),
+            table_id: 1024,
+            engine: "mito".to_string(),
+        };
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(
+            table_engine.clone(),
+            table_ident.clone(),
+            1,
+            |_| async move { finished_clone.store(true, Ordering::Relaxed) },
+        );
+        let tx = handle.tx.clone();
+
+        // assert countdown task is running
+        tx.send(CountdownCommand::Start(5000)).await.unwrap();
+        assert!(!finished.load(Ordering::Relaxed));
+
+        drop(handle);
+        tokio::time::sleep(Duration::from_secs(1)).await;
+
+        // assert countdown task is stopped
+        assert!(tx
+            .try_send(CountdownCommand::Reset(
+                Instant::now() + Duration::from_secs(10)
+            ))
+            .is_err());
+        // assert `on_task_finished` is not called (because the task is aborted by the handle's drop)
+        assert!(!finished.load(Ordering::Relaxed));
+
+        let finished = Arc::new(AtomicBool::new(false));
+        let finished_clone = finished.clone();
+        let handle = CountdownTaskHandle::new(table_engine, table_ident, 1, |_| async move {
+            finished_clone.store(true, Ordering::Relaxed)
+        });
+        handle.tx.send(CountdownCommand::Start(100)).await.unwrap();
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        // assert `on_task_finished` is called when task is finished normally
+        assert!(finished.load(Ordering::Relaxed));
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_countdown_task_run() {
+        let ctx = &EngineContext::default();
+        let catalog = "my_catalog";
+        let schema = "my_schema";
+        let table = "my_table";
+        let table_id = 1;
+        let request = CreateTableRequest {
+            id: table_id,
+            catalog_name: catalog.to_string(),
+            schema_name: schema.to_string(),
+            table_name: table.to_string(),
+            desc: None,
+            schema: RawSchema {
+                column_schemas: vec![],
+                timestamp_index: None,
+                version: 0,
+            },
+            region_numbers: vec![],
+            primary_key_indices: vec![],
+            create_if_not_exists: false,
+            table_options: TableOptions::default(),
+            engine: "mito".to_string(),
+        };
+
+        let table_engine = Arc::new(MockTableEngine::default());
+        let _ = table_engine.create_table(ctx, request).await.unwrap();
+
+        let table_ident = TableIdent {
+            catalog: catalog.to_string(),
+            schema: schema.to_string(),
+            table: table.to_string(),
+            table_id,
+            engine: "mito".to_string(),
+        };
+        let (tx, rx) = mpsc::channel(10);
+        let mut task = CountdownTask {
+            table_engine: table_engine.clone(),
+            table_ident,
+            region: 1,
+            rx,
+        };
+        let _handle = common_runtime::spawn_bg(async move {
+            task.run().await;
+        });
+
+        async fn deadline(tx: &mpsc::Sender<CountdownCommand>) -> Instant {
+            let (s, r) = oneshot::channel();
+            tx.send(CountdownCommand::Deadline(s)).await.unwrap();
+            r.await.unwrap()
+        }
+
+        // if countdown task is not started, its deadline is set to far future
+        assert!(deadline(&tx).await > Instant::now() + Duration::from_secs(86400 * 365 * 29));
+
+        // start countdown in 250ms * 4 = 1s
+        tx.send(CountdownCommand::Start(250)).await.unwrap();
+        // assert deadline is correctly set
+        assert!(deadline(&tx).await <= Instant::now() + Duration::from_secs(1));
+
+        // reset countdown in 1.5s
+        tx.send(CountdownCommand::Reset(
+            Instant::now() + Duration::from_millis(1500),
+        ))
+        .await
+        .unwrap();
+
+        // assert the table is closed after deadline is reached
+        assert!(table_engine.table_exists(ctx, table_id));
+        // spare 500ms for the task to close the table
+        tokio::time::sleep(Duration::from_millis(2000)).await;
+        assert!(!table_engine.table_exists(ctx, table_id));
+    }
+}

src/catalog/src/schema.rs

@@ -1,69 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::any::Any;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use table::TableRef;
-
-use crate::error::{NotSupportedSnafu, Result};
-
-/// Represents a schema, comprising a number of named tables.
-#[async_trait]
-pub trait SchemaProvider: Sync + Send {
-    /// Returns the schema provider as [`Any`](std::any::Any)
-    /// so that it can be downcast to a specific implementation.
-    fn as_any(&self) -> &dyn Any;
-
-    /// Retrieves the list of available table names in this schema.
-    async fn table_names(&self) -> Result<Vec<String>>;
-
-    /// Retrieves a specific table from the schema by name, provided it exists.
-    async fn table(&self, name: &str) -> Result<Option<TableRef>>;
-
-    /// If supported by the implementation, adds a new table to this schema.
-    /// If a table of the same name existed before, it returns "Table already exists" error.
-    async fn register_table(&self, name: String, _table: TableRef) -> Result<Option<TableRef>> {
-        NotSupportedSnafu {
-            op: format!("register_table({name}, <table>)"),
-        }
-        .fail()
-    }
-
-    /// If supported by the implementation, renames an existing table from this schema and returns it.
-    /// If no table of that name exists, returns "Table not found" error.
-    async fn rename_table(&self, name: &str, new_name: String) -> Result<TableRef> {
-        NotSupportedSnafu {
-            op: format!("rename_table({name}, {new_name})"),
-        }
-        .fail()
-    }
-
-    /// If supported by the implementation, removes an existing table from this schema and returns it.
-    /// If no table of that name exists, returns Ok(None).
-    async fn deregister_table(&self, name: &str) -> Result<Option<TableRef>> {
-        NotSupportedSnafu {
-            op: format!("deregister_table({name})"),
-        }
-        .fail()
-    }
-
-    /// If supported by the implementation, checks the table exist in the schema provider or not.
-    /// If no matched table in the schema provider, return false.
-    /// Otherwise, return true.
-    async fn table_exist(&self, name: &str) -> Result<bool>;
-}
-
-pub type SchemaProviderRef = Arc<dyn SchemaProvider>;

src/catalog/src/system.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::any::Any;
 use std::collections::HashMap;
 use std::sync::Arc;
 
@@ -20,27 +19,24 @@ use common_catalog::consts::{
     DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
     SYSTEM_CATALOG_NAME, SYSTEM_CATALOG_TABLE_ID, SYSTEM_CATALOG_TABLE_NAME,
 };
-use common_query::logical_plan::Expr;
-use common_query::physical_plan::{PhysicalPlanRef, SessionContext};
 use common_recordbatch::SendableRecordBatchStream;
-use common_telemetry::debug;
+use common_telemetry::{debug, warn};
 use common_time::util;
 use datatypes::prelude::{ConcreteDataType, ScalarVector, VectorRef};
-use datatypes::schema::{ColumnSchema, RawSchema, SchemaRef};
+use datatypes::schema::{ColumnSchema, RawSchema};
 use datatypes::vectors::{BinaryVector, TimestampMillisecondVector, UInt8Vector};
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::storage::ScanRequest;
 use table::engine::{EngineContext, TableEngineRef};
-use table::metadata::{TableId, TableInfoRef};
-use table::requests::{
-    CreateTableRequest, DeleteRequest, InsertRequest, OpenTableRequest, TableOptions,
-};
-use table::{Result as TableResult, Table, TableRef};
+use table::metadata::TableId;
+use table::requests::{CreateTableRequest, InsertRequest, OpenTableRequest, TableOptions};
+use table::TableRef;
 
 use crate::error::{
-    self, CreateSystemCatalogSnafu, EmptyValueSnafu, Error, InvalidEntryTypeSnafu, InvalidKeySnafu,
-    OpenSystemCatalogSnafu, Result, ValueDeserializeSnafu,
+    self, CreateSystemCatalogSnafu, DeregisterTableSnafu, EmptyValueSnafu, Error,
+    InsertCatalogRecordSnafu, InvalidEntryTypeSnafu, InvalidKeySnafu, OpenSystemCatalogSnafu,
+    Result, ValueDeserializeSnafu,
 };
 use crate::DeregisterTableRequest;
 
@@ -50,47 +46,6 @@ pub const VALUE_INDEX: usize = 3;
 
 pub struct SystemCatalogTable(TableRef);
 
-#[async_trait::async_trait]
-impl Table for SystemCatalogTable {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.0.schema()
-    }
-
-    async fn scan(
-        &self,
-        projection: Option<&Vec<usize>>,
-        filters: &[Expr],
-        limit: Option<usize>,
-    ) -> table::Result<PhysicalPlanRef> {
-        self.0.scan(projection, filters, limit).await
-    }
-
-    async fn scan_to_stream(&self, request: ScanRequest) -> TableResult<SendableRecordBatchStream> {
-        self.0.scan_to_stream(request).await
-    }
-
-    /// Insert values into table.
-    async fn insert(&self, request: InsertRequest) -> TableResult<usize> {
-        self.0.insert(request).await
-    }
-
-    fn table_info(&self) -> TableInfoRef {
-        self.0.table_info()
-    }
-
-    async fn delete(&self, request: DeleteRequest) -> TableResult<usize> {
-        self.0.delete(request).await
-    }
-
-    fn statistics(&self) -> Option<table::stats::TableStatistics> {
-        self.0.statistics()
-    }
-}
-
 impl SystemCatalogTable {
     pub async fn new(engine: TableEngineRef) -> Result<Self> {
         let request = OpenTableRequest {
@@ -133,19 +88,75 @@ impl SystemCatalogTable {
         }
     }
 
+    pub async fn register_table(
+        &self,
+        catalog: String,
+        schema: String,
+        table_name: String,
+        table_id: TableId,
+        engine: String,
+    ) -> Result<usize> {
+        let insert_request =
+            build_table_insert_request(catalog, schema, table_name, table_id, engine);
+        self.0
+            .insert(insert_request)
+            .await
+            .context(InsertCatalogRecordSnafu)
+    }
+
+    pub(crate) async fn deregister_table(
+        &self,
+        request: &DeregisterTableRequest,
+        table_id: TableId,
+    ) -> Result<()> {
+        let deletion_request = build_table_deletion_request(request, table_id);
+        self.0
+            .insert(deletion_request)
+            .await
+            .map(|x| {
+                if x != 1 {
+                    let table = common_catalog::format_full_table_name(
+                        &request.catalog,
+                        &request.schema,
+                        &request.table_name
+                    );
+                    warn!("Failed to delete table record from information_schema, unexpected returned result: {x}, table: {table}");
+                }
+            })
+            .with_context(|_| DeregisterTableSnafu {
+                request: request.clone(),
+            })
+    }
+
+    pub async fn register_schema(&self, catalog: String, schema: String) -> Result<usize> {
+        let insert_request = build_schema_insert_request(catalog, schema);
+        self.0
+            .insert(insert_request)
+            .await
+            .context(InsertCatalogRecordSnafu)
+    }
+
     /// Create a stream of all entries inside system catalog table
     pub async fn records(&self) -> Result<SendableRecordBatchStream> {
         let full_projection = None;
-        let ctx = SessionContext::new();
-        let scan = self
-            .scan(full_projection, &[], None)
+        let scan_req = ScanRequest {
+            sequence: None,
+            projection: full_projection,
+            filters: vec![],
+            output_ordering: None,
+            limit: None,
+        };
+        let stream = self
+            .0
+            .scan_to_stream(scan_req)
             .await
             .context(error::SystemCatalogTableScanSnafu)?;
-        let stream = scan
-            .execute(0, ctx.task_ctx())
-            .context(error::SystemCatalogTableScanExecSnafu)?;
         Ok(stream)
     }
+
+    pub fn as_table_ref(&self) -> TableRef {
+        self.0.clone()
+    }
 }
 
 /// Build system catalog table schema.
@@ -211,38 +222,50 @@ pub fn build_table_insert_request(
     build_insert_request(
         EntryType::Table,
         entry_key.as_bytes(),
-        serde_json::to_string(&TableEntryValue { table_name, engine })
-            .unwrap()
-            .as_bytes(),
+        serde_json::to_string(&TableEntryValue {
+            table_name,
+            engine,
+            is_deleted: false,
+        })
+        .unwrap()
+        .as_bytes(),
     )
 }
 
 pub(crate) fn build_table_deletion_request(
     request: &DeregisterTableRequest,
     table_id: TableId,
-) -> DeleteRequest {
-    let table_key = format_table_entry_key(&request.catalog, &request.schema, table_id);
-    DeleteRequest {
-        key_column_values: build_primary_key_columns(EntryType::Table, table_key.as_bytes()),
-    }
+) -> InsertRequest {
+    let entry_key = format_table_entry_key(&request.catalog, &request.schema, table_id);
+    build_insert_request(
+        EntryType::Table,
+        entry_key.as_bytes(),
+        serde_json::to_string(&TableEntryValue {
+            table_name: "".to_string(),
+            engine: "".to_string(),
+            is_deleted: true,
+        })
+        .unwrap()
+        .as_bytes(),
+    )
 }
 
 fn build_primary_key_columns(entry_type: EntryType, key: &[u8]) -> HashMap<String, VectorRef> {
-    let mut m = HashMap::with_capacity(3);
-    m.insert(
-        "entry_type".to_string(),
-        Arc::new(UInt8Vector::from_slice([entry_type as u8])) as _,
-    );
-    m.insert(
-        "key".to_string(),
-        Arc::new(BinaryVector::from_slice(&[key])) as _,
-    );
-    // Timestamp in key part is intentionally left to 0
-    m.insert(
-        "timestamp".to_string(),
-        Arc::new(TimestampMillisecondVector::from_slice([0])) as _,
-    );
-    m
+    HashMap::from([
+        (
+            "entry_type".to_string(),
+            Arc::new(UInt8Vector::from_slice([entry_type as u8])) as VectorRef,
+        ),
+        (
+            "key".to_string(),
+            Arc::new(BinaryVector::from_slice(&[key])) as VectorRef,
+        ),
+        (
+            "timestamp".to_string(),
+            // Timestamp in key part is intentionally left to 0
+            Arc::new(TimestampMillisecondVector::from_slice([0])) as VectorRef,
+        ),
+    ])
 }
 
 pub fn build_schema_insert_request(catalog_name: String, schema_name: String) -> InsertRequest {
@@ -260,20 +283,20 @@ pub fn build_insert_request(entry_type: EntryType, key: &[u8], value: &[u8]) ->
     let primary_key_columns = build_primary_key_columns(entry_type, key);
 
     let mut columns_values = HashMap::with_capacity(6);
-    columns_values.extend(primary_key_columns.into_iter());
+    columns_values.extend(primary_key_columns);
 
-    columns_values.insert(
+    let _ = columns_values.insert(
         "value".to_string(),
         Arc::new(BinaryVector::from_slice(&[value])) as _,
     );
 
     let now = util::current_time_millis();
-    columns_values.insert(
+    let _ = columns_values.insert(
         "gmt_created".to_string(),
         Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
     );
 
-    columns_values.insert(
+    let _ = columns_values.insert(
         "gmt_modified".to_string(),
         Arc::new(TimestampMillisecondVector::from_slice([now])) as _,
     );
@@ -343,6 +366,7 @@ pub fn decode_system_catalog(
                 table_name: table_meta.table_name,
                 table_id,
                 engine: table_meta.engine,
+                is_deleted: table_meta.is_deleted,
             }))
         }
     }
@@ -399,6 +423,7 @@ pub struct TableEntry {
     pub table_name: String,
     pub table_id: TableId,
     pub engine: String,
+    pub is_deleted: bool,
 }
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
@@ -407,12 +432,19 @@ pub struct TableEntryValue {
 
     #[serde(default = "mito_engine")]
     pub engine: String,
+
+    #[serde(default = "not_deleted")]
+    pub is_deleted: bool,
 }
 
 fn mito_engine() -> String {
     MITO_ENGINE.to_string()
 }
 
+fn not_deleted() -> bool {
+    false
+}
+
 #[cfg(test)]
 mod tests {
     use common_recordbatch::RecordBatches;
@@ -482,14 +514,13 @@ mod tests {
     }
 
     #[test]
-    #[should_panic]
     pub fn test_decode_mismatch() {
-        decode_system_catalog(
+        assert!(decode_system_catalog(
             Some(EntryType::Table as u8),
             Some("some_catalog.some_schema.42".as_bytes()),
             None,
         )
-        .unwrap();
+        .is_err());
     }
 
     #[test]
@@ -504,14 +535,14 @@ mod tests {
         let dir = create_temp_dir("system-table-test");
         let store_dir = dir.path().to_string_lossy();
         let mut builder = object_store::services::Fs::default();
-        builder.root(&store_dir);
+        let _ = builder.root(&store_dir);
         let object_store = ObjectStore::new(builder).unwrap().finish();
         let noop_compaction_scheduler = Arc::new(NoopCompactionScheduler::default());
         let table_engine = Arc::new(MitoEngine::new(
             EngineConfig::default(),
             EngineImpl::new(
                 StorageEngineConfig::default(),
-                Arc::new(NoopLogStore::default()),
+                Arc::new(NoopLogStore),
                 object_store.clone(),
                 noop_compaction_scheduler,
             )
@@ -525,14 +556,14 @@ mod tests {
     async fn test_system_table_type() {
         let (_dir, table_engine) = prepare_table_engine().await;
         let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
-        assert_eq!(Base, system_table.table_type());
+        assert_eq!(Base, system_table.as_table_ref().table_type());
     }
 
     #[tokio::test]
     async fn test_system_table_info() {
         let (_dir, table_engine) = prepare_table_engine().await;
         let system_table = SystemCatalogTable::new(table_engine).await.unwrap();
-        let info = system_table.table_info();
+        let info = system_table.as_table_ref().table_info();
         assert_eq!(TableType::Base, info.table_type);
         assert_eq!(SYSTEM_CATALOG_TABLE_NAME, info.name);
         assert_eq!(SYSTEM_CATALOG_TABLE_ID, info.ident.table_id);
@@ -545,14 +576,16 @@ mod tests {
         let (_, table_engine) = prepare_table_engine().await;
         let catalog_table = SystemCatalogTable::new(table_engine).await.unwrap();
 
-        let table_insertion = build_table_insert_request(
-            DEFAULT_CATALOG_NAME.to_string(),
-            DEFAULT_SCHEMA_NAME.to_string(),
-            "my_table".to_string(),
-            1,
-            MITO_ENGINE.to_string(),
-        );
-        let result = catalog_table.insert(table_insertion).await.unwrap();
+        let result = catalog_table
+            .register_table(
+                DEFAULT_CATALOG_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string(),
+                "my_table".to_string(),
+                1,
+                MITO_ENGINE.to_string(),
+            )
+            .await
+            .unwrap();
         assert_eq!(result, 1);
 
         let records = catalog_table.records().await.unwrap();
@@ -562,9 +595,15 @@ mod tests {
         assert_eq!(batch.num_rows(), 1);
 
         let row = batch.rows().next().unwrap();
-        let Value::UInt8(entry_type) = row[0] else { unreachable!() };
-        let Value::Binary(key) = row[1].clone() else { unreachable!() };
-        let Value::Binary(value) = row[3].clone() else { unreachable!() };
+        let Value::UInt8(entry_type) = row[0] else {
+            unreachable!()
+        };
+        let Value::Binary(key) = row[1].clone() else {
+            unreachable!()
+        };
+        let Value::Binary(value) = row[3].clone() else {
+            unreachable!()
+        };
         let entry = decode_system_catalog(Some(entry_type), Some(&*key), Some(&*value)).unwrap();
         let expected = Entry::Table(TableEntry {
             catalog_name: DEFAULT_CATALOG_NAME.to_string(),
@@ -572,22 +611,24 @@ mod tests {
             table_name: "my_table".to_string(),
             table_id: 1,
             engine: MITO_ENGINE.to_string(),
+            is_deleted: false,
         });
         assert_eq!(entry, expected);
 
-        let table_deletion = build_table_deletion_request(
-            &DeregisterTableRequest {
-                catalog: DEFAULT_CATALOG_NAME.to_string(),
-                schema: DEFAULT_SCHEMA_NAME.to_string(),
-                table_name: "my_table".to_string(),
-            },
-            1,
-        );
-        let result = catalog_table.delete(table_deletion).await.unwrap();
-        assert_eq!(result, 1);
+        catalog_table
+            .deregister_table(
+                &DeregisterTableRequest {
+                    catalog: DEFAULT_CATALOG_NAME.to_string(),
+                    schema: DEFAULT_SCHEMA_NAME.to_string(),
+                    table_name: "my_table".to_string(),
+                },
+                1,
+            )
+            .await
+            .unwrap();
 
         let records = catalog_table.records().await.unwrap();
         let batches = RecordBatches::try_collect(records).await.unwrap().take();
-        assert_eq!(batches.len(), 0);
+        assert_eq!(batches.len(), 1);
     }
 }

src/catalog/src/table_source.rs

@@ -24,10 +24,7 @@ use session::context::QueryContext;
 use snafu::{ensure, OptionExt};
 use table::table::adapter::DfTableProviderAdapter;
 
-use crate::error::{
-    CatalogNotFoundSnafu, QueryAccessDeniedSnafu, Result, SchemaNotFoundSnafu, TableNotExistSnafu,
-};
-use crate::information_schema::InformationSchemaProvider;
+use crate::error::{QueryAccessDeniedSnafu, Result, TableNotExistSnafu};
 use crate::CatalogManagerRef;
 
 pub struct DfTableSourceProvider {
@@ -48,8 +45,8 @@ impl DfTableSourceProvider {
             catalog_manager,
             disallow_cross_schema_query,
             resolved_tables: HashMap::new(),
-            default_catalog: query_ctx.current_catalog(),
-            default_schema: query_ctx.current_schema(),
+            default_catalog: query_ctx.current_catalog().to_owned(),
+            default_schema: query_ctx.current_schema().to_owned(),
         }
     }
 
@@ -104,41 +101,18 @@ impl DfTableSourceProvider {
         let schema_name = table_ref.schema.as_ref();
         let table_name = table_ref.table.as_ref();
 
-        let schema = if schema_name != INFORMATION_SCHEMA_NAME {
-            let catalog = self
-                .catalog_manager
-                .catalog(catalog_name)
-                .await?
-                .context(CatalogNotFoundSnafu { catalog_name })?;
-            catalog
-                .schema(schema_name)
-                .await?
-                .context(SchemaNotFoundSnafu {
-                    catalog: catalog_name,
-                    schema: schema_name,
-                })?
-        } else {
-            let catalog_provider = self
-                .catalog_manager
-                .catalog(catalog_name)
-                .await?
-                .context(CatalogNotFoundSnafu { catalog_name })?;
-            Arc::new(InformationSchemaProvider::new(
-                catalog_name.to_string(),
-                catalog_provider,
-            ))
-        };
-        let table = schema
-            .table(table_name)
+        let table = self
+            .catalog_manager
+            .table(catalog_name, schema_name, table_name)
             .await?
             .with_context(|| TableNotExistSnafu {
                 table: format_full_table_name(catalog_name, schema_name, table_name),
             })?;
 
-        let table = DfTableProviderAdapter::new(table);
-        let table = provider_as_source(Arc::new(table));
-        self.resolved_tables.insert(resolved_name, table.clone());
-        Ok(table)
+        let provider = DfTableProviderAdapter::new(table);
+        let source = provider_as_source(Arc::new(provider));
+        let _ = self.resolved_tables.insert(resolved_name, source.clone());
+        Ok(source)
     }
 }
 
@@ -156,20 +130,20 @@ mod tests {
         let query_ctx = &QueryContext::with("greptime", "public");
 
         let table_provider =
-            DfTableSourceProvider::new(Arc::new(MemoryCatalogManager::default()), true, query_ctx);
+            DfTableSourceProvider::new(MemoryCatalogManager::with_default_setup(), true, query_ctx);
 
         let table_ref = TableReference::Bare {
             table: Cow::Borrowed("table_name"),
         };
         let result = table_provider.resolve_table_ref(table_ref);
-        assert!(result.is_ok());
+        let _ = result.unwrap();
 
         let table_ref = TableReference::Partial {
             schema: Cow::Borrowed("public"),
             table: Cow::Borrowed("table_name"),
         };
         let result = table_provider.resolve_table_ref(table_ref);
-        assert!(result.is_ok());
+        let _ = result.unwrap();
 
         let table_ref = TableReference::Partial {
             schema: Cow::Borrowed("wrong_schema"),
@@ -184,7 +158,7 @@ mod tests {
             table: Cow::Borrowed("table_name"),
         };
         let result = table_provider.resolve_table_ref(table_ref);
-        assert!(result.is_ok());
+        let _ = result.unwrap();
 
         let table_ref = TableReference::Full {
             catalog: Cow::Borrowed("wrong_catalog"),
@@ -198,14 +172,14 @@ mod tests {
             schema: Cow::Borrowed("information_schema"),
             table: Cow::Borrowed("columns"),
         };
-        assert!(table_provider.resolve_table_ref(table_ref).is_ok());
+        let _ = table_provider.resolve_table_ref(table_ref).unwrap();
 
         let table_ref = TableReference::Full {
             catalog: Cow::Borrowed("greptime"),
             schema: Cow::Borrowed("information_schema"),
             table: Cow::Borrowed("columns"),
         };
-        assert!(table_provider.resolve_table_ref(table_ref).is_ok());
+        let _ = table_provider.resolve_table_ref(table_ref).unwrap();
 
         let table_ref = TableReference::Full {
             catalog: Cow::Borrowed("dummy"),

src/catalog/src/tables.rs

@@ -14,49 +14,17 @@
 
 // The `tables` table in system catalog keeps a record of all tables created by user.
 
-use std::any::Any;
 use std::sync::Arc;
 
-use async_trait::async_trait;
-use common_catalog::consts::{INFORMATION_SCHEMA_NAME, SYSTEM_CATALOG_TABLE_NAME};
-use snafu::ResultExt;
 use table::metadata::TableId;
-use table::{Table, TableRef};
 
-use crate::error::{self, Error, InsertCatalogRecordSnafu, Result as CatalogResult};
-use crate::system::{
-    build_schema_insert_request, build_table_deletion_request, build_table_insert_request,
-    SystemCatalogTable,
-};
-use crate::{CatalogProvider, DeregisterTableRequest, SchemaProvider, SchemaProviderRef};
+use crate::system::SystemCatalogTable;
+use crate::DeregisterTableRequest;
 
 pub struct InformationSchema {
     pub system: Arc<SystemCatalogTable>,
 }
 
-#[async_trait]
-impl SchemaProvider for InformationSchema {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    async fn table_names(&self) -> Result<Vec<String>, Error> {
-        Ok(vec![SYSTEM_CATALOG_TABLE_NAME.to_string()])
-    }
-
-    async fn table(&self, name: &str) -> Result<Option<TableRef>, Error> {
-        if name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME) {
-            Ok(Some(self.system.clone()))
-        } else {
-            Ok(None)
-        }
-    }
-
-    async fn table_exist(&self, name: &str) -> Result<bool, Error> {
-        Ok(name.eq_ignore_ascii_case(SYSTEM_CATALOG_TABLE_NAME))
-    }
-}
-
 pub struct SystemCatalog {
     pub information_schema: Arc<InformationSchema>,
 }
@@ -79,27 +47,21 @@ impl SystemCatalog {
         table_id: TableId,
         engine: String,
     ) -> crate::error::Result<usize> {
-        let request = build_table_insert_request(catalog, schema, table_name, table_id, engine);
         self.information_schema
             .system
-            .insert(request)
+            .register_table(catalog, schema, table_name, table_id, engine)
             .await
-            .context(InsertCatalogRecordSnafu)
     }
 
     pub(crate) async fn deregister_table(
         &self,
         request: &DeregisterTableRequest,
         table_id: TableId,
-    ) -> CatalogResult<bool> {
+    ) -> crate::error::Result<()> {
         self.information_schema
             .system
-            .delete(build_table_deletion_request(request, table_id))
+            .deregister_table(request, table_id)
             .await
-            .map(|x| x == 1)
-            .with_context(|_| error::DeregisterTableSnafu {
-                request: request.clone(),
-            })
     }
 
     pub async fn register_schema(
@@ -107,38 +69,9 @@ impl SystemCatalog {
         catalog: String,
         schema: String,
     ) -> crate::error::Result<usize> {
-        let request = build_schema_insert_request(catalog, schema);
         self.information_schema
             .system
-            .insert(request)
+            .register_schema(catalog, schema)
             .await
-            .context(InsertCatalogRecordSnafu)
-    }
-}
-
-#[async_trait::async_trait]
-impl CatalogProvider for SystemCatalog {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    async fn schema_names(&self) -> Result<Vec<String>, Error> {
-        Ok(vec![INFORMATION_SCHEMA_NAME.to_string()])
-    }
-
-    async fn register_schema(
-        &self,
-        _name: String,
-        _schema: SchemaProviderRef,
-    ) -> Result<Option<SchemaProviderRef>, Error> {
-        panic!("System catalog does not support registering schema!")
-    }
-
-    async fn schema(&self, name: &str) -> Result<Option<Arc<dyn SchemaProvider>>, Error> {
-        if name.eq_ignore_ascii_case(INFORMATION_SCHEMA_NAME) {
-            Ok(Some(self.information_schema.clone()))
-        } else {
-            Ok(None)
-        }
     }
 }

src/catalog/tests/local_catalog_tests.rs

@@ -24,7 +24,6 @@ mod tests {
     use mito::config::EngineConfig;
     use table::engine::manager::MemoryTableEngineManager;
     use table::table::numbers::NumbersTable;
-    use table::TableRef;
     use tokio::sync::Mutex;
 
     async fn create_local_catalog_manager(
@@ -49,13 +48,12 @@ mod tests {
         // register table
         let table_name = "test_table";
         let table_id = 42;
-        let table = Arc::new(NumbersTable::new(table_id));
         let request = RegisterTableRequest {
             catalog: DEFAULT_CATALOG_NAME.to_string(),
             schema: DEFAULT_SCHEMA_NAME.to_string(),
             table_name: table_name.to_string(),
             table_id,
-            table: table.clone(),
+            table: NumbersTable::table(table_id),
         };
         assert!(catalog_manager.register_table(request).await.unwrap());
 
@@ -89,7 +87,7 @@ mod tests {
             schema: DEFAULT_SCHEMA_NAME.to_string(),
             table_name: "test_table".to_string(),
             table_id: 42,
-            table: Arc::new(NumbersTable::new(42)),
+            table: NumbersTable::table(42),
         };
         assert!(catalog_manager
             .register_table(request.clone())
@@ -105,7 +103,7 @@ mod tests {
                 schema: DEFAULT_SCHEMA_NAME.to_string(),
                 table_name: "test_table".to_string(),
                 table_id: 43,
-                table: Arc::new(NumbersTable::new(43)),
+                table: NumbersTable::table(43),
             })
             .await
             .unwrap_err();
@@ -124,7 +122,7 @@ mod tests {
             rt.block_on(async { create_local_catalog_manager().await.unwrap() });
         let catalog_manager = Arc::new(catalog_manager);
 
-        let succeed: Arc<Mutex<Option<TableRef>>> = Arc::new(Mutex::new(None));
+        let succeed = Arc::new(Mutex::new(None));
 
         let mut handles = Vec::with_capacity(8);
         for i in 0..8 {
@@ -132,20 +130,21 @@ mod tests {
             let succeed = succeed.clone();
             let handle = rt.spawn(async move {
                 let table_id = 42 + i;
-                let table = Arc::new(NumbersTable::new(table_id));
+                let table = NumbersTable::table(table_id);
+                let table_info = table.table_info();
                 let req = RegisterTableRequest {
                     catalog: DEFAULT_CATALOG_NAME.to_string(),
                     schema: DEFAULT_SCHEMA_NAME.to_string(),
                     table_name: "test_table".to_string(),
                     table_id,
-                    table: table.clone(),
+                    table,
                 };
                 match catalog.register_table(req).await {
                     Ok(res) => {
                         if res {
                             let mut succeed = succeed.lock().await;
                             info!("Successfully registered table: {}", table_id);
-                            *succeed = Some(table);
+                            *succeed = Some(table_info);
                         }
                     }
                     Err(_) => {
@@ -161,7 +160,7 @@ mod tests {
                 handle.await.unwrap();
             }
             let guard = succeed.lock().await;
-            let table = guard.as_ref().unwrap();
+            let table_info = guard.as_ref().unwrap();
             let table_registered = catalog_manager
                 .table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "test_table")
                 .await
@@ -169,7 +168,7 @@ mod tests {
                 .unwrap();
             assert_eq!(
                 table_registered.table_info().ident.table_id,
-                table.table_info().ident.table_id
+                table_info.ident.table_id
             );
         });
     }

src/catalog/tests/remote_catalog_tests.rs

@@ -19,155 +19,142 @@ mod tests {
     use std::assert_matches::assert_matches;
     use std::collections::HashSet;
     use std::sync::Arc;
+    use std::time::Duration;
 
-    use catalog::helper::{CatalogKey, CatalogValue, SchemaKey, SchemaValue};
-    use catalog::remote::mock::{MockKvBackend, MockTableEngine};
-    use catalog::remote::{
-        CachedMetaKvBackend, KvBackend, KvBackendRef, RemoteCatalogManager, RemoteCatalogProvider,
-        RemoteSchemaProvider,
+    use catalog::remote::mock::MockTableEngine;
+    use catalog::remote::region_alive_keeper::RegionAliveKeepers;
+    use catalog::remote::{CachedMetaKvBackend, RemoteCatalogManager};
+    use catalog::{CatalogManager, RegisterSchemaRequest, RegisterTableRequest};
+    use common_catalog::consts::{
+        DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, MITO_ENGINE,
     };
-    use catalog::{CatalogManager, RegisterTableRequest};
-    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
+    use common_meta::helper::CatalogValue;
+    use common_meta::ident::TableIdent;
+    use common_meta::key::catalog_name::CatalogNameKey;
+    use common_meta::key::TableMetadataManager;
+    use common_meta::kv_backend::memory::MemoryKvBackend;
+    use common_meta::kv_backend::KvBackend;
+    use common_meta::rpc::store::{CompareAndPutRequest, PutRequest};
     use datatypes::schema::RawSchema;
-    use futures_util::StreamExt;
     use table::engine::manager::{MemoryTableEngineManager, TableEngineManagerRef};
     use table::engine::{EngineContext, TableEngineRef};
     use table::requests::CreateTableRequest;
+    use table::test_util::EmptyTable;
+    use tokio::time::Instant;
 
-    #[tokio::test]
-    async fn test_backend() {
-        common_telemetry::init_default_ut_logging();
-        let backend = MockKvBackend::default();
+    struct TestingComponents {
+        catalog_manager: Arc<RemoteCatalogManager>,
+        table_engine_manager: TableEngineManagerRef,
+        region_alive_keepers: Arc<RegionAliveKeepers>,
+    }
 
-        let default_catalog_key = CatalogKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+    impl TestingComponents {
+        fn table_engine(&self) -> TableEngineRef {
+            self.table_engine_manager.engine(MITO_ENGINE).unwrap()
         }
-        .to_string();
-
-        backend
-            .set(
-                default_catalog_key.as_bytes(),
-                &CatalogValue {}.as_bytes().unwrap(),
-            )
-            .await
-            .unwrap();
-
-        let schema_key = SchemaKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
-        }
-        .to_string();
-        backend
-            .set(schema_key.as_bytes(), &SchemaValue {}.as_bytes().unwrap())
-            .await
-            .unwrap();
-
-        let mut iter = backend.range("__c-".as_bytes());
-        let mut res = HashSet::new();
-        while let Some(r) = iter.next().await {
-            let kv = r.unwrap();
-            res.insert(String::from_utf8_lossy(&kv.0).to_string());
-        }
-        assert_eq!(
-            vec!["__c-greptime".to_string()],
-            res.into_iter().collect::<Vec<_>>()
-        );
     }
 
     #[tokio::test]
     async fn test_cached_backend() {
-        common_telemetry::init_default_ut_logging();
-        let backend = CachedMetaKvBackend::wrap(Arc::new(MockKvBackend::default()));
+        let backend = CachedMetaKvBackend::wrap(Arc::new(MemoryKvBackend::default()));
 
-        let default_catalog_key = CatalogKey {
-            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
-        }
-        .to_string();
+        let default_catalog_key = CatalogNameKey::new(DEFAULT_CATALOG_NAME).to_string();
+
+        let req = PutRequest::new()
+            .with_key(default_catalog_key.as_bytes())
+            .with_value(CatalogValue.as_bytes().unwrap());
+        backend.put(req).await.unwrap();
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        let _ = ret.unwrap();
+
+        let req = CompareAndPutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_expect(CatalogValue.as_bytes().unwrap())
+            .with_value(b"123".to_vec());
+        let _ = backend.compare_and_put(req).await.unwrap();
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        assert_eq!(b"123", ret.as_ref().unwrap().value.as_slice());
+
+        let req = PutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_value(b"1234".to_vec());
+        let _ = backend.put(req).await;
+
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
+        assert_eq!(b"1234", ret.unwrap().value.as_slice());
 
         backend
-            .set(
-                default_catalog_key.as_bytes(),
-                &CatalogValue {}.as_bytes().unwrap(),
-            )
+            .delete(b"__catalog_name/greptime", false)
             .await
             .unwrap();
 
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert!(ret.is_some());
-
-        let _ = backend
-            .compare_and_set(
-                b"__c-greptime",
-                &CatalogValue {}.as_bytes().unwrap(),
-                b"123",
-            )
-            .await
-            .unwrap();
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert!(ret.is_some());
-        assert_eq!(&b"123"[..], &(ret.as_ref().unwrap().1));
-
-        let _ = backend.set(b"__c-greptime", b"1234").await;
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
-        assert!(ret.is_some());
-        assert_eq!(&b"1234"[..], &(ret.as_ref().unwrap().1));
-
-        backend.delete(b"__c-greptime").await.unwrap();
-
-        let ret = backend.get(b"__c-greptime").await.unwrap();
+        let ret = backend.get(b"__catalog_name/greptime").await.unwrap();
         assert!(ret.is_none());
     }
 
-    async fn prepare_components(
-        node_id: u64,
-    ) -> (
-        KvBackendRef,
-        TableEngineRef,
-        Arc<RemoteCatalogManager>,
-        TableEngineManagerRef,
-    ) {
-        let cached_backend = Arc::new(CachedMetaKvBackend::wrap(
-            Arc::new(MockKvBackend::default()),
-        ));
+    async fn prepare_components(node_id: u64) -> TestingComponents {
+        let backend = Arc::new(MemoryKvBackend::default());
+
+        let req = PutRequest::new()
+            .with_key(b"__catalog_name/greptime".to_vec())
+            .with_value(b"".to_vec());
+        backend.put(req).await.unwrap();
+
+        let req = PutRequest::new()
+            .with_key(b"__schema_name/greptime-public".to_vec())
+            .with_value(b"".to_vec());
+        backend.put(req).await.unwrap();
+
+        let cached_backend = Arc::new(CachedMetaKvBackend::wrap(backend));
 
         let table_engine = Arc::new(MockTableEngine::default());
         let engine_manager = Arc::new(MemoryTableEngineManager::alias(
             MITO_ENGINE.to_string(),
-            table_engine.clone(),
+            table_engine,
         ));
 
-        let catalog_manager =
-            RemoteCatalogManager::new(engine_manager.clone(), node_id, cached_backend.clone());
+        let region_alive_keepers = Arc::new(RegionAliveKeepers::new(engine_manager.clone(), 5000));
+
+        let catalog_manager = RemoteCatalogManager::new(
+            engine_manager.clone(),
+            node_id,
+            region_alive_keepers.clone(),
+            Arc::new(TableMetadataManager::new(cached_backend)),
+        );
         catalog_manager.start().await.unwrap();
 
-        (
-            cached_backend,
-            table_engine,
-            Arc::new(catalog_manager),
-            engine_manager as Arc<_>,
-        )
+        TestingComponents {
+            catalog_manager: Arc::new(catalog_manager),
+            table_engine_manager: engine_manager,
+            region_alive_keepers,
+        }
     }
 
     #[tokio::test]
     async fn test_remote_catalog_default() {
         common_telemetry::init_default_ut_logging();
         let node_id = 42;
-        let (_, _, catalog_manager, _) = prepare_components(node_id).await;
+        let TestingComponents {
+            catalog_manager, ..
+        } = prepare_components(node_id).await;
         assert_eq!(
             vec![DEFAULT_CATALOG_NAME.to_string()],
             catalog_manager.catalog_names().await.unwrap()
         );
 
-        let default_catalog = catalog_manager
-            .catalog(DEFAULT_CATALOG_NAME)
+        let mut schema_names = catalog_manager
+            .schema_names(DEFAULT_CATALOG_NAME)
             .await
-            .unwrap()
             .unwrap();
+        schema_names.sort_unstable();
         assert_eq!(
-            vec![DEFAULT_SCHEMA_NAME.to_string()],
-            default_catalog.schema_names().await.unwrap()
+            vec![
+                INFORMATION_SCHEMA_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string()
+            ],
+            schema_names
         );
     }
 
@@ -175,14 +162,16 @@ mod tests {
     async fn test_remote_catalog_register_nonexistent() {
         common_telemetry::init_default_ut_logging();
         let node_id = 42;
-        let (_, table_engine, catalog_manager, _) = prepare_components(node_id).await;
+        let components = prepare_components(node_id).await;
+
         // register a new table with an nonexistent catalog
         let catalog_name = "nonexistent_catalog".to_string();
         let schema_name = "nonexistent_schema".to_string();
         let table_name = "fail_table".to_string();
         // this schema has no effect
         let table_schema = RawSchema::new(vec![]);
-        let table = table_engine
+        let table = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -208,7 +197,7 @@ mod tests {
             table_id: 1,
             table,
         };
-        let res = catalog_manager.register_table(reg_req).await;
+        let res = components.catalog_manager.register_table(reg_req).await;
 
         // because nonexistent_catalog does not exist yet.
         assert_matches!(
@@ -220,23 +209,21 @@ mod tests {
     #[tokio::test]
     async fn test_register_table() {
         let node_id = 42;
-        let (_, table_engine, catalog_manager, _) = prepare_components(node_id).await;
-        let default_catalog = catalog_manager
-            .catalog(DEFAULT_CATALOG_NAME)
+        let components = prepare_components(node_id).await;
+        let mut schema_names = components
+            .catalog_manager
+            .schema_names(DEFAULT_CATALOG_NAME)
             .await
-            .unwrap()
             .unwrap();
+        schema_names.sort_unstable();
         assert_eq!(
-            vec![DEFAULT_SCHEMA_NAME.to_string()],
-            default_catalog.schema_names().await.unwrap()
+            vec![
+                INFORMATION_SCHEMA_NAME.to_string(),
+                DEFAULT_SCHEMA_NAME.to_string(),
+            ],
+            schema_names
         );
 
-        let default_schema = default_catalog
-            .schema(DEFAULT_SCHEMA_NAME)
-            .await
-            .unwrap()
-            .unwrap();
-
         // register a new table with an nonexistent catalog
         let catalog_name = DEFAULT_CATALOG_NAME.to_string();
         let schema_name = DEFAULT_SCHEMA_NAME.to_string();
@@ -244,7 +231,8 @@ mod tests {
         let table_id = 1;
         // this schema has no effect
         let table_schema = RawSchema::new(vec![]);
-        let table = table_engine
+        let table = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -270,40 +258,46 @@ mod tests {
             table_id,
             table,
         };
-        assert!(catalog_manager.register_table(reg_req).await.unwrap());
+        assert!(components
+            .catalog_manager
+            .register_table(reg_req)
+            .await
+            .unwrap());
         assert_eq!(
             vec![table_name],
-            default_schema.table_names().await.unwrap()
+            components
+                .catalog_manager
+                .table_names(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME)
+                .await
+                .unwrap()
         );
     }
 
     #[tokio::test]
     async fn test_register_catalog_schema_table() {
         let node_id = 42;
-        let (backend, table_engine, catalog_manager, engine_manager) =
-            prepare_components(node_id).await;
+        let components = prepare_components(node_id).await;
 
         let catalog_name = "test_catalog".to_string();
         let schema_name = "nonexistent_schema".to_string();
-        let catalog = Arc::new(RemoteCatalogProvider::new(
-            catalog_name.clone(),
-            backend.clone(),
-            engine_manager.clone(),
-            node_id,
-        ));
 
         // register catalog to catalog manager
-        CatalogManager::register_catalog(&*catalog_manager, catalog_name.clone(), catalog)
+        assert!(components
+            .catalog_manager
+            .clone()
+            .register_catalog(catalog_name.clone())
             .await
-            .unwrap();
+            .is_ok());
         assert_eq!(
-            HashSet::<String>::from_iter(
-                vec![DEFAULT_CATALOG_NAME.to_string(), catalog_name.clone()].into_iter()
-            ),
-            HashSet::from_iter(catalog_manager.catalog_names().await.unwrap().into_iter())
+            HashSet::<String>::from_iter(vec![
+                DEFAULT_CATALOG_NAME.to_string(),
+                catalog_name.clone()
+            ]),
+            HashSet::from_iter(components.catalog_manager.catalog_names().await.unwrap())
         );
 
-        let table_to_register = table_engine
+        let table_to_register = components
+            .table_engine()
             .create_table(
                 &EngineContext {},
                 CreateTableRequest {
@@ -332,41 +326,128 @@ mod tests {
         };
         // this register will fail since schema does not exist yet
         assert_matches!(
-            catalog_manager
+            components
+                .catalog_manager
                 .register_table(reg_req.clone())
                 .await
                 .unwrap_err(),
             catalog::error::Error::SchemaNotFound { .. }
         );
 
-        let new_catalog = catalog_manager
-            .catalog(&catalog_name)
+        let register_schema_request = RegisterSchemaRequest {
+            catalog: catalog_name.to_string(),
+            schema: schema_name.to_string(),
+        };
+        assert!(components
+            .catalog_manager
+            .register_schema(register_schema_request)
             .await
-            .unwrap()
-            .expect("catalog should exist since it's already registered");
-        let schema = Arc::new(RemoteSchemaProvider::new(
-            catalog_name.clone(),
-            schema_name.clone(),
-            node_id,
-            engine_manager,
-            backend.clone(),
-        ));
-
-        let prev = new_catalog
-            .register_schema(schema_name.clone(), schema.clone())
+            .expect("Register schema should not fail"));
+        assert!(components
+            .catalog_manager
+            .register_table(reg_req)
             .await
-            .expect("Register schema should not fail");
-        assert!(prev.is_none());
-        assert!(catalog_manager.register_table(reg_req).await.unwrap());
+            .unwrap());
 
         assert_eq!(
-            HashSet::from([schema_name.clone()]),
-            new_catalog
-                .schema_names()
+            HashSet::from([schema_name.clone(), INFORMATION_SCHEMA_NAME.to_string()]),
+            components
+                .catalog_manager
+                .schema_names(&catalog_name)
                 .await
                 .unwrap()
                 .into_iter()
                 .collect()
         )
     }
+
+    #[tokio::test]
+    async fn test_register_table_before_and_after_region_alive_keeper_started() {
+        let components = prepare_components(42).await;
+        let catalog_manager = &components.catalog_manager;
+        let region_alive_keepers = &components.region_alive_keepers;
+
+        let table_before = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_before".to_string(),
+            table_id: 1,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_before.catalog.clone(),
+            schema: table_before.schema.clone(),
+            table_name: table_before.table.clone(),
+            table_id: table_before.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_before.table_id,
+                catalog_name: table_before.catalog.clone(),
+                schema_name: table_before.schema.clone(),
+                table_name: table_before.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_before.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 29);
+        // assert region alive countdown is not started
+        assert!(deadline > far_future);
+
+        region_alive_keepers.start().await;
+
+        let table_after = TableIdent {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            table: "table_after".to_string(),
+            table_id: 2,
+            engine: MITO_ENGINE.to_string(),
+        };
+        let request = RegisterTableRequest {
+            catalog: table_after.catalog.clone(),
+            schema: table_after.schema.clone(),
+            table_name: table_after.table.clone(),
+            table_id: table_after.table_id,
+            table: Arc::new(EmptyTable::new(CreateTableRequest {
+                id: table_after.table_id,
+                catalog_name: table_after.catalog.clone(),
+                schema_name: table_after.schema.clone(),
+                table_name: table_after.table.clone(),
+                desc: None,
+                schema: RawSchema::new(vec![]),
+                region_numbers: vec![0],
+                primary_key_indices: vec![],
+                create_if_not_exists: false,
+                table_options: Default::default(),
+                engine: MITO_ENGINE.to_string(),
+            })),
+        };
+        assert!(catalog_manager.register_table(request).await.unwrap());
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_after.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered after [RegionAliveKeepers] started
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+
+        let keeper = region_alive_keepers
+            .find_keeper(table_before.table_id)
+            .await
+            .unwrap();
+        let deadline = keeper.deadline(0).await.unwrap();
+        // assert countdown is started for the table registered before [RegionAliveKeepers] started, too
+        assert!(deadline <= Instant::now() + Duration::from_secs(20));
+    }
 }

src/client/Cargo.toml

@@ -8,21 +8,21 @@ license.workspace = true
 testing = []
 
 [dependencies]
-api = { path = "../api" }
+api = { workspace = true }
 arrow-flight.workspace = true
 async-stream.workspace = true
-common-base = { path = "../common/base" }
-common-catalog = { path = "../common/catalog" }
-common-error = { path = "../common/error" }
-common-grpc = { path = "../common/grpc" }
-common-grpc-expr = { path = "../common/grpc-expr" }
-common-query = { path = "../common/query" }
-common-recordbatch = { path = "../common/recordbatch" }
-common-time = { path = "../common/time" }
-common-meta = { path = "../common/meta" }
-common-telemetry = { path = "../common/telemetry" }
+common-base = { workspace = true }
+common-catalog = { workspace = true }
+common-error = { workspace = true }
+common-grpc = { workspace = true }
+common-meta = { workspace = true }
+common-query = { workspace = true }
+common-recordbatch = { workspace = true }
+common-telemetry = { workspace = true }
+common-time = { workspace = true }
 datafusion.workspace = true
-datatypes = { path = "../datatypes" }
+datatypes = { workspace = true }
+derive_builder.workspace = true
 enum_dispatch = "0.3"
 futures-util.workspace = true
 moka = { version = "0.9", features = ["future"] }
@@ -35,11 +35,13 @@ tokio.workspace = true
 tonic.workspace = true
 
 [dev-dependencies]
-datanode = { path = "../datanode" }
-substrait = { path = "../common/substrait" }
+common-grpc-expr = { workspace = true }
+datanode = { workspace = true }
+derive-new = "0.5"
+prost.workspace = true
+substrait = { workspace = true }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-prost.workspace = true
 
 [dev-dependencies.substrait_proto]
 package = "substrait"

src/client/examples/logical.rs

@@ -73,7 +73,7 @@ async fn run() {
 
     let logical = mock_logical_plan();
     event!(Level::INFO, "plan size: {:#?}", logical.len());
-    let result = db.logical_plan(logical).await.unwrap();
+    let result = db.logical_plan(logical, None).await.unwrap();
 
     event!(Level::INFO, "result: {:#?}", result);
 }

src/client/examples/stream_ingest.rs

@@ -0,0 +1,182 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::*;
+use client::{Client, Database, DEFAULT_SCHEMA_NAME};
+use derive_new::new;
+use tracing::{error, info};
+
+fn main() {
+    tracing::subscriber::set_global_default(tracing_subscriber::FmtSubscriber::builder().finish())
+        .unwrap();
+
+    run();
+}
+
+#[tokio::main]
+async fn run() {
+    let greptimedb_endpoint =
+        std::env::var("GREPTIMEDB_ENDPOINT").unwrap_or_else(|_| "localhost:4001".to_owned());
+
+    let greptimedb_dbname =
+        std::env::var("GREPTIMEDB_DBNAME").unwrap_or_else(|_| DEFAULT_SCHEMA_NAME.to_owned());
+
+    let grpc_client = Client::with_urls(vec![&greptimedb_endpoint]);
+
+    let client = Database::new_with_dbname(greptimedb_dbname, grpc_client);
+
+    let stream_inserter = client.streaming_inserter().unwrap();
+
+    if let Err(e) = stream_inserter
+        .insert(vec![to_insert_request(weather_records_1())])
+        .await
+    {
+        error!("Error: {e}");
+    }
+
+    if let Err(e) = stream_inserter
+        .insert(vec![to_insert_request(weather_records_2())])
+        .await
+    {
+        error!("Error: {e}");
+    }
+
+    let result = stream_inserter.finish().await;
+
+    match result {
+        Ok(rows) => {
+            info!("Rows written: {rows}");
+        }
+        Err(e) => {
+            error!("Error: {e}");
+        }
+    };
+}
+
+#[derive(new)]
+struct WeatherRecord {
+    timestamp_millis: i64,
+    collector: String,
+    temperature: f32,
+    humidity: i32,
+}
+
+fn weather_records_1() -> Vec<WeatherRecord> {
+    vec![
+        WeatherRecord::new(1686109527000, "c1".to_owned(), 26.4, 15),
+        WeatherRecord::new(1686023127000, "c1".to_owned(), 29.3, 20),
+        WeatherRecord::new(1685936727000, "c1".to_owned(), 31.8, 13),
+        WeatherRecord::new(1686109527000, "c2".to_owned(), 20.4, 67),
+        WeatherRecord::new(1686023127000, "c2".to_owned(), 18.0, 74),
+        WeatherRecord::new(1685936727000, "c2".to_owned(), 19.2, 81),
+    ]
+}
+
+fn weather_records_2() -> Vec<WeatherRecord> {
+    vec![
+        WeatherRecord::new(1686109527001, "c3".to_owned(), 26.4, 15),
+        WeatherRecord::new(1686023127002, "c3".to_owned(), 29.3, 20),
+        WeatherRecord::new(1685936727003, "c3".to_owned(), 31.8, 13),
+        WeatherRecord::new(1686109527004, "c4".to_owned(), 20.4, 67),
+        WeatherRecord::new(1686023127005, "c4".to_owned(), 18.0, 74),
+        WeatherRecord::new(1685936727006, "c4".to_owned(), 19.2, 81),
+    ]
+}
+
+/// This function generates some random data and bundle them into a
+/// `InsertRequest`.
+///
+/// Data structure:
+///
+/// - `ts`: a timestamp column
+/// - `collector`: a tag column
+/// - `temperature`: a value field of f32
+/// - `humidity`: a value field of i32
+///
+fn to_insert_request(records: Vec<WeatherRecord>) -> InsertRequest {
+    // convert records into columns
+    let rows = records.len();
+
+    // transpose records into columns
+    let (timestamp_millis, collectors, temp, humidity) = records.into_iter().fold(
+        (
+            Vec::with_capacity(rows),
+            Vec::with_capacity(rows),
+            Vec::with_capacity(rows),
+            Vec::with_capacity(rows),
+        ),
+        |mut acc, rec| {
+            acc.0.push(rec.timestamp_millis);
+            acc.1.push(rec.collector);
+            acc.2.push(rec.temperature);
+            acc.3.push(rec.humidity);
+
+            acc
+        },
+    );
+
+    let columns = vec![
+        // timestamp column: `ts`
+        Column {
+            column_name: "ts".to_owned(),
+            values: Some(column::Values {
+                ts_millisecond_values: timestamp_millis,
+                ..Default::default()
+            }),
+            semantic_type: SemanticType::Timestamp as i32,
+            datatype: ColumnDataType::TimestampMillisecond as i32,
+            ..Default::default()
+        },
+        // tag column: collectors
+        Column {
+            column_name: "collector".to_owned(),
+            values: Some(column::Values {
+                string_values: collectors.into_iter().collect(),
+                ..Default::default()
+            }),
+            semantic_type: SemanticType::Tag as i32,
+            datatype: ColumnDataType::String as i32,
+            ..Default::default()
+        },
+        // field column: temperature
+        Column {
+            column_name: "temperature".to_owned(),
+            values: Some(column::Values {
+                f32_values: temp,
+                ..Default::default()
+            }),
+            semantic_type: SemanticType::Field as i32,
+            datatype: ColumnDataType::Float32 as i32,
+            ..Default::default()
+        },
+        // field column: humidity
+        Column {
+            column_name: "humidity".to_owned(),
+            values: Some(column::Values {
+                i32_values: humidity,
+                ..Default::default()
+            }),
+            semantic_type: SemanticType::Field as i32,
+            datatype: ColumnDataType::Int32 as i32,
+            ..Default::default()
+        },
+    ];
+
+    InsertRequest {
+        table_name: "weather_demo".to_owned(),
+        columns,
+        row_count: rows as u32,
+        ..Default::default()
+    }
+}

src/client/src/client.rs

@@ -17,6 +17,7 @@ use std::sync::Arc;
 use api::v1::greptime_database_client::GreptimeDatabaseClient;
 use api::v1::health_check_client::HealthCheckClient;
 use api::v1::prometheus_gateway_client::PrometheusGatewayClient;
+use api::v1::region::region_client::RegionClient as PbRegionClient;
 use api::v1::HealthCheckRequest;
 use arrow_flight::flight_service_client::FlightServiceClient;
 use common_grpc::channel_manager::ChannelManager;
@@ -82,11 +83,6 @@ impl Client {
         Default::default()
     }
 
-    pub fn with_manager(channel_manager: ChannelManager) -> Self {
-        let inner = Arc::new(Inner::with_manager(channel_manager));
-        Self { inner }
-    }
-
     pub fn with_urls<U, A>(urls: A) -> Self
     where
         U: AsRef<str>,
@@ -157,6 +153,11 @@ impl Client {
         })
     }
 
+    pub(crate) fn raw_region_client(&self) -> Result<PbRegionClient<Channel>> {
+        let (_, channel) = self.find_channel()?;
+        Ok(PbRegionClient::new(channel))
+    }
+
     pub fn make_prometheus_gateway_client(&self) -> Result<PrometheusGatewayClient<Channel>> {
         let (_, channel) = self.find_channel()?;
         Ok(PrometheusGatewayClient::new(channel))
@@ -165,7 +166,7 @@ impl Client {
     pub async fn health_check(&self) -> Result<()> {
         let (_, channel) = self.find_channel()?;
         let mut client = HealthCheckClient::new(channel);
-        client.health_check(HealthCheckRequest {}).await?;
+        let _ = client.health_check(HealthCheckRequest {}).await?;
         Ok(())
     }
 }

src/client/src/client_manager.rs

@@ -13,12 +13,10 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
-use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
 use common_meta::peer::Peer;
-use common_telemetry::info;
 use moka::future::{Cache, CacheBuilder};
 
 use crate::Client;
@@ -26,22 +24,11 @@ use crate::Client;
 pub struct DatanodeClients {
     channel_manager: ChannelManager,
     clients: Cache<Peer, Client>,
-    started: Arc<Mutex<bool>>,
 }
 
 impl Default for DatanodeClients {
     fn default() -> Self {
-        // TODO(LFC): Make this channel config configurable.
-        let config = ChannelConfig::new().timeout(Duration::from_secs(8));
-
-        Self {
-            channel_manager: ChannelManager::with_config(config),
-            clients: CacheBuilder::new(1024)
-                .time_to_live(Duration::from_secs(30 * 60))
-                .time_to_idle(Duration::from_secs(5 * 60))
-                .build(),
-            started: Arc::new(Mutex::new(false)),
-        }
+        Self::new(ChannelConfig::new())
     }
 }
 
@@ -54,16 +41,14 @@ impl Debug for DatanodeClients {
 }
 
 impl DatanodeClients {
-    pub fn start(&self) {
-        let mut started = self.started.lock().unwrap();
-        if *started {
-            return;
+    pub fn new(config: ChannelConfig) -> Self {
+        Self {
+            channel_manager: ChannelManager::with_config(config),
+            clients: CacheBuilder::new(1024)
+                .time_to_live(Duration::from_secs(30 * 60))
+                .time_to_idle(Duration::from_secs(5 * 60))
+                .build(),
         }
-
-        self.channel_manager.start_channel_recycle();
-
-        info!("Datanode clients manager is started!");
-        *started = true;
     }
 
     pub async fn get_client(&self, datanode: &Peer) -> Client {

src/client/src/database.rs

@@ -17,26 +17,24 @@ use api::v1::ddl_request::Expr as DdlExpr;
 use api::v1::greptime_request::Request;
 use api::v1::query_request::Query;
 use api::v1::{
-    greptime_response, AffectedRows, AlterExpr, AuthHeader, CreateTableExpr, DdlRequest,
-    DeleteRequest, DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery,
-    QueryRequest, RequestHeader,
+    AlterExpr, AuthHeader, CompactTableExpr, CreateTableExpr, DdlRequest, DeleteRequests,
+    DropTableExpr, FlushTableExpr, GreptimeRequest, InsertRequests, PromRangeQuery, QueryRequest,
+    RequestHeader, RowInsertRequests, TruncateTableExpr,
 };
-use arrow_flight::{FlightData, Ticket};
-use common_error::prelude::*;
-use common_grpc::flight::{flight_messages_to_recordbatches, FlightDecoder, FlightMessage};
+use arrow_flight::Ticket;
+use async_stream::stream;
+use common_error::ext::{BoxedError, ErrorExt};
+use common_grpc::flight::{FlightDecoder, FlightMessage};
 use common_query::Output;
+use common_recordbatch::error::ExternalSnafu;
+use common_recordbatch::RecordBatchStreamAdaptor;
 use common_telemetry::{logging, timer};
-use futures_util::{TryFutureExt, TryStreamExt};
+use futures_util::StreamExt;
 use prost::Message;
 use snafu::{ensure, ResultExt};
-use tokio::sync::mpsc::Sender;
-use tokio::sync::{mpsc, OnceCell};
-use tokio_stream::wrappers::ReceiverStream;
 
-use crate::error::{
-    ConvertFlightDataSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
-};
-use crate::{error, metrics, Client, Result};
+use crate::error::{ConvertFlightDataSnafu, Error, IllegalFlightMessagesSnafu, ServerSnafu};
+use crate::{error, from_grpc_response, metrics, Client, Result, StreamInserter};
 
 #[derive(Clone, Debug, Default)]
 pub struct Database {
@@ -50,7 +48,6 @@ pub struct Database {
     dbname: String,
 
     client: Client,
-    streaming_client: OnceCell<Sender<GreptimeRequest>>,
     ctx: FlightContext,
 }
 
@@ -62,7 +59,6 @@ impl Database {
             schema: schema.into(),
             dbname: "".to_string(),
             client,
-            streaming_client: OnceCell::new(),
             ctx: FlightContext::default(),
         }
     }
@@ -80,7 +76,6 @@ impl Database {
             schema: "".to_string(),
             dbname: dbname.into(),
             client,
-            streaming_client: OnceCell::new(),
             ctx: FlightContext::default(),
         }
     }
@@ -120,76 +115,81 @@ impl Database {
         self.handle(Request::Inserts(requests)).await
     }
 
-    pub async fn insert_to_stream(&self, requests: InsertRequests) -> Result<()> {
-        let streaming_client = self
-            .streaming_client
-            .get_or_try_init(|| self.client_stream())
-            .await?;
-
-        let request = self.to_rpc_request(Request::Inserts(requests));
-
-        streaming_client.send(request).await.map_err(|e| {
-            error::ClientStreamingSnafu {
-                err_msg: e.to_string(),
-            }
-            .build()
-        })
+    pub async fn row_insert(&self, requests: RowInsertRequests) -> Result<u32> {
+        let _timer = timer!(metrics::METRIC_GRPC_INSERT);
+        self.handle(Request::RowInserts(requests)).await
     }
 
-    pub async fn delete(&self, request: DeleteRequest) -> Result<u32> {
+    pub fn streaming_inserter(&self) -> Result<StreamInserter> {
+        self.streaming_inserter_with_channel_size(65536)
+    }
+
+    pub fn streaming_inserter_with_channel_size(
+        &self,
+        channel_size: usize,
+    ) -> Result<StreamInserter> {
+        let client = self.client.make_database_client()?.inner;
+
+        let stream_inserter = StreamInserter::new(
+            client,
+            self.dbname().to_string(),
+            self.ctx.auth_header.clone(),
+            channel_size,
+        );
+
+        Ok(stream_inserter)
+    }
+
+    pub async fn delete(&self, request: DeleteRequests) -> Result<u32> {
         let _timer = timer!(metrics::METRIC_GRPC_DELETE);
-        self.handle(Request::Delete(request)).await
+        self.handle(Request::Deletes(request)).await
     }
 
     async fn handle(&self, request: Request) -> Result<u32> {
         let mut client = self.client.make_database_client()?.inner;
-        let request = self.to_rpc_request(request);
-        let response = client
-            .handle(request)
-            .await?
-            .into_inner()
-            .response
-            .context(IllegalDatabaseResponseSnafu {
-                err_msg: "GreptimeResponse is empty",
-            })?;
-        let greptime_response::Response::AffectedRows(AffectedRows { value }) = response;
-        Ok(value)
+        let request = self.to_rpc_request(request, None);
+        let response = client.handle(request).await?.into_inner();
+        from_grpc_response(response)
     }
 
     #[inline]
-    fn to_rpc_request(&self, request: Request) -> GreptimeRequest {
+    fn to_rpc_request(&self, request: Request, trace_id: Option<u64>) -> GreptimeRequest {
         GreptimeRequest {
             header: Some(RequestHeader {
                 catalog: self.catalog.clone(),
                 schema: self.schema.clone(),
                 authorization: self.ctx.auth_header.clone(),
                 dbname: self.dbname.clone(),
+                trace_id,
+                span_id: None,
             }),
             request: Some(request),
         }
     }
 
-    async fn client_stream(&self) -> Result<Sender<GreptimeRequest>> {
-        let mut client = self.client.make_database_client()?.inner;
-        let (sender, receiver) = mpsc::channel::<GreptimeRequest>(65536);
-        let receiver = ReceiverStream::new(receiver);
-        client.handle_requests(receiver).await?;
-        Ok(sender)
-    }
-
     pub async fn sql(&self, sql: &str) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_SQL);
-        self.do_get(Request::Query(QueryRequest {
-            query: Some(Query::Sql(sql.to_string())),
-        }))
+        self.do_get(
+            Request::Query(QueryRequest {
+                query: Some(Query::Sql(sql.to_string())),
+            }),
+            None,
+        )
         .await
     }
 
-    pub async fn logical_plan(&self, logical_plan: Vec<u8>) -> Result<Output> {
+    pub async fn logical_plan(
+        &self,
+        logical_plan: Vec<u8>,
+        trace_id: Option<u64>,
+    ) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_LOGICAL_PLAN);
-        self.do_get(Request::Query(QueryRequest {
-            query: Some(Query::LogicalPlan(logical_plan)),
-        }))
+        self.do_get(
+            Request::Query(QueryRequest {
+                query: Some(Query::LogicalPlan(logical_plan)),
+            }),
+            trace_id,
+        )
         .await
     }
 
@@ -201,109 +201,171 @@ impl Database {
         step: &str,
     ) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_PROMQL_RANGE_QUERY);
-        self.do_get(Request::Query(QueryRequest {
-            query: Some(Query::PromRangeQuery(PromRangeQuery {
-                query: promql.to_string(),
-                start: start.to_string(),
-                end: end.to_string(),
-                step: step.to_string(),
-            })),
-        }))
+        self.do_get(
+            Request::Query(QueryRequest {
+                query: Some(Query::PromRangeQuery(PromRangeQuery {
+                    query: promql.to_string(),
+                    start: start.to_string(),
+                    end: end.to_string(),
+                    step: step.to_string(),
+                })),
+            }),
+            None,
+        )
         .await
     }
 
     pub async fn create(&self, expr: CreateTableExpr) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_CREATE_TABLE);
-        self.do_get(Request::Ddl(DdlRequest {
-            expr: Some(DdlExpr::CreateTable(expr)),
-        }))
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::CreateTable(expr)),
+            }),
+            None,
+        )
         .await
     }
 
     pub async fn alter(&self, expr: AlterExpr) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_ALTER);
-        self.do_get(Request::Ddl(DdlRequest {
-            expr: Some(DdlExpr::Alter(expr)),
-        }))
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::Alter(expr)),
+            }),
+            None,
+        )
         .await
     }
 
     pub async fn drop_table(&self, expr: DropTableExpr) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_DROP_TABLE);
-        self.do_get(Request::Ddl(DdlRequest {
-            expr: Some(DdlExpr::DropTable(expr)),
-        }))
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::DropTable(expr)),
+            }),
+            None,
+        )
         .await
     }
 
     pub async fn flush_table(&self, expr: FlushTableExpr) -> Result<Output> {
         let _timer = timer!(metrics::METRIC_GRPC_FLUSH_TABLE);
-        self.do_get(Request::Ddl(DdlRequest {
-            expr: Some(DdlExpr::FlushTable(expr)),
-        }))
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::FlushTable(expr)),
+            }),
+            None,
+        )
         .await
     }
 
-    async fn do_get(&self, request: Request) -> Result<Output> {
+    pub async fn compact_table(&self, expr: CompactTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_COMPACT_TABLE);
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::CompactTable(expr)),
+            }),
+            None,
+        )
+        .await
+    }
+
+    pub async fn truncate_table(&self, expr: TruncateTableExpr) -> Result<Output> {
+        let _timer = timer!(metrics::METRIC_GRPC_TRUNCATE_TABLE);
+        self.do_get(
+            Request::Ddl(DdlRequest {
+                expr: Some(DdlExpr::TruncateTable(expr)),
+            }),
+            None,
+        )
+        .await
+    }
+
+    async fn do_get(&self, request: Request, trace_id: Option<u64>) -> Result<Output> {
         // FIXME(paomian): should be added some labels for metrics
         let _timer = timer!(metrics::METRIC_GRPC_DO_GET);
-        let request = self.to_rpc_request(request);
+        let request = self.to_rpc_request(request, trace_id);
         let request = Ticket {
             ticket: request.encode_to_vec().into(),
         };
 
         let mut client = self.client.make_flight_client()?;
 
-        // TODO(LFC): Streaming get flight data.
-        let flight_data: Vec<FlightData> = client
-            .mut_inner()
-            .do_get(request)
-            .and_then(|response| response.into_inner().try_collect())
-            .await
-            .map_err(|e| {
-                let tonic_code = e.code();
-                let e: error::Error = e.into();
-                let code = e.status_code();
-                let msg = e.to_string();
-                error::ServerSnafu { code, msg }
-                    .fail::<()>()
-                    .map_err(BoxedError::new)
-                    .context(error::FlightGetSnafu {
-                        tonic_code,
-                        addr: client.addr(),
-                    })
-                    .map_err(|error| {
-                        logging::error!(
-                            "Failed to do Flight get, addr: {}, code: {}, source: {}",
-                            client.addr(),
-                            tonic_code,
-                            error
-                        );
-                        error
-                    })
-                    .unwrap_err()
-            })?;
-
-        let decoder = &mut FlightDecoder::default();
-        let flight_messages = flight_data
-            .into_iter()
-            .map(|x| decoder.try_decode(x).context(ConvertFlightDataSnafu))
-            .collect::<Result<Vec<_>>>()?;
-
-        let output = if let Some(FlightMessage::AffectedRows(rows)) = flight_messages.get(0) {
-            ensure!(
-                flight_messages.len() == 1,
-                IllegalFlightMessagesSnafu {
-                    reason: "Expect 'AffectedRows' Flight messages to be one and only!"
-                }
+        let response = client.mut_inner().do_get(request).await.map_err(|e| {
+            let tonic_code = e.code();
+            let e: error::Error = e.into();
+            let code = e.status_code();
+            let msg = e.to_string();
+            let error = Error::FlightGet {
+                tonic_code,
+                addr: client.addr().to_string(),
+                source: BoxedError::new(ServerSnafu { code, msg }.build()),
+            };
+            logging::error!(
+                "Failed to do Flight get, addr: {}, code: {}, source: {}",
+                client.addr(),
+                tonic_code,
+                error
             );
-            Output::AffectedRows(*rows)
-        } else {
-            let recordbatches = flight_messages_to_recordbatches(flight_messages)
-                .context(ConvertFlightDataSnafu)?;
-            Output::RecordBatches(recordbatches)
+            error
+        })?;
+
+        let flight_data_stream = response.into_inner();
+        let mut decoder = FlightDecoder::default();
+
+        let mut flight_message_stream = flight_data_stream.map(move |flight_data| {
+            flight_data
+                .map_err(Error::from)
+                .and_then(|data| decoder.try_decode(data).context(ConvertFlightDataSnafu))
+        });
+
+        let Some(first_flight_message) = flight_message_stream.next().await else {
+            return IllegalFlightMessagesSnafu {
+                reason: "Expect the response not to be empty",
+            }
+            .fail();
         };
-        Ok(output)
+
+        let first_flight_message = first_flight_message?;
+
+        match first_flight_message {
+            FlightMessage::AffectedRows(rows) => {
+                ensure!(
+                    flight_message_stream.next().await.is_none(),
+                    IllegalFlightMessagesSnafu {
+                        reason: "Expect 'AffectedRows' Flight messages to be the one and the only!"
+                    }
+                );
+                Ok(Output::AffectedRows(rows))
+            }
+            FlightMessage::Recordbatch(_) => IllegalFlightMessagesSnafu {
+                reason: "The first flight message cannot be a RecordBatch message",
+            }
+            .fail(),
+            FlightMessage::Schema(schema) => {
+                let stream = Box::pin(stream!({
+                    while let Some(flight_message) = flight_message_stream.next().await {
+                        let flight_message = flight_message
+                            .map_err(BoxedError::new)
+                            .context(ExternalSnafu)?;
+                        let FlightMessage::Recordbatch(record_batch) = flight_message else {
+                            yield IllegalFlightMessagesSnafu {reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"}
+                                        .fail()
+                                        .map_err(BoxedError::new)
+                                        .context(ExternalSnafu);
+                            break;
+                        };
+                        yield Ok(record_batch);
+                    }
+                }));
+                let record_batch_stream = RecordBatchStreamAdaptor {
+                    schema,
+                    stream,
+                    output_ordering: None,
+                };
+                Ok(Output::Stream(Box::pin(record_batch_stream)))
+            }
+        }
     }
 }
 
@@ -314,106 +376,11 @@ pub struct FlightContext {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
-    use api::helper::ColumnDataTypeWrapper;
     use api::v1::auth_header::AuthScheme;
-    use api::v1::{AuthHeader, Basic, Column};
-    use common_grpc::select::{null_mask, values};
-    use common_grpc_expr::column_to_vector;
-    use datatypes::prelude::{Vector, VectorRef};
-    use datatypes::vectors::{
-        BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,
-        Int16Vector, Int32Vector, Int64Vector, Int8Vector, StringVector, UInt16Vector,
-        UInt32Vector, UInt64Vector, UInt8Vector,
-    };
+    use api::v1::{AuthHeader, Basic};
 
     use crate::database::FlightContext;
 
-    #[test]
-    fn test_column_to_vector() {
-        let mut column = create_test_column(Arc::new(BooleanVector::from(vec![true])));
-        column.datatype = -100;
-        let result = column_to_vector(&column, 1);
-        assert!(result.is_err());
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Column datatype error, source: Unknown proto column datatype: -100"
-        );
-
-        macro_rules! test_with_vector {
-            ($vector: expr) => {
-                let vector = Arc::new($vector);
-                let column = create_test_column(vector.clone());
-                let result = column_to_vector(&column, vector.len() as u32).unwrap();
-                assert_eq!(result, vector as VectorRef);
-            };
-        }
-
-        test_with_vector!(BooleanVector::from(vec![Some(true), None, Some(false)]));
-        test_with_vector!(Int8Vector::from(vec![Some(i8::MIN), None, Some(i8::MAX)]));
-        test_with_vector!(Int16Vector::from(vec![
-            Some(i16::MIN),
-            None,
-            Some(i16::MAX)
-        ]));
-        test_with_vector!(Int32Vector::from(vec![
-            Some(i32::MIN),
-            None,
-            Some(i32::MAX)
-        ]));
-        test_with_vector!(Int64Vector::from(vec![
-            Some(i64::MIN),
-            None,
-            Some(i64::MAX)
-        ]));
-        test_with_vector!(UInt8Vector::from(vec![Some(u8::MIN), None, Some(u8::MAX)]));
-        test_with_vector!(UInt16Vector::from(vec![
-            Some(u16::MIN),
-            None,
-            Some(u16::MAX)
-        ]));
-        test_with_vector!(UInt32Vector::from(vec![
-            Some(u32::MIN),
-            None,
-            Some(u32::MAX)
-        ]));
-        test_with_vector!(UInt64Vector::from(vec![
-            Some(u64::MIN),
-            None,
-            Some(u64::MAX)
-        ]));
-        test_with_vector!(Float32Vector::from(vec![
-            Some(f32::MIN),
-            None,
-            Some(f32::MAX)
-        ]));
-        test_with_vector!(Float64Vector::from(vec![
-            Some(f64::MIN),
-            None,
-            Some(f64::MAX)
-        ]));
-        test_with_vector!(BinaryVector::from(vec![
-            Some(b"".to_vec()),
-            None,
-            Some(b"hello".to_vec())
-        ]));
-        test_with_vector!(StringVector::from(vec![Some(""), None, Some("foo"),]));
-        test_with_vector!(DateVector::from(vec![Some(1), None, Some(3)]));
-        test_with_vector!(DateTimeVector::from(vec![Some(4), None, Some(6)]));
-    }
-
-    fn create_test_column(vector: VectorRef) -> Column {
-        let wrapper: ColumnDataTypeWrapper = vector.data_type().try_into().unwrap();
-        Column {
-            column_name: "test".to_string(),
-            semantic_type: 1,
-            values: Some(values(&[vector.clone()]).unwrap()),
-            null_mask: null_mask(&[vector.clone()], vector.len()),
-            datatype: wrapper.datatype() as i32,
-        }
-    }
-
     #[test]
     fn test_flight_ctx() {
         let mut ctx = FlightContext::default();

src/client/src/error.rs

@@ -13,10 +13,11 @@
 // limitations under the License.
 
 use std::any::Any;
-use std::str::FromStr;
 
-use common_error::prelude::*;
-use snafu::Location;
+use common_error::ext::{BoxedError, ErrorExt};
+use common_error::status_code::StatusCode;
+use common_error::{GREPTIME_ERROR_CODE, GREPTIME_ERROR_MSG};
+use snafu::{Location, Snafu};
 use tonic::{Code, Status};
 
 #[derive(Debug, Snafu)]
@@ -34,13 +35,13 @@ pub enum Error {
 
     #[snafu(display("Failed to convert FlightData, source: {}", source))]
     ConvertFlightData {
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc::Error,
     },
 
     #[snafu(display("Column datatype error, source: {}", source))]
     ColumnDataType {
-        #[snafu(backtrace)]
+        location: Location,
         source: api::error::Error,
     },
 
@@ -57,7 +58,7 @@ pub enum Error {
     ))]
     CreateChannel {
         addr: String,
-        #[snafu(backtrace)]
+        location: Location,
         source: common_grpc::error::Error,
     },
 
@@ -85,7 +86,7 @@ impl ErrorExt for Error {
 
             Error::Server { code, .. } => *code,
             Error::FlightGet { source, .. } => source.status_code(),
-            Error::CreateChannel { source, .. } | Error::ConvertFlightData { source } => {
+            Error::CreateChannel { source, .. } | Error::ConvertFlightData { source, .. } => {
                 source.status_code()
             }
             Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected,
@@ -105,11 +106,18 @@ impl From<Status> for Error {
                 .and_then(|v| String::from_utf8(v.as_bytes().to_vec()).ok())
         }
 
-        let code = get_metadata_value(&e, INNER_ERROR_CODE)
-            .and_then(|s| StatusCode::from_str(&s).ok())
+        let code = get_metadata_value(&e, GREPTIME_ERROR_CODE)
+            .and_then(|s| {
+                if let Ok(code) = s.parse::<u32>() {
+                    StatusCode::from_u32(code)
+                } else {
+                    None
+                }
+            })
             .unwrap_or(StatusCode::Unknown);
 
-        let msg = get_metadata_value(&e, INNER_ERROR_MSG).unwrap_or(e.to_string());
+        let msg =
+            get_metadata_value(&e, GREPTIME_ERROR_MSG).unwrap_or_else(|| e.message().to_string());
 
         Self::Server { code, msg }
     }

src/client/src/lib.rs

@@ -15,13 +15,49 @@
 mod client;
 pub mod client_manager;
 mod database;
-mod error;
+pub mod error;
 pub mod load_balance;
 mod metrics;
+pub mod region;
+mod stream_insert;
 
 pub use api;
+use api::v1::greptime_response::Response;
+use api::v1::{AffectedRows, GreptimeResponse};
 pub use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_error::status_code::StatusCode;
+use snafu::OptionExt;
 
 pub use self::client::Client;
 pub use self::database::Database;
 pub use self::error::{Error, Result};
+pub use self::stream_insert::StreamInserter;
+use crate::error::{IllegalDatabaseResponseSnafu, ServerSnafu};
+
+pub fn from_grpc_response(response: GreptimeResponse) -> Result<u32> {
+    let header = response.header.context(IllegalDatabaseResponseSnafu {
+        err_msg: "missing header",
+    })?;
+    let status = header.status.context(IllegalDatabaseResponseSnafu {
+        err_msg: "missing status",
+    })?;
+
+    if StatusCode::is_success(status.status_code) {
+        let res = response.response.context(IllegalDatabaseResponseSnafu {
+            err_msg: "missing response",
+        })?;
+        match res {
+            Response::AffectedRows(AffectedRows { value }) => Ok(value),
+        }
+    } else {
+        let status_code =
+            StatusCode::from_u32(status.status_code).context(IllegalDatabaseResponseSnafu {
+                err_msg: format!("invalid status: {:?}", status),
+            })?;
+        ServerSnafu {
+            code: status_code,
+            msg: status.err_msg,
+        }
+        .fail()
+    }
+}

src/client/src/load_balance.rs

@@ -60,7 +60,7 @@ mod tests {
         let random = Random;
         for _ in 0..100 {
             let peer = random.get_peer(&peers).unwrap();
-            all.contains(peer);
+            assert!(all.contains(peer));
         }
     }
 }

src/client/src/metrics.rs

@@ -22,4 +22,7 @@ pub const METRIC_GRPC_LOGICAL_PLAN: &str = "grpc.logical_plan";
 pub const METRIC_GRPC_ALTER: &str = "grpc.alter";
 pub const METRIC_GRPC_DROP_TABLE: &str = "grpc.drop_table";
 pub const METRIC_GRPC_FLUSH_TABLE: &str = "grpc.flush_table";
+pub const METRIC_GRPC_COMPACT_TABLE: &str = "grpc.compact_table";
+pub const METRIC_GRPC_TRUNCATE_TABLE: &str = "grpc.truncate_table";
 pub const METRIC_GRPC_DO_GET: &str = "grpc.do_get";
+pub(crate) const METRIC_REGION_REQUEST_GRPC: &str = "grpc.region_request";

src/client/src/region.rs

@@ -0,0 +1,146 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::region::{region_request, RegionRequest, RegionRequestHeader, RegionResponse};
+use api::v1::ResponseHeader;
+use common_error::status_code::StatusCode;
+use common_telemetry::timer;
+use snafu::OptionExt;
+
+use crate::error::{IllegalDatabaseResponseSnafu, Result, ServerSnafu};
+use crate::{metrics, Client};
+
+type AffectedRows = u64;
+
+#[derive(Debug)]
+pub struct RegionRequester {
+    trace_id: Option<u64>,
+    span_id: Option<u64>,
+    client: Client,
+}
+
+impl RegionRequester {
+    pub fn new(client: Client) -> Self {
+        // TODO(LFC): Pass in trace_id and span_id from some context when we have it.
+        Self {
+            trace_id: None,
+            span_id: None,
+            client,
+        }
+    }
+
+    pub async fn handle(self, request: region_request::Body) -> Result<AffectedRows> {
+        let request_type = request.as_ref().to_string();
+
+        let request = RegionRequest {
+            header: Some(RegionRequestHeader {
+                trace_id: self.trace_id,
+                span_id: self.span_id,
+            }),
+            body: Some(request),
+        };
+
+        let _timer = timer!(
+            metrics::METRIC_REGION_REQUEST_GRPC,
+            &[("request_type", request_type)]
+        );
+
+        let mut client = self.client.raw_region_client()?;
+
+        let RegionResponse {
+            header,
+            affected_rows,
+        } = client.handle(request).await?.into_inner();
+
+        check_response_header(header)?;
+
+        Ok(affected_rows)
+    }
+}
+
+fn check_response_header(header: Option<ResponseHeader>) -> Result<()> {
+    let status = header
+        .and_then(|header| header.status)
+        .context(IllegalDatabaseResponseSnafu {
+            err_msg: "either response header or status is missing",
+        })?;
+
+    if StatusCode::is_success(status.status_code) {
+        Ok(())
+    } else {
+        let code =
+            StatusCode::from_u32(status.status_code).context(IllegalDatabaseResponseSnafu {
+                err_msg: format!("unknown server status: {:?}", status),
+            })?;
+        ServerSnafu {
+            code,
+            msg: status.err_msg,
+        }
+        .fail()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use api::v1::Status as PbStatus;
+
+    use super::*;
+    use crate::Error::{IllegalDatabaseResponse, Server};
+
+    #[test]
+    fn test_check_response_header() {
+        let result = check_response_header(None);
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader { status: None }));
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: StatusCode::Success as u32,
+                err_msg: "".to_string(),
+            }),
+        }));
+        assert!(result.is_ok());
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: u32::MAX,
+                err_msg: "".to_string(),
+            }),
+        }));
+        assert!(matches!(
+            result.unwrap_err(),
+            IllegalDatabaseResponse { .. }
+        ));
+
+        let result = check_response_header(Some(ResponseHeader {
+            status: Some(PbStatus {
+                status_code: StatusCode::Internal as u32,
+                err_msg: "blabla".to_string(),
+            }),
+        }));
+        let Server { code, msg } = result.unwrap_err() else {
+            unreachable!()
+        };
+        assert_eq!(code, StatusCode::Internal);
+        assert_eq!(msg, "blabla");
+    }
+}

src/client/src/stream_insert.rs

@@ -0,0 +1,118 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::greptime_database_client::GreptimeDatabaseClient;
+use api::v1::greptime_request::Request;
+use api::v1::{
+    AuthHeader, GreptimeRequest, GreptimeResponse, InsertRequest, InsertRequests, RequestHeader,
+    RowInsertRequest, RowInsertRequests,
+};
+use tokio::sync::mpsc;
+use tokio::task::JoinHandle;
+use tokio_stream::wrappers::ReceiverStream;
+use tonic::transport::Channel;
+use tonic::{Response, Status};
+
+use crate::error::{self, Result};
+use crate::from_grpc_response;
+
+/// A structure that provides some methods for streaming data insert.
+///
+/// [`StreamInserter`] cannot be constructed via the `StreamInserter::new` method.
+/// You can use the following way to obtain [`StreamInserter`].
+///
+/// ```ignore
+/// let grpc_client = Client::with_urls(vec!["127.0.0.1:4002"]);
+/// let client = Database::new_with_dbname("db_name", grpc_client);
+/// let stream_inserter = client.streaming_inserter().unwrap();
+/// ```
+///
+/// If you want to see a concrete usage example, please see
+/// [stream_inserter.rs](https://github.com/GreptimeTeam/greptimedb/blob/develop/src/client/examples/stream_ingest.rs).
+pub struct StreamInserter {
+    sender: mpsc::Sender<GreptimeRequest>,
+
+    auth_header: Option<AuthHeader>,
+
+    dbname: String,
+
+    join: JoinHandle<std::result::Result<Response<GreptimeResponse>, Status>>,
+}
+
+impl StreamInserter {
+    pub(crate) fn new(
+        mut client: GreptimeDatabaseClient<Channel>,
+        dbname: String,
+        auth_header: Option<AuthHeader>,
+        channel_size: usize,
+    ) -> StreamInserter {
+        let (send, recv) = tokio::sync::mpsc::channel(channel_size);
+
+        let join: JoinHandle<std::result::Result<Response<GreptimeResponse>, Status>> =
+            tokio::spawn(async move {
+                let recv_stream = ReceiverStream::new(recv);
+                client.handle_requests(recv_stream).await
+            });
+
+        StreamInserter {
+            sender: send,
+            auth_header,
+            dbname,
+            join,
+        }
+    }
+
+    pub async fn insert(&self, requests: Vec<InsertRequest>) -> Result<()> {
+        let inserts = InsertRequests { inserts: requests };
+        let request = self.to_rpc_request(Request::Inserts(inserts));
+
+        self.sender.send(request).await.map_err(|e| {
+            error::ClientStreamingSnafu {
+                err_msg: e.to_string(),
+            }
+            .build()
+        })
+    }
+
+    pub async fn row_insert(&self, requests: Vec<RowInsertRequest>) -> Result<()> {
+        let inserts = RowInsertRequests { inserts: requests };
+        let request = self.to_rpc_request(Request::RowInserts(inserts));
+
+        self.sender.send(request).await.map_err(|e| {
+            error::ClientStreamingSnafu {
+                err_msg: e.to_string(),
+            }
+            .build()
+        })
+    }
+
+    pub async fn finish(self) -> Result<u32> {
+        drop(self.sender);
+
+        let response = self.join.await.unwrap()?;
+        let response = response.into_inner();
+        from_grpc_response(response)
+    }
+
+    fn to_rpc_request(&self, request: Request) -> GreptimeRequest {
+        GreptimeRequest {
+            header: Some(RequestHeader {
+                authorization: self.auth_header.clone(),
+                dbname: self.dbname.clone(),
+                ..Default::default()
+            }),
+            request: Some(request),
+        }
+    }
+}

src/cmd/Cargo.toml

@@ -10,48 +10,61 @@ name = "greptime"
 path = "src/bin/greptime.rs"
 
 [features]
-mem-prof = ["tikv-jemallocator", "tikv-jemalloc-ctl"]
+default = ["metrics-process"]
 tokio-console = ["common-telemetry/tokio-console"]
+metrics-process = ["servers/metrics-process"]
 
 [dependencies]
 anymap = "1.0.0-beta.2"
-catalog = { path = "../catalog" }
+async-trait.workspace = true
+auth.workspace = true
+catalog = { workspace = true }
+chrono.workspace = true
 clap = { version = "3.1", features = ["derive"] }
-client = { path = "../client" }
-common-base = { path = "../common/base" }
-common-error = { path = "../common/error" }
-common-query = { path = "../common/query" }
-common-recordbatch = { path = "../common/recordbatch" }
-common-telemetry = { path = "../common/telemetry", features = [
+client = { workspace = true }
+common-base = { workspace = true }
+common-error = { workspace = true }
+common-meta = { workspace = true }
+common-query = { workspace = true }
+common-recordbatch = { workspace = true }
+common-telemetry = { workspace = true, features = [
     "deadlock_detection",
 ] }
 config = "0.13"
-datanode = { path = "../datanode" }
+datanode = { workspace = true }
+datatypes = { workspace = true }
 either = "1.8"
-frontend = { path = "../frontend" }
+etcd-client.workspace = true
+frontend = { workspace = true }
 futures.workspace = true
-meta-client = { path = "../meta-client" }
-meta-srv = { path = "../meta-srv" }
+meta-client = { workspace = true }
+meta-srv = { workspace = true }
 metrics.workspace = true
 nu-ansi-term = "0.46"
-partition = { path = "../partition" }
-query = { path = "../query" }
+partition = { workspace = true }
+prost.workspace = true
+query = { workspace = true }
+rand.workspace = true
 rustyline = "10.1"
 serde.workspace = true
-servers = { path = "../servers" }
-session = { path = "../session" }
+servers = { workspace = true }
+session = { workspace = true }
 snafu.workspace = true
-substrait = { path = "../common/substrait" }
-tikv-jemalloc-ctl = { version = "0.5", optional = true }
-tikv-jemallocator = { version = "0.5", optional = true }
+substrait = { workspace = true }
+table = { workspace = true }
 tokio.workspace = true
 
+[target.'cfg(not(windows))'.dependencies]
+tikv-jemallocator = "0.5"
+
 [dev-dependencies]
-common-test-util = { path = "../common/test-util" }
-rexpect = "0.5"
-temp-env = "0.3"
+common-test-util = { workspace = true }
 serde.workspace = true
-toml = "0.5"
+temp-env = "0.3"
+toml.workspace = true
+
+[target.'cfg(not(windows))'.dev-dependencies]
+rexpect = "0.5"
 
 [build-dependencies]
-build-data = "0.1.3"
+common-version = { workspace = true }

src/cmd/build.rs

@@ -12,22 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-const DEFAULT_VALUE: &str = "unknown";
 fn main() {
-    println!(
-        "cargo:rustc-env=GIT_COMMIT={}",
-        build_data::get_git_commit().unwrap_or_else(|_| DEFAULT_VALUE.to_string())
-    );
-    println!(
-        "cargo:rustc-env=GIT_COMMIT_SHORT={}",
-        build_data::get_git_commit_short().unwrap_or_else(|_| DEFAULT_VALUE.to_string())
-    );
-    println!(
-        "cargo:rustc-env=GIT_BRANCH={}",
-        build_data::get_git_branch().unwrap_or_else(|_| DEFAULT_VALUE.to_string())
-    );
-    println!(
-        "cargo:rustc-env=GIT_DIRTY={}",
-        build_data::get_git_dirty().map_or(DEFAULT_VALUE.to_string(), |v| v.to_string())
-    );
+    common_version::setup_git_versions();
 }